1 """ File defining the importer class and all the methods needed to import
2 the nodes, users and slices from OAR and LDAP to the SFA database.
3 Also creates the iotlab specific databse and table to keep track
4 of which slice hrn contains which job.
6 from sfa.util.config import Config
7 from sfa.util.xrn import Xrn, get_authority, hrn_to_urn
8 from sfa.iotlab.iotlabshell import IotlabShell
9 # from sfa.iotlab.iotlabdriver import IotlabDriver
10 # from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB
11 from sfa.trust.certificate import Keypair, convert_public_key
12 from sfa.trust.gid import create_uuid
14 # using global alchemy.session() here is fine
15 # as importer is on standalone one-shot process
17 from sfa.storage.alchemy import global_dbsession, engine
18 from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, \
19 RegUser, RegKey, init_tables
21 from sqlalchemy import Table, MetaData
22 from sqlalchemy.exc import SQLAlchemyError, NoSuchTableError
28 IotlabImporter class, generic importer_class. Used to populate the SFA DB
29 with iotlab resources' records.
30 Used to update records when new resources, users or nodes, are added
34 def __init__(self, auth_hierarchy, loc_logger):
36 Sets and defines import logger and the authority name. Gathers all the
37 records already registerd in the SFA DB, broke them into 3 dicts, by
38 type and hrn, by email and by type and pointer.
40 :param auth_hierarchy: authority name
41 :type auth_hierarchy: string
42 :param loc_logger: local logger
43 :type loc_logger: _SfaLogger
46 self.auth_hierarchy = auth_hierarchy
47 self.logger = loc_logger
48 self.logger.setLevelDebug()
50 #retrieve all existing SFA objects
51 self.all_records = global_dbsession.query(RegRecord).all()
53 # initialize record.stale to True by default,
54 # then mark stale=False on the ones that are in use
55 for record in self.all_records:
57 #create hash by (type,hrn)
58 #used to know if a given record is already known to SFA
59 self.records_by_type_hrn = \
60 dict([((record.type, record.hrn), record)
61 for record in self.all_records])
63 self.users_rec_by_email = \
64 dict([(record.email, record)
65 for record in self.all_records if record.type == 'user'])
67 # create hash by (type,pointer)
68 self.records_by_type_pointer = \
69 dict([((str(record.type), record.pointer), record)
70 for record in self.all_records if record.pointer != -1])
74 def exists(self, tablename, engine):
76 Checks if the table specified as tablename exists.
77 :param tablename: name of the table in the db that has to be checked.
78 :type tablename: string
79 :returns: True if the table exists, False otherwise.
83 metadata = MetaData(bind=engine)
85 table = Table(tablename, metadata, autoload=True)
88 except NoSuchTableError:
89 self.logger.log_exc("Iotlabimporter tablename %s does not exist"
95 def hostname_to_hrn_escaped(root_auth, hostname):
98 Returns a node's hrn based on its hostname and the root authority and by
99 removing special caracters from the hostname.
101 :param root_auth: root authority name
102 :param hostname: nodes's hostname
103 :type root_auth: string
104 :type hostname: string
107 return '.'.join([root_auth, Xrn.escape(hostname)])
111 def slicename_to_hrn(person_hrn):
114 Returns the slicename associated to a given person's hrn.
116 :param person_hrn: user's hrn
117 :type person_hrn: string
120 return (person_hrn + '_slice')
122 def add_options(self, parser):
124 .. warning:: not used
126 # we don't have any options for now
129 def find_record_by_type_hrn(self, record_type, hrn):
131 Finds the record associated with the hrn and its type given in parameter
132 if the tuple (hrn, type hrn) is an existing key in the dictionary.
134 :param record_type: the record's type (slice, node, authority...)
135 :type record_type: string
136 :param hrn: Human readable name of the object's record
138 :returns: Returns the record associated with a given hrn and hrn type.
139 Returns None if the key tuple is not in the dictionary.
140 :rtype: RegUser if user, RegSlice if slice, RegNode if node...or None if
141 record does not exist.
144 return self.records_by_type_hrn.get((record_type, hrn), None)
146 def locate_by_type_pointer(self, record_type, pointer):
149 Returns the record corresponding to the key pointer and record
150 type. Returns None if the record does not exist and is not in the
151 records_by_type_pointer dictionnary.
153 :param record_type: the record's type (slice, node, authority...)
154 :type record_type: string
155 :param pointer:Pointer to where the record is in the origin db,
156 used in case the record comes from a trusted authority.
157 :type pointer: integer
158 :rtype: RegUser if user, RegSlice if slice, RegNode if node...
159 or None if record does not exist.
161 return self.records_by_type_pointer.get((record_type, pointer), None)
164 def update_just_added_records_dict(self, record):
167 Updates the records_by_type_hrn dictionnary if the record has
170 :param record: Record to add in the records_by_type_hrn dict.
171 :type record: dictionary
173 rec_tuple = (record.type, record.hrn)
174 if rec_tuple in self.records_by_type_hrn:
175 self.logger.warning("IotlabImporter.update_just_added_records_dict:\
176 duplicate (%s,%s)" % rec_tuple)
178 self.records_by_type_hrn[rec_tuple] = record
181 def import_nodes(self, site_node_ids, nodes_by_id, testbed_shell):
184 Creates appropriate hostnames and RegNode records for each node in
185 site_node_ids, based on the information given by the dict nodes_by_id
186 that was made from data from OAR. Saves the records to the DB.
188 :param site_node_ids: site's node ids
189 :type site_node_ids: list of integers
190 :param nodes_by_id: dictionary , key is the node id, value is the a dict
191 with node information.
192 :type nodes_by_id: dictionary
193 :param testbed_shell: IotlabDriver object, used to have access to
194 testbed_shell attributes.
195 :type testbed_shell: IotlabDriver
202 for node_id in site_node_ids:
204 node = nodes_by_id[node_id]
206 self.logger.warning("IotlabImporter: cannot find node_id %s \
207 - ignored" % (node_id))
210 self.hostname_to_hrn_escaped(testbed_shell.root_auth,
212 self.logger.info("IOTLABIMPORTER node %s " % (node))
215 # xxx this sounds suspicious
218 node_record = self.find_record_by_type_hrn('node', hrn)
220 pkey = Keypair(create=True)
221 urn = hrn_to_urn(escaped_hrn, 'node')
223 self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
225 def iotlab_get_authority(hrn):
226 """ Gets the authority part in the hrn.
227 :param hrn: hrn whose authority we are looking for.
229 :returns: splits the hrn using the '.' separator and returns
230 the authority part of the hrn.
234 return hrn.split(".")[0]
236 node_record = RegNode(hrn=hrn, gid=node_gid,
238 authority=iotlab_get_authority(hrn))
241 node_record.just_created()
242 global_dbsession.add(node_record)
243 global_dbsession.commit()
244 self.logger.info("IotlabImporter: imported node: %s"
246 self.update_just_added_records_dict(node_record)
247 except SQLAlchemyError:
248 self.logger.log_exc("IotlabImporter: failed to import node")
250 #TODO: xxx update the record ...
252 node_record.stale = False
254 def import_sites_and_nodes(self, testbed_shell):
257 Gets all the sites and nodes from OAR, process the information,
258 creates hrns and RegAuthority for sites, and feed them to the database.
259 For each site, import the site's nodes to the DB by calling
262 :param testbed_shell: IotlabDriver object, used to have access to
263 testbed_shell methods and fetching info on sites and nodes.
264 :type testbed_shell: IotlabDriver
267 sites_listdict = testbed_shell.GetSites()
268 nodes_listdict = testbed_shell.GetNodes()
269 nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict])
270 for site in sites_listdict:
271 site_hrn = site['name']
272 site_record = self.find_record_by_type_hrn ('authority', site_hrn)
273 self.logger.info("IotlabImporter: import_sites_and_nodes \
274 (site) %s \r\n " % site_record)
277 urn = hrn_to_urn(site_hrn, 'authority')
278 if not self.auth_hierarchy.auth_exists(urn):
279 self.auth_hierarchy.create_auth(urn)
281 auth_info = self.auth_hierarchy.get_auth_info(urn)
283 RegAuthority(hrn=site_hrn,
284 gid=auth_info.get_gid_object(),
286 authority=get_authority(site_hrn))
287 site_record.just_created()
288 global_dbsession.add(site_record)
289 global_dbsession.commit()
290 self.logger.info("IotlabImporter: imported authority \
291 (site) %s" % site_record)
292 self.update_just_added_records_dict(site_record)
293 except SQLAlchemyError:
294 # if the site import fails then there is no point in
295 # trying to import the
296 # site's child records(node, slices, persons), so skip them.
297 self.logger.log_exc("IotlabImporter: failed to import \
298 site. Skipping child records")
301 # xxx update the record ...
304 site_record.stale = False
305 self.import_nodes(site['node_ids'], nodes_by_id, testbed_shell)
311 def init_person_key(self, person, iotlab_key):
314 Returns a tuple pubkey and pkey.
316 :param person Person's data.
318 :param iotlab_key: SSH public key, from LDAP user's data.
320 :type iotlab_key: string
321 :rtype (string, Keypair)
326 # randomly pick first key in set
330 pkey = convert_public_key(pubkey)
332 #key not good. create another pkey
333 self.logger.warn("IotlabImporter: \
334 unable to convert public \
335 key for %s" % person['hrn'])
336 pkey = Keypair(create=True)
339 # the user has no keys.
340 #Creating a random keypair for the user's gid
341 self.logger.warn("IotlabImporter: person %s does not have a \
342 public key" % (person['hrn']))
343 pkey = Keypair(create=True)
344 return (pubkey, pkey)
346 def import_persons_and_slices(self, testbed_shell):
349 Gets user data from LDAP, process the information.
350 Creates hrn for the user's slice, the user's gid, creates
351 the RegUser record associated with user. Creates the RegKey record
352 associated nwith the user's key.
353 Saves those records into the SFA DB.
354 import the user's slice onto the database as well by calling
357 :param testbed_shell: IotlabDriver object, used to have access to
358 testbed_shell attributes.
359 :type testbed_shell: IotlabDriver
361 .. warning:: does not support multiple keys per user
363 ldap_person_listdict = testbed_shell.GetPersons()
364 self.logger.info("IOTLABIMPORT \t ldap_person_listdict %s \r\n"
365 % (ldap_person_listdict))
368 for person in ldap_person_listdict:
370 self.logger.info("IotlabImporter: person :" % (person))
371 if 'ssh-rsa' not in person['pkey']:
372 #people with invalid ssh key (ssh-dss, empty, bullshit keys...)
375 person_hrn = person['hrn']
376 slice_hrn = self.slicename_to_hrn(person['hrn'])
378 # xxx suspicious again
379 if len(person_hrn) > 64:
380 person_hrn = person_hrn[:64]
381 person_urn = hrn_to_urn(person_hrn, 'user')
384 self.logger.info("IotlabImporter: users_rec_by_email %s "
385 % (self.users_rec_by_email))
387 #Check if user using person['email'] from LDAP is already registered
388 #in SFA. One email = one person. In this case, do not create another
389 #record for this person
390 #person_hrn returned by GetPerson based on iotlab root auth +
392 user_record = self.find_record_by_type_hrn('user', person_hrn)
394 if not user_record and person['email'] in self.users_rec_by_email:
395 user_record = self.users_rec_by_email[person['email']]
396 person_hrn = user_record.hrn
397 person_urn = hrn_to_urn(person_hrn, 'user')
400 slice_record = self.find_record_by_type_hrn('slice', slice_hrn)
402 iotlab_key = person['pkey']
405 (pubkey, pkey) = self.init_person_key(person, iotlab_key)
406 if pubkey is not None and pkey is not None:
408 self.auth_hierarchy.create_gid(person_urn,
411 self.logger.debug("IOTLAB IMPORTER \
412 PERSON EMAIL OK email %s " % (person['email']))
413 person_gid.set_email(person['email'])
415 RegUser(hrn=person_hrn,
418 authority=get_authority(person_hrn),
419 email=person['email'])
422 RegUser(hrn=person_hrn,
425 authority=get_authority(person_hrn))
428 user_record.reg_keys = [RegKey(pubkey)]
430 self.logger.warning("No key found for user %s"
434 user_record.just_created()
435 global_dbsession.add (user_record)
436 global_dbsession.commit()
437 self.logger.info("IotlabImporter: imported person \
439 self.update_just_added_records_dict(user_record)
441 except SQLAlchemyError:
442 self.logger.log_exc("IotlabImporter: \
443 failed to import person %s" % (person))
445 # update the record ?
446 # if user's primary key has changed then we need to update
447 # the users gid by forcing an update here
448 sfa_keys = user_record.reg_keys
451 if iotlab_key is not sfa_keys:
454 self.logger.info("IotlabImporter: \t \t USER UPDATE \
455 person: %s" % (person['hrn']))
456 (pubkey, pkey) = self.init_person_key(person, iotlab_key)
458 self.auth_hierarchy.create_gid(person_urn,
461 user_record.reg_keys = []
463 user_record.reg_keys = [RegKey(pubkey)]
464 self.logger.info("IotlabImporter: updated person: %s"
468 user_record.email = person['email']
471 global_dbsession.commit()
472 user_record.stale = False
473 except SQLAlchemyError:
474 self.logger.log_exc("IotlabImporter: \
475 failed to update person %s"% (person))
477 self.import_slice(slice_hrn, slice_record, user_record)
480 def import_slice(self, slice_hrn, slice_record, user_record):
483 Create RegSlice record according to the slice hrn if the slice
484 does not exist yet.Creates a relationship with the user record
485 associated with the slice.
486 Commit the record to the database.
489 :param slice_hrn: Human readable name of the slice.
490 :type slice_hrn: string
491 :param slice_record: record of the slice found in the DB, if any.
492 :type slice_record: RegSlice or None
493 :param user_record: user record found in the DB if any.
494 :type user_record: RegUser
496 .. todo::Update the record if a slice record already exists.
499 pkey = Keypair(create=True)
500 urn = hrn_to_urn(slice_hrn, 'slice')
502 self.auth_hierarchy.create_gid(urn,
504 slice_record = RegSlice(hrn=slice_hrn, gid=slice_gid,
506 authority=get_authority(slice_hrn))
508 slice_record.just_created()
509 global_dbsession.add(slice_record)
510 global_dbsession.commit()
513 self.update_just_added_records_dict(slice_record)
515 except SQLAlchemyError:
516 self.logger.log_exc("IotlabImporter: failed to import slice")
518 #No slice update upon import in iotlab
520 # xxx update the record ...
521 self.logger.warning("Slice update not yet implemented")
523 # record current users affiliated with the slice
526 slice_record.reg_researchers = [user_record]
528 global_dbsession.commit()
529 slice_record.stale = False
530 except SQLAlchemyError:
531 self.logger.log_exc("IotlabImporter: failed to update slice")
534 def run(self, options):
536 Create the special iotlab table, lease_table, in the iotlab database.
537 Import everything (users, slices, nodes and sites from OAR
538 and LDAP) into the SFA database.
539 Delete stale records that are no longer in OAR or LDAP.
545 interface_hrn = config.SFA_INTERFACE_HRN
546 root_auth = config.SFA_REGISTRY_ROOT_AUTH
548 testbed_shell = IotlabShell(config)
549 # leases_db = TestbedAdditionalSfaDB(config)
550 #Create special slice table for iotlab
552 if not self.exists('lease_table', engine):
554 self.logger.info("IotlabImporter.run: lease_table table created ")
556 # import site and node records in site into the SFA db.
557 self.import_sites_and_nodes(testbed_shell)
558 #import users and slice into the SFA DB.
559 self.import_persons_and_slices(testbed_shell)
561 ### remove stale records
562 # special records must be preserved
563 system_hrns = [interface_hrn, root_auth,
564 interface_hrn + '.slicemanager']
565 for record in self.all_records:
566 if record.hrn in system_hrns:
568 if record.peer_authority:
571 for record in self.all_records:
572 if record.type == 'user':
573 self.logger.info("IotlabImporter: stale records: hrn %s %s"
574 % (record.hrn, record.stale))
579 self.logger.warning("stale not found with %s" % record)
581 self.logger.info("IotlabImporter: deleting stale record: %s"
585 global_dbsession.delete(record)
586 global_dbsession.commit()
587 except SQLAlchemyError:
588 self.logger.log_exc("IotlabImporter: failed to delete \
589 stale record %s" % (record))