1 """ File defining the importer class and all the methods needed to import
2 the nodes, users and slices from OAR and LDAP to the SFA database.
3 Also creates the iotlab specific databse and table to keep track
4 of which slice hrn contains which job.
6 from sfa.util.config import Config
7 from sfa.generic import Generic
8 from sfa.util.xrn import Xrn, get_authority, hrn_to_urn
9 from sfa.iotlab.iotlabshell import IotlabShell
10 # from sfa.iotlab.iotlabdriver import IotlabDriver
11 from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB
12 from sfa.trust.certificate import Keypair, convert_public_key
13 from sfa.trust.gid import create_uuid
15 # using global alchemy.session() here is fine
16 # as importer is on standalone one-shot process
17 from sfa.storage.alchemy import global_dbsession
18 from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, \
22 from sqlalchemy.exc import SQLAlchemyError
27 IotlabImporter class, generic importer_class. Used to populate the SFA DB
28 with iotlab resources' records.
29 Used to update records when new resources, users or nodes, are added
33 def __init__(self, auth_hierarchy, loc_logger):
35 Sets and defines import logger and the authority name. Gathers all the
36 records already registerd in the SFA DB, broke them into 3 dicts, by
37 type and hrn, by email and by type and pointer.
39 :param auth_hierarchy: authority name
40 :type auth_hierarchy: string
41 :param loc_logger: local logger
42 :type loc_logger: _SfaLogger
45 self.auth_hierarchy = auth_hierarchy
46 self.logger = loc_logger
47 self.logger.setLevelDebug()
48 #retrieve all existing SFA objects
49 self.all_records = global_dbsession.query(RegRecord).all()
51 # initialize record.stale to True by default,
52 # then mark stale=False on the ones that are in use
53 for record in self.all_records:
55 #create hash by (type,hrn)
56 #used to know if a given record is already known to SFA
57 self.records_by_type_hrn = \
58 dict([((record.type, record.hrn), record)
59 for record in self.all_records])
61 self.users_rec_by_email = \
62 dict([(record.email, record)
63 for record in self.all_records if record.type == 'user'])
65 # create hash by (type,pointer)
66 self.records_by_type_pointer = \
67 dict([((str(record.type), record.pointer), record)
68 for record in self.all_records if record.pointer != -1])
72 def hostname_to_hrn_escaped(root_auth, hostname):
75 Returns a node's hrn based on its hostname and the root authority and by
76 removing special caracters from the hostname.
78 :param root_auth: root authority name
79 :param hostname: nodes's hostname
80 :type root_auth: string
81 :type hostname: string
84 return '.'.join([root_auth, Xrn.escape(hostname)])
88 def slicename_to_hrn(person_hrn):
91 Returns the slicename associated to a given person's hrn.
93 :param person_hrn: user's hrn
94 :type person_hrn: string
97 return (person_hrn + '_slice')
99 def add_options(self, parser):
101 .. warning:: not used
103 # we don't have any options for now
106 def find_record_by_type_hrn(self, record_type, hrn):
108 Finds the record associated with the hrn and its type given in parameter
109 if the tuple (hrn, type hrn) is an existing key in the dictionary.
111 :param record_type: the record's type (slice, node, authority...)
112 :type record_type: string
113 :param hrn: Human readable name of the object's record
115 :returns: Returns the record associated with a given hrn and hrn type.
116 Returns None if the key tuple is not in the dictionary.
117 :rtype: RegUser if user, RegSlice if slice, RegNode if node...or None if
118 record does not exist.
121 return self.records_by_type_hrn.get((record_type, hrn), None)
123 def locate_by_type_pointer(self, record_type, pointer):
126 Returns the record corresponding to the key pointer and record
127 type. Returns None if the record does not exist and is not in the
128 records_by_type_pointer dictionnary.
130 :param record_type: the record's type (slice, node, authority...)
131 :type record_type: string
132 :param pointer:Pointer to where the record is in the origin db,
133 used in case the record comes from a trusted authority.
134 :type pointer: integer
135 :rtype: RegUser if user, RegSlice if slice, RegNode if node...
136 or None if record does not exist.
138 return self.records_by_type_pointer.get((record_type, pointer), None)
141 def update_just_added_records_dict(self, record):
144 Updates the records_by_type_hrn dictionnary if the record has
147 :param record: Record to add in the records_by_type_hrn dict.
148 :type record: dictionary
150 rec_tuple = (record.type, record.hrn)
151 if rec_tuple in self.records_by_type_hrn:
152 self.logger.warning("IotlabImporter.update_just_added_records_dict:\
153 duplicate (%s,%s)" % rec_tuple)
155 self.records_by_type_hrn[rec_tuple] = record
158 def import_nodes(self, site_node_ids, nodes_by_id, testbed_shell):
161 Creates appropriate hostnames and RegNode records for each node in
162 site_node_ids, based on the information given by the dict nodes_by_id
163 that was made from data from OAR. Saves the records to the DB.
165 :param site_node_ids: site's node ids
166 :type site_node_ids: list of integers
167 :param nodes_by_id: dictionary , key is the node id, value is the a dict
168 with node information.
169 :type nodes_by_id: dictionary
170 :param testbed_shell: IotlabDriver object, used to have access to
171 testbed_shell attributes.
172 :type testbed_shell: IotlabDriver
179 for node_id in site_node_ids:
181 node = nodes_by_id[node_id]
183 self.logger.warning("IotlabImporter: cannot find node_id %s \
184 - ignored" % (node_id))
187 self.hostname_to_hrn_escaped(testbed_shell.root_auth,
189 self.logger.info("IOTLABIMPORTER node %s " % (node))
192 # xxx this sounds suspicious
195 node_record = self.find_record_by_type_hrn('node', hrn)
197 pkey = Keypair(create=True)
198 urn = hrn_to_urn(escaped_hrn, 'node')
200 self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
202 def iotlab_get_authority(hrn):
203 """ Gets the authority part in the hrn.
204 :param hrn: hrn whose authority we are looking for.
206 :returns: splits the hrn using the '.' separator and returns
207 the authority part of the hrn.
211 return hrn.split(".")[0]
213 node_record = RegNode(hrn=hrn, gid=node_gid,
215 authority=iotlab_get_authority(hrn))
218 node_record.just_created()
219 global_dbsession.add(node_record)
220 global_dbsession.commit()
221 self.logger.info("IotlabImporter: imported node: %s"
223 self.update_just_added_records_dict(node_record)
224 except SQLAlchemyError:
225 self.logger.log_exc("IotlabImporter: failed to import node")
227 #TODO: xxx update the record ...
229 node_record.stale = False
231 def import_sites_and_nodes(self, testbed_shell):
234 Gets all the sites and nodes from OAR, process the information,
235 creates hrns and RegAuthority for sites, and feed them to the database.
236 For each site, import the site's nodes to the DB by calling
239 :param testbed_shell: IotlabDriver object, used to have access to
240 testbed_shell methods and fetching info on sites and nodes.
241 :type testbed_shell: IotlabDriver
244 sites_listdict = testbed_shell.GetSites()
245 nodes_listdict = testbed_shell.GetNodes()
246 nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict])
247 for site in sites_listdict:
248 site_hrn = site['name']
249 site_record = self.find_record_by_type_hrn ('authority', site_hrn)
250 self.logger.info("IotlabImporter: import_sites_and_nodes \
251 (site) %s \r\n " % site_record)
254 urn = hrn_to_urn(site_hrn, 'authority')
255 if not self.auth_hierarchy.auth_exists(urn):
256 self.auth_hierarchy.create_auth(urn)
258 auth_info = self.auth_hierarchy.get_auth_info(urn)
260 RegAuthority(hrn=site_hrn,
261 gid=auth_info.get_gid_object(),
263 authority=get_authority(site_hrn))
264 site_record.just_created()
265 global_dbsession.add(site_record)
266 global_dbsession.commit()
267 self.logger.info("IotlabImporter: imported authority \
268 (site) %s" % site_record)
269 self.update_just_added_records_dict(site_record)
270 except SQLAlchemyError:
271 # if the site import fails then there is no point in
272 # trying to import the
273 # site's child records(node, slices, persons), so skip them.
274 self.logger.log_exc("IotlabImporter: failed to import \
275 site. Skipping child records")
278 # xxx update the record ...
281 site_record.stale = False
282 self.import_nodes(site['node_ids'], nodes_by_id, testbed_shell)
288 def init_person_key(self, person, iotlab_key):
291 Returns a tuple pubkey and pkey.
293 :param person Person's data.
295 :param iotlab_key: SSH public key, from LDAP user's data.
297 :type iotlab_key: string
298 :rtype (string, Keypair)
303 # randomly pick first key in set
307 pkey = convert_public_key(pubkey)
309 #key not good. create another pkey
310 self.logger.warn("IotlabImporter: \
311 unable to convert public \
312 key for %s" % person['hrn'])
313 pkey = Keypair(create=True)
316 # the user has no keys.
317 #Creating a random keypair for the user's gid
318 self.logger.warn("IotlabImporter: person %s does not have a \
319 public key" % (person['hrn']))
320 pkey = Keypair(create=True)
321 return (pubkey, pkey)
323 def import_persons_and_slices(self, testbed_shell):
326 Gets user data from LDAP, process the information.
327 Creates hrn for the user's slice, the user's gid, creates
328 the RegUser record associated with user. Creates the RegKey record
329 associated nwith the user's key.
330 Saves those records into the SFA DB.
331 import the user's slice onto the database as well by calling
334 :param testbed_shell: IotlabDriver object, used to have access to
335 testbed_shell attributes.
336 :type testbed_shell: IotlabDriver
338 .. warning:: does not support multiple keys per user
340 ldap_person_listdict = testbed_shell.GetPersons()
341 self.logger.info("IOTLABIMPORT \t ldap_person_listdict %s \r\n"
342 % (ldap_person_listdict))
345 for person in ldap_person_listdict:
347 self.logger.info("IotlabImporter: person :" % (person))
348 if 'ssh-rsa' not in person['pkey']:
349 #people with invalid ssh key (ssh-dss, empty, bullshit keys...)
352 person_hrn = person['hrn']
353 slice_hrn = self.slicename_to_hrn(person['hrn'])
355 # xxx suspicious again
356 if len(person_hrn) > 64:
357 person_hrn = person_hrn[:64]
358 person_urn = hrn_to_urn(person_hrn, 'user')
361 self.logger.info("IotlabImporter: users_rec_by_email %s "
362 % (self.users_rec_by_email))
364 #Check if user using person['email'] from LDAP is already registered
365 #in SFA. One email = one person. In this case, do not create another
366 #record for this person
367 #person_hrn returned by GetPerson based on iotlab root auth +
369 user_record = self.find_record_by_type_hrn('user', person_hrn)
371 if not user_record and person['email'] in self.users_rec_by_email:
372 user_record = self.users_rec_by_email[person['email']]
373 person_hrn = user_record.hrn
374 person_urn = hrn_to_urn(person_hrn, 'user')
377 slice_record = self.find_record_by_type_hrn('slice', slice_hrn)
379 iotlab_key = person['pkey']
382 (pubkey, pkey) = self.init_person_key(person, iotlab_key)
383 if pubkey is not None and pkey is not None:
385 self.auth_hierarchy.create_gid(person_urn,
388 self.logger.debug("IOTLAB IMPORTER \
389 PERSON EMAIL OK email %s " % (person['email']))
390 person_gid.set_email(person['email'])
392 RegUser(hrn=person_hrn,
395 authority=get_authority(person_hrn),
396 email=person['email'])
399 RegUser(hrn=person_hrn,
402 authority=get_authority(person_hrn))
405 user_record.reg_keys = [RegKey(pubkey)]
407 self.logger.warning("No key found for user %s"
411 user_record.just_created()
412 global_dbsession.add (user_record)
413 global_dbsession.commit()
414 self.logger.info("IotlabImporter: imported person \
416 self.update_just_added_records_dict(user_record)
418 except SQLAlchemyError:
419 self.logger.log_exc("IotlabImporter: \
420 failed to import person %s" % (person))
422 # update the record ?
423 # if user's primary key has changed then we need to update
424 # the users gid by forcing an update here
425 sfa_keys = user_record.reg_keys
428 if iotlab_key is not sfa_keys:
431 self.logger.info("IotlabImporter: \t \t USER UPDATE \
432 person: %s" % (person['hrn']))
433 (pubkey, pkey) = self.init_person_key(person, iotlab_key)
435 self.auth_hierarchy.create_gid(person_urn,
438 user_record.reg_keys = []
440 user_record.reg_keys = [RegKey(pubkey)]
441 self.logger.info("IotlabImporter: updated person: %s"
445 user_record.email = person['email']
448 global_dbsession.commit()
449 user_record.stale = False
450 except SQLAlchemyError:
451 self.logger.log_exc("IotlabImporter: \
452 failed to update person %s"% (person))
454 self.import_slice(slice_hrn, slice_record, user_record)
457 def import_slice(self, slice_hrn, slice_record, user_record):
460 Create RegSlice record according to the slice hrn if the slice
461 does not exist yet.Creates a relationship with the user record
462 associated with the slice.
463 Commit the record to the database.
466 :param slice_hrn: Human readable name of the slice.
467 :type slice_hrn: string
468 :param slice_record: record of the slice found in the DB, if any.
469 :type slice_record: RegSlice or None
470 :param user_record: user record found in the DB if any.
471 :type user_record: RegUser
473 .. todo::Update the record if a slice record already exists.
476 pkey = Keypair(create=True)
477 urn = hrn_to_urn(slice_hrn, 'slice')
479 self.auth_hierarchy.create_gid(urn,
481 slice_record = RegSlice(hrn=slice_hrn, gid=slice_gid,
483 authority=get_authority(slice_hrn))
485 slice_record.just_created()
486 global_dbsession.add(slice_record)
487 global_dbsession.commit()
490 self.update_just_added_records_dict(slice_record)
492 except SQLAlchemyError:
493 self.logger.log_exc("IotlabImporter: failed to import slice")
495 #No slice update upon import in iotlab
497 # xxx update the record ...
498 self.logger.warning("Slice update not yet implemented")
500 # record current users affiliated with the slice
503 slice_record.reg_researchers = [user_record]
505 global_dbsession.commit()
506 slice_record.stale = False
507 except SQLAlchemyError:
508 self.logger.log_exc("IotlabImporter: failed to update slice")
511 def run(self, options):
513 Create the special iotlab table, lease_table, in the iotlab database.
514 Import everything (users, slices, nodes and sites from OAR
515 and LDAP) into the SFA database.
516 Delete stale records that are no longer in OAR or LDAP.
522 interface_hrn = config.SFA_INTERFACE_HRN
523 root_auth = config.SFA_REGISTRY_ROOT_AUTH
524 api = Generic.the_flavour().make_api(interface='registry')
525 testbed_shell = IotlabShell(api)
526 leases_db = TestbedAdditionalSfaDB(config)
527 #Create special slice table for iotlab
529 if not leases_db.exists('lease_table'):
530 leases_db.createtable()
531 self.logger.info("IotlabImporter.run: lease_table table created ")
533 # import site and node records in site into the SFA db.
534 self.import_sites_and_nodes(testbed_shell)
535 #import users and slice into the SFA DB.
536 self.import_persons_and_slices(testbed_shell)
538 ### remove stale records
539 # special records must be preserved
540 system_hrns = [interface_hrn, root_auth,
541 interface_hrn + '.slicemanager']
542 for record in self.all_records:
543 if record.hrn in system_hrns:
545 if record.peer_authority:
548 for record in self.all_records:
549 if record.type == 'user':
550 self.logger.info("IotlabImporter: stale records: hrn %s %s"
551 % (record.hrn, record.stale))
556 self.logger.warning("stale not found with %s" % record)
558 self.logger.info("IotlabImporter: deleting stale record: %s"
562 global_dbsession.delete(record)
563 global_dbsession.commit()
564 except SQLAlchemyError:
565 self.logger.log_exc("IotlabImporter: failed to delete \
566 stale record %s" % (record))