1 """ File defining the importer class and all the methods needed to import
2 the nodes, users and slices from OAR and LDAP to the SFA database.
3 Also creates the iotlab specific databse and table to keep track
4 of which slice hrn contains which job.
6 from sfa.util.config import Config
7 from sfa.util.xrn import Xrn, get_authority, hrn_to_urn
8 from sfa.iotlab.iotlabshell import IotlabShell
9 # from sfa.iotlab.iotlabdriver import IotlabDriver
10 from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB
11 from sfa.trust.certificate import Keypair, convert_public_key
12 from sfa.trust.gid import create_uuid
14 # using global alchemy.session() here is fine
15 # as importer is on standalone one-shot process
16 from sfa.storage.alchemy import global_dbsession
17 from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, \
21 from sqlalchemy.exc import SQLAlchemyError
26 IotlabImporter class, generic importer_class. Used to populate the SFA DB
27 with iotlab resources' records.
28 Used to update records when new resources, users or nodes, are added
32 def __init__(self, auth_hierarchy, loc_logger):
34 Sets and defines import logger and the authority name. Gathers all the
35 records already registerd in the SFA DB, broke them into 3 dicts, by
36 type and hrn, by email and by type and pointer.
38 :param auth_hierarchy: authority name
39 :type auth_hierarchy: string
40 :param loc_logger: local logger
41 :type loc_logger: _SfaLogger
44 self.auth_hierarchy = auth_hierarchy
45 self.logger = loc_logger
46 self.logger.setLevelDebug()
47 #retrieve all existing SFA objects
48 self.all_records = global_dbsession.query(RegRecord).all()
50 # initialize record.stale to True by default,
51 # then mark stale=False on the ones that are in use
52 for record in self.all_records:
54 #create hash by (type,hrn)
55 #used to know if a given record is already known to SFA
56 self.records_by_type_hrn = \
57 dict([((record.type, record.hrn), record)
58 for record in self.all_records])
60 self.users_rec_by_email = \
61 dict([(record.email, record)
62 for record in self.all_records if record.type == 'user'])
64 # create hash by (type,pointer)
65 self.records_by_type_pointer = \
66 dict([((str(record.type), record.pointer), record)
67 for record in self.all_records if record.pointer != -1])
71 def hostname_to_hrn_escaped(root_auth, hostname):
74 Returns a node's hrn based on its hostname and the root authority and by
75 removing special caracters from the hostname.
77 :param root_auth: root authority name
78 :param hostname: nodes's hostname
79 :type root_auth: string
80 :type hostname: string
83 return '.'.join([root_auth, Xrn.escape(hostname)])
87 def slicename_to_hrn(person_hrn):
90 Returns the slicename associated to a given person's hrn.
92 :param person_hrn: user's hrn
93 :type person_hrn: string
96 return (person_hrn + '_slice')
98 def add_options(self, parser):
100 .. warning:: not used
102 # we don't have any options for now
105 def find_record_by_type_hrn(self, record_type, hrn):
107 Finds the record associated with the hrn and its type given in parameter
108 if the tuple (hrn, type hrn) is an existing key in the dictionary.
110 :param record_type: the record's type (slice, node, authority...)
111 :type record_type: string
112 :param hrn: Human readable name of the object's record
114 :returns: Returns the record associated with a given hrn and hrn type.
115 Returns None if the key tuple is not in the dictionary.
116 :rtype: RegUser if user, RegSlice if slice, RegNode if node...or None if
117 record does not exist.
120 return self.records_by_type_hrn.get((record_type, hrn), None)
122 def locate_by_type_pointer(self, record_type, pointer):
125 Returns the record corresponding to the key pointer and record
126 type. Returns None if the record does not exist and is not in the
127 records_by_type_pointer dictionnary.
129 :param record_type: the record's type (slice, node, authority...)
130 :type record_type: string
131 :param pointer:Pointer to where the record is in the origin db,
132 used in case the record comes from a trusted authority.
133 :type pointer: integer
134 :rtype: RegUser if user, RegSlice if slice, RegNode if node...
135 or None if record does not exist.
137 return self.records_by_type_pointer.get((record_type, pointer), None)
140 def update_just_added_records_dict(self, record):
143 Updates the records_by_type_hrn dictionnary if the record has
146 :param record: Record to add in the records_by_type_hrn dict.
147 :type record: dictionary
149 rec_tuple = (record.type, record.hrn)
150 if rec_tuple in self.records_by_type_hrn:
151 self.logger.warning("IotlabImporter.update_just_added_records_dict:\
152 duplicate (%s,%s)" % rec_tuple)
154 self.records_by_type_hrn[rec_tuple] = record
157 def import_nodes(self, site_node_ids, nodes_by_id, testbed_shell):
160 Creates appropriate hostnames and RegNode records for each node in
161 site_node_ids, based on the information given by the dict nodes_by_id
162 that was made from data from OAR. Saves the records to the DB.
164 :param site_node_ids: site's node ids
165 :type site_node_ids: list of integers
166 :param nodes_by_id: dictionary , key is the node id, value is the a dict
167 with node information.
168 :type nodes_by_id: dictionary
169 :param testbed_shell: IotlabDriver object, used to have access to
170 testbed_shell attributes.
171 :type testbed_shell: IotlabDriver
178 for node_id in site_node_ids:
180 node = nodes_by_id[node_id]
182 self.logger.warning("IotlabImporter: cannot find node_id %s \
183 - ignored" % (node_id))
186 self.hostname_to_hrn_escaped(testbed_shell.root_auth,
188 self.logger.info("IOTLABIMPORTER node %s " % (node))
191 # xxx this sounds suspicious
194 node_record = self.find_record_by_type_hrn('node', hrn)
196 pkey = Keypair(create=True)
197 urn = hrn_to_urn(escaped_hrn, 'node')
199 self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
201 def iotlab_get_authority(hrn):
202 """ Gets the authority part in the hrn.
203 :param hrn: hrn whose authority we are looking for.
205 :returns: splits the hrn using the '.' separator and returns
206 the authority part of the hrn.
210 return hrn.split(".")[0]
212 node_record = RegNode(hrn=hrn, gid=node_gid,
214 authority=iotlab_get_authority(hrn))
217 node_record.just_created()
218 global_dbsession.add(node_record)
219 global_dbsession.commit()
220 self.logger.info("IotlabImporter: imported node: %s"
222 self.update_just_added_records_dict(node_record)
223 except SQLAlchemyError:
224 self.logger.log_exc("IotlabImporter: failed to import node")
226 #TODO: xxx update the record ...
228 node_record.stale = False
230 def import_sites_and_nodes(self, testbed_shell):
233 Gets all the sites and nodes from OAR, process the information,
234 creates hrns and RegAuthority for sites, and feed them to the database.
235 For each site, import the site's nodes to the DB by calling
238 :param testbed_shell: IotlabDriver object, used to have access to
239 testbed_shell methods and fetching info on sites and nodes.
240 :type testbed_shell: IotlabDriver
243 sites_listdict = testbed_shell.GetSites()
244 nodes_listdict = testbed_shell.GetNodes()
245 nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict])
246 for site in sites_listdict:
247 site_hrn = site['name']
248 site_record = self.find_record_by_type_hrn ('authority', site_hrn)
249 self.logger.info("IotlabImporter: import_sites_and_nodes \
250 (site) %s \r\n " % site_record)
253 urn = hrn_to_urn(site_hrn, 'authority')
254 if not self.auth_hierarchy.auth_exists(urn):
255 self.auth_hierarchy.create_auth(urn)
257 auth_info = self.auth_hierarchy.get_auth_info(urn)
259 RegAuthority(hrn=site_hrn,
260 gid=auth_info.get_gid_object(),
262 authority=get_authority(site_hrn))
263 site_record.just_created()
264 global_dbsession.add(site_record)
265 global_dbsession.commit()
266 self.logger.info("IotlabImporter: imported authority \
267 (site) %s" % site_record)
268 self.update_just_added_records_dict(site_record)
269 except SQLAlchemyError:
270 # if the site import fails then there is no point in
271 # trying to import the
272 # site's child records(node, slices, persons), so skip them.
273 self.logger.log_exc("IotlabImporter: failed to import \
274 site. Skipping child records")
277 # xxx update the record ...
280 site_record.stale = False
281 self.import_nodes(site['node_ids'], nodes_by_id, testbed_shell)
287 def init_person_key(self, person, iotlab_key):
290 Returns a tuple pubkey and pkey.
292 :param person Person's data.
294 :param iotlab_key: SSH public key, from LDAP user's data.
296 :type iotlab_key: string
297 :rtype (string, Keypair)
302 # randomly pick first key in set
306 pkey = convert_public_key(pubkey)
308 #key not good. create another pkey
309 self.logger.warn("IotlabImporter: \
310 unable to convert public \
311 key for %s" % person['hrn'])
312 pkey = Keypair(create=True)
315 # the user has no keys.
316 #Creating a random keypair for the user's gid
317 self.logger.warn("IotlabImporter: person %s does not have a \
318 public key" % (person['hrn']))
319 pkey = Keypair(create=True)
320 return (pubkey, pkey)
322 def import_persons_and_slices(self, testbed_shell):
325 Gets user data from LDAP, process the information.
326 Creates hrn for the user's slice, the user's gid, creates
327 the RegUser record associated with user. Creates the RegKey record
328 associated nwith the user's key.
329 Saves those records into the SFA DB.
330 import the user's slice onto the database as well by calling
333 :param testbed_shell: IotlabDriver object, used to have access to
334 testbed_shell attributes.
335 :type testbed_shell: IotlabDriver
337 ldap_person_listdict = testbed_shell.GetPersons()
338 self.logger.info("IOTLABIMPORT \t ldap_person_listdict %s \r\n"
339 % (ldap_person_listdict))
342 for person in ldap_person_listdict:
344 self.logger.info("IotlabImporter: person :" % (person))
345 if 'ssh-rsa' not in person['pkey']:
346 #people with invalid ssh key (ssh-dss, empty, bullshit keys...)
349 person_hrn = person['hrn']
350 slice_hrn = self.slicename_to_hrn(person['hrn'])
352 # xxx suspicious again
353 if len(person_hrn) > 64:
354 person_hrn = person_hrn[:64]
355 person_urn = hrn_to_urn(person_hrn, 'user')
358 self.logger.info("IotlabImporter: users_rec_by_email %s "
359 % (self.users_rec_by_email))
361 #Check if user using person['email'] from LDAP is already registered
362 #in SFA. One email = one person. In this case, do not create another
363 #record for this person
364 #person_hrn returned by GetPerson based on iotlab root auth +
366 user_record = self.find_record_by_type_hrn('user', person_hrn)
368 if not user_record and person['email'] in self.users_rec_by_email:
369 user_record = self.users_rec_by_email[person['email']]
370 person_hrn = user_record.hrn
371 person_urn = hrn_to_urn(person_hrn, 'user')
374 slice_record = self.find_record_by_type_hrn('slice', slice_hrn)
376 iotlab_key = person['pkey']
379 (pubkey, pkey) = self.init_person_key(person, iotlab_key)
380 if pubkey is not None and pkey is not None:
382 self.auth_hierarchy.create_gid(person_urn,
385 self.logger.debug("IOTLAB IMPORTER \
386 PERSON EMAIL OK email %s " % (person['email']))
387 person_gid.set_email(person['email'])
389 RegUser(hrn=person_hrn,
392 authority=get_authority(person_hrn),
393 email=person['email'])
396 RegUser(hrn=person_hrn,
399 authority=get_authority(person_hrn))
402 user_record.reg_keys = [RegKey(pubkey)]
404 self.logger.warning("No key found for user %s"
408 user_record.just_created()
409 global_dbsession.add (user_record)
410 global_dbsession.commit()
411 self.logger.info("IotlabImporter: imported person \
413 self.update_just_added_records_dict(user_record)
415 except SQLAlchemyError:
416 self.logger.log_exc("IotlabImporter: \
417 failed to import person %s" % (person))
419 # update the record ?
420 # if user's primary key has changed then we need to update
421 # the users gid by forcing an update here
422 sfa_keys = user_record.reg_keys
425 if iotlab_key is not sfa_keys:
428 self.logger.info("IotlabImporter: \t \t USER UPDATE \
429 person: %s" % (person['hrn']))
430 (pubkey, pkey) = self.init_person_key(person, iotlab_key)
432 self.auth_hierarchy.create_gid(person_urn,
435 user_record.reg_keys = []
437 user_record.reg_keys = [RegKey(pubkey)]
438 self.logger.info("IotlabImporter: updated person: %s"
442 user_record.email = person['email']
445 global_dbsession.commit()
446 user_record.stale = False
447 except SQLAlchemyError:
448 self.logger.log_exc("IotlabImporter: \
449 failed to update person %s"% (person))
451 self.import_slice(slice_hrn, slice_record, user_record)
454 def import_slice(self, slice_hrn, slice_record, user_record):
457 Create RegSlice record according to the slice hrn if the slice
458 does not exist yet.Creates a relationship with the user record
459 associated with the slice.
460 Commit the record to the database.
463 :param slice_hrn: Human readable name of the slice.
464 :type slice_hrn: string
465 :param slice_record: record of the slice found in the DB, if any.
466 :type slice_record: RegSlice or None
467 :param user_record: user record found in the DB if any.
468 :type user_record: RegUser
470 .. todo::Update the record if a slice record already exists.
473 pkey = Keypair(create=True)
474 urn = hrn_to_urn(slice_hrn, 'slice')
476 self.auth_hierarchy.create_gid(urn,
478 slice_record = RegSlice(hrn=slice_hrn, gid=slice_gid,
480 authority=get_authority(slice_hrn))
482 slice_record.just_created()
483 global_dbsession.add(slice_record)
484 global_dbsession.commit()
487 self.update_just_added_records_dict(slice_record)
489 except SQLAlchemyError:
490 self.logger.log_exc("IotlabImporter: failed to import slice")
492 #No slice update upon import in iotlab
494 # xxx update the record ...
495 self.logger.warning("Slice update not yet implemented")
497 # record current users affiliated with the slice
500 slice_record.reg_researchers = [user_record]
502 global_dbsession.commit()
503 slice_record.stale = False
504 except SQLAlchemyError:
505 self.logger.log_exc("IotlabImporter: failed to update slice")
508 def run(self, options):
510 Create the special iotlab table, testbed_xp, in the iotlab database.
511 Import everything (users, slices, nodes and sites from OAR
512 and LDAP) into the SFA database.
513 Delete stale records that are no longer in OAR or LDAP.
519 interface_hrn = config.SFA_INTERFACE_HRN
520 root_auth = config.SFA_REGISTRY_ROOT_AUTH
522 testbed_shell = IotlabShell(config)
523 leases_db = TestbedAdditionalSfaDB(config)
524 #Create special slice table for iotlab
526 if not leases_db.exists('testbed_xp'):
527 leases_db.createtable()
528 self.logger.info("IotlabImporter.run: testbed_xp table created ")
530 # import site and node records in site into the SFA db.
531 self.import_sites_and_nodes(testbed_shell)
532 #import users and slice into the SFA DB.
533 self.import_persons_and_slices(testbed_shell)
535 ### remove stale records
536 # special records must be preserved
537 system_hrns = [interface_hrn, root_auth,
538 interface_hrn + '.slicemanager']
539 for record in self.all_records:
540 if record.hrn in system_hrns:
542 if record.peer_authority:
545 for record in self.all_records:
546 if record.type == 'user':
547 self.logger.info("IotlabImporter: stale records: hrn %s %s"
548 % (record.hrn, record.stale))
553 self.logger.warning("stale not found with %s" % record)
555 self.logger.info("IotlabImporter: deleting stale record: %s"
559 global_dbsession.delete(record)
560 global_dbsession.commit()
561 except SQLAlchemyError:
562 self.logger.log_exc("IotlabImporter: failed to delete \
563 stale record %s" % (record))