1 """ File defining the importer class and all the methods needed to import
2 the nodes, users and slices from OAR and LDAP to the SFA database.
3 Also creates the iotlab specific table to keep track
4 of which slice hrn contains which job.
6 from sfa.util.config import Config
7 from sfa.util.xrn import Xrn, get_authority, hrn_to_urn
8 from sfa.iotlab.iotlabshell import IotlabShell
9 # from sfa.iotlab.iotlabdriver import IotlabDriver
10 # from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB
11 from sfa.trust.certificate import Keypair, convert_public_key
12 from sfa.trust.gid import create_uuid
14 # using global alchemy.session() here is fine
15 # as importer is on standalone one-shot process
17 from sfa.storage.alchemy import global_dbsession, engine
18 from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, \
19 RegUser, RegKey, init_tables
21 from sqlalchemy import Table, MetaData
22 from sqlalchemy.exc import SQLAlchemyError, NoSuchTableError
28 IotlabImporter class, generic importer_class. Used to populate the SFA DB
29 with iotlab resources' records.
30 Used to update records when new resources, users or nodes, are added
34 def __init__(self, auth_hierarchy, loc_logger):
36 Sets and defines import logger and the authority name. Gathers all the
37 records already registerd in the SFA DB, broke them into 3 dicts, by
38 type and hrn, by email and by type and pointer.
40 :param auth_hierarchy: authority name
41 :type auth_hierarchy: string
42 :param loc_logger: local logger
43 :type loc_logger: _SfaLogger
46 self.auth_hierarchy = auth_hierarchy
47 self.logger = loc_logger
48 self.logger.setLevelDebug()
50 #retrieve all existing SFA objects
51 self.all_records = global_dbsession.query(RegRecord).all()
53 # initialize record.stale to True by default,
54 # then mark stale=False on the ones that are in use
55 for record in self.all_records:
57 #create hash by (type,hrn)
58 #used to know if a given record is already known to SFA
59 self.records_by_type_hrn = \
60 dict([((record.type, record.hrn), record)
61 for record in self.all_records])
63 self.users_rec_by_email = \
64 dict([(record.email, record)
65 for record in self.all_records if record.type == 'user'])
67 # create hash by (type,pointer)
68 self.records_by_type_pointer = \
69 dict([((str(record.type), record.pointer), record)
70 for record in self.all_records if record.pointer != -1])
74 def exists(self, tablename):
76 Checks if the table specified as tablename exists.
77 :param tablename: name of the table in the db that has to be checked.
78 :type tablename: string
79 :returns: True if the table exists, False otherwise.
83 metadata = MetaData(bind=engine)
85 table = Table(tablename, metadata, autoload=True)
88 except NoSuchTableError:
89 self.logger.log_exc("Iotlabimporter tablename %s does not exist"
95 def hostname_to_hrn_escaped(root_auth, hostname):
98 Returns a node's hrn based on its hostname and the root authority and by
99 removing special caracters from the hostname.
101 :param root_auth: root authority name
102 :param hostname: nodes's hostname
103 :type root_auth: string
104 :type hostname: string
107 return '.'.join([root_auth, Xrn.escape(hostname)])
111 def slicename_to_hrn(person_hrn):
114 Returns the slicename associated to a given person's hrn.
116 :param person_hrn: user's hrn
117 :type person_hrn: string
120 return (person_hrn + '_slice')
122 def add_options(self, parser):
124 .. warning:: not used
126 # we don't have any options for now
129 def find_record_by_type_hrn(self, record_type, hrn):
131 Finds the record associated with the hrn and its type given in parameter
132 if the tuple (hrn, type hrn) is an existing key in the dictionary.
134 :param record_type: the record's type (slice, node, authority...)
135 :type record_type: string
136 :param hrn: Human readable name of the object's record
138 :returns: Returns the record associated with a given hrn and hrn type.
139 Returns None if the key tuple is not in the dictionary.
140 :rtype: RegUser if user, RegSlice if slice, RegNode if node...or None if
141 record does not exist.
144 return self.records_by_type_hrn.get((record_type, hrn), None)
146 def locate_by_type_pointer(self, record_type, pointer):
148 Returns the record corresponding to the key pointer and record type.
149 Returns None if the record does not exist and is not in the
150 records_by_type_pointer dictionnary.
152 :param record_type: the record's type (slice, node, authority...)
153 :type record_type: string
154 :param pointer: Pointer to where the record is in the origin db,
155 used in case the record comes from a trusted authority.
156 :type pointer: integer
157 :rtype: RegUser if user, RegSlice if slice, RegNode if node, or None if
158 record does not exist.
160 return self.records_by_type_pointer.get((record_type, pointer), None)
163 def update_just_added_records_dict(self, record):
166 Updates the records_by_type_hrn dictionnary if the record has
169 :param record: Record to add in the records_by_type_hrn dict.
170 :type record: dictionary
172 rec_tuple = (record.type, record.hrn)
173 if rec_tuple in self.records_by_type_hrn:
174 self.logger.warning("IotlabImporter.update_just_added_records_dict:\
175 duplicate (%s,%s)" % rec_tuple)
177 self.records_by_type_hrn[rec_tuple] = record
180 def import_nodes(self, site_node_ids, nodes_by_id, testbed_shell):
183 Creates appropriate hostnames and RegNode records for each node in
184 site_node_ids, based on the information given by the dict nodes_by_id
185 that was made from data from OAR. Saves the records to the DB.
187 :param site_node_ids: site's node ids
188 :type site_node_ids: list of integers
189 :param nodes_by_id: dictionary , key is the node id, value is the a dict
190 with node information.
191 :type nodes_by_id: dictionary
192 :param testbed_shell: IotlabDriver object, used to have access to
193 testbed_shell attributes.
194 :type testbed_shell: IotlabDriver
201 for node_id in site_node_ids:
203 node = nodes_by_id[node_id]
205 self.logger.warning("IotlabImporter: cannot find node_id %s \
206 - ignored" % (node_id))
209 self.hostname_to_hrn_escaped(testbed_shell.root_auth,
211 self.logger.info("IOTLABIMPORTER node %s " % (node))
214 # xxx this sounds suspicious
217 node_record = self.find_record_by_type_hrn('node', hrn)
219 pkey = Keypair(create=True)
220 urn = hrn_to_urn(escaped_hrn, 'node')
222 self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
224 def testbed_get_authority(hrn):
225 """ Gets the authority part in the hrn.
226 :param hrn: hrn whose authority we are looking for.
228 :returns: splits the hrn using the '.' separator and returns
229 the authority part of the hrn.
233 return hrn.split(".")[0]
235 node_record = RegNode(hrn=hrn, gid=node_gid,
237 authority=testbed_get_authority(hrn))
240 node_record.just_created()
241 global_dbsession.add(node_record)
242 global_dbsession.commit()
243 self.logger.info("IotlabImporter: imported node: %s"
245 self.update_just_added_records_dict(node_record)
246 except SQLAlchemyError:
247 self.logger.log_exc("IotlabImporter: failed to import node")
249 #TODO: xxx update the record ...
251 node_record.stale = False
253 def import_sites_and_nodes(self, testbed_shell):
256 Gets all the sites and nodes from OAR, process the information,
257 creates hrns and RegAuthority for sites, and feed them to the database.
258 For each site, import the site's nodes to the DB by calling
261 :param testbed_shell: IotlabDriver object, used to have access to
262 testbed_shell methods and fetching info on sites and nodes.
263 :type testbed_shell: IotlabDriver
266 sites_listdict = testbed_shell.GetSites()
267 nodes_listdict = testbed_shell.GetNodes()
268 nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict])
269 for site in sites_listdict:
270 site_hrn = site['name']
271 site_record = self.find_record_by_type_hrn ('authority', site_hrn)
272 self.logger.info("IotlabImporter: import_sites_and_nodes \
273 (site) %s \r\n " % site_record)
276 urn = hrn_to_urn(site_hrn, 'authority')
277 if not self.auth_hierarchy.auth_exists(urn):
278 self.auth_hierarchy.create_auth(urn)
280 auth_info = self.auth_hierarchy.get_auth_info(urn)
282 RegAuthority(hrn=site_hrn,
283 gid=auth_info.get_gid_object(),
285 authority=get_authority(site_hrn))
286 site_record.just_created()
287 global_dbsession.add(site_record)
288 global_dbsession.commit()
289 self.logger.info("IotlabImporter: imported authority \
290 (site) %s" % site_record)
291 self.update_just_added_records_dict(site_record)
292 except SQLAlchemyError:
293 # if the site import fails then there is no point in
294 # trying to import the
295 # site's child records(node, slices, persons), so skip them.
296 self.logger.log_exc("IotlabImporter: failed to import \
297 site. Skipping child records")
300 # xxx update the record ...
303 site_record.stale = False
304 self.import_nodes(site['node_ids'], nodes_by_id, testbed_shell)
310 def init_person_key(self, person, iotlab_key):
312 Returns a tuple pubkey and pkey.
314 :param person Person's data.
316 :param iotlab_key: SSH public key, from LDAP user's data. RSA type
318 :type iotlab_key: string
319 :rtype: (string, Keypair)
324 # randomly pick first key in set
328 pkey = convert_public_key(pubkey)
330 #key not good. create another pkey
331 self.logger.warn("IotlabImporter: \
332 unable to convert public \
333 key for %s" % person['hrn'])
334 pkey = Keypair(create=True)
337 # the user has no keys.
338 #Creating a random keypair for the user's gid
339 self.logger.warn("IotlabImporter: person %s does not have a \
340 public key" % (person['hrn']))
341 pkey = Keypair(create=True)
342 return (pubkey, pkey)
344 def import_persons_and_slices(self, testbed_shell):
347 Gets user data from LDAP, process the information.
348 Creates hrn for the user's slice, the user's gid, creates
349 the RegUser record associated with user. Creates the RegKey record
350 associated nwith the user's key.
351 Saves those records into the SFA DB.
352 import the user's slice onto the database as well by calling
355 :param testbed_shell: IotlabDriver object, used to have access to
356 testbed_shell attributes.
357 :type testbed_shell: IotlabDriver
359 .. warning:: does not support multiple keys per user
361 ldap_person_listdict = testbed_shell.GetPersons()
362 self.logger.info("IOTLABIMPORT \t ldap_person_listdict %s \r\n"
363 % (ldap_person_listdict))
366 for person in ldap_person_listdict:
368 self.logger.info("IotlabImporter: person :" % (person))
369 if 'ssh-rsa' not in person['pkey']:
370 #people with invalid ssh key (ssh-dss, empty, bullshit keys...)
373 person_hrn = person['hrn']
374 slice_hrn = self.slicename_to_hrn(person['hrn'])
376 # xxx suspicious again
377 if len(person_hrn) > 64:
378 person_hrn = person_hrn[:64]
379 person_urn = hrn_to_urn(person_hrn, 'user')
382 self.logger.info("IotlabImporter: users_rec_by_email %s "
383 % (self.users_rec_by_email))
385 #Check if user using person['email'] from LDAP is already registered
386 #in SFA. One email = one person. In this case, do not create another
387 #record for this person
388 #person_hrn returned by GetPerson based on iotlab root auth +
390 user_record = self.find_record_by_type_hrn('user', person_hrn)
392 if not user_record and person['email'] in self.users_rec_by_email:
393 user_record = self.users_rec_by_email[person['email']]
394 person_hrn = user_record.hrn
395 person_urn = hrn_to_urn(person_hrn, 'user')
398 slice_record = self.find_record_by_type_hrn('slice', slice_hrn)
400 iotlab_key = person['pkey']
403 (pubkey, pkey) = self.init_person_key(person, iotlab_key)
404 if pubkey is not None and pkey is not None:
406 self.auth_hierarchy.create_gid(person_urn,
409 self.logger.debug("IOTLAB IMPORTER \
410 PERSON EMAIL OK email %s " % (person['email']))
411 person_gid.set_email(person['email'])
413 RegUser(hrn=person_hrn,
416 authority=get_authority(person_hrn),
417 email=person['email'])
420 RegUser(hrn=person_hrn,
423 authority=get_authority(person_hrn))
426 user_record.reg_keys = [RegKey(pubkey)]
428 self.logger.warning("No key found for user %s"
432 user_record.just_created()
433 global_dbsession.add (user_record)
434 global_dbsession.commit()
435 self.logger.info("IotlabImporter: imported person \
437 self.update_just_added_records_dict(user_record)
439 except SQLAlchemyError:
440 self.logger.log_exc("IotlabImporter: \
441 failed to import person %s" % (person))
443 # update the record ?
444 # if user's primary key has changed then we need to update
445 # the users gid by forcing an update here
446 sfa_keys = user_record.reg_keys
449 if iotlab_key is not sfa_keys:
452 self.logger.info("IotlabImporter: \t \t USER UPDATE \
453 person: %s" % (person['hrn']))
454 (pubkey, pkey) = self.init_person_key(person, iotlab_key)
456 self.auth_hierarchy.create_gid(person_urn,
459 user_record.reg_keys = []
461 user_record.reg_keys = [RegKey(pubkey)]
462 self.logger.info("IotlabImporter: updated person: %s"
466 user_record.email = person['email']
469 global_dbsession.commit()
470 user_record.stale = False
471 except SQLAlchemyError:
472 self.logger.log_exc("IotlabImporter: \
473 failed to update person %s"% (person))
475 self.import_slice(slice_hrn, slice_record, user_record)
478 def import_slice(self, slice_hrn, slice_record, user_record):
481 Create RegSlice record according to the slice hrn if the slice
482 does not exist yet.Creates a relationship with the user record
483 associated with the slice.
484 Commit the record to the database.
487 :param slice_hrn: Human readable name of the slice.
488 :type slice_hrn: string
489 :param slice_record: record of the slice found in the DB, if any.
490 :type slice_record: RegSlice or None
491 :param user_record: user record found in the DB if any.
492 :type user_record: RegUser
494 .. todo::Update the record if a slice record already exists.
497 pkey = Keypair(create=True)
498 urn = hrn_to_urn(slice_hrn, 'slice')
500 self.auth_hierarchy.create_gid(urn,
502 slice_record = RegSlice(hrn=slice_hrn, gid=slice_gid,
504 authority=get_authority(slice_hrn))
506 slice_record.just_created()
507 global_dbsession.add(slice_record)
508 global_dbsession.commit()
511 self.update_just_added_records_dict(slice_record)
513 except SQLAlchemyError:
514 self.logger.log_exc("IotlabImporter: failed to import slice")
516 #No slice update upon import in iotlab
518 # xxx update the record ...
519 self.logger.warning("Iotlab Slice update not implemented")
521 # record current users affiliated with the slice
522 slice_record.reg_researchers = [user_record]
524 global_dbsession.commit()
525 slice_record.stale = False
526 except SQLAlchemyError:
527 self.logger.log_exc("IotlabImporter: failed to update slice")
530 def run(self, options):
532 Create the special iotlab table, lease_table, in the SFA database.
533 Import everything (users, slices, nodes and sites from OAR
534 and LDAP) into the SFA database.
535 Delete stale records that are no longer in OAR or LDAP.
541 interface_hrn = config.SFA_INTERFACE_HRN
542 root_auth = config.SFA_REGISTRY_ROOT_AUTH
544 testbed_shell = IotlabShell(config)
545 # leases_db = TestbedAdditionalSfaDB(config)
546 #Create special slice table for iotlab
548 if not self.exists('lease_table'):
550 self.logger.info("IotlabImporter.run: lease_table table created ")
552 # import site and node records in site into the SFA db.
553 self.import_sites_and_nodes(testbed_shell)
554 #import users and slice into the SFA DB.
555 self.import_persons_and_slices(testbed_shell)
557 ### remove stale records
558 # special records must be preserved
559 system_hrns = [interface_hrn, root_auth,
560 interface_hrn + '.slicemanager']
561 for record in self.all_records:
562 if record.hrn in system_hrns:
564 if record.peer_authority:
567 for record in self.all_records:
568 if record.type == 'user':
569 self.logger.info("IotlabImporter: stale records: hrn %s %s"
570 % (record.hrn, record.stale))
575 self.logger.warning("stale not found with %s" % record)
577 self.logger.info("IotlabImporter: deleting stale record: %s"
581 global_dbsession.delete(record)
582 global_dbsession.commit()
583 except SQLAlchemyError:
584 self.logger.log_exc("IotlabImporter: failed to delete \
585 stale record %s" % (record))