1 """ File defining the importer class and all the methods needed to import
2 the nodes, users and slices from OAR and LDAP to the SFA database.
3 Also creates the iotlab specific databse and table to keep track
4 of which slice hrn contains which job.
6 from sfa.util.config import Config
7 from sfa.util.xrn import Xrn, get_authority, hrn_to_urn
9 from sfa.iotlab.iotlabdriver import IotlabDriver
10 from sfa.iotlab.iotlabpostgres import IotlabDB
11 from sfa.trust.certificate import Keypair, convert_public_key
12 from sfa.trust.gid import create_uuid
14 from sfa.storage.alchemy import dbsession
15 from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, \
19 from sqlalchemy.exc import SQLAlchemyError
24 IotlabImporter class, generic importer_class. Used to populate the SFA DB
25 with iotlab resources' records.
26 Used to update records when new resources, users or nodes, are added
30 def __init__(self, auth_hierarchy, loc_logger):
32 Sets and defines import logger and the authority name. Gathers all the
33 records already registerd in the SFA DB, broke them into 3 dicts, by
34 type and hrn, by email and by type and pointer.
36 :param auth_hierarchy: authority name
37 :type auth_hierarchy: string
38 :param loc_logger: local logger
39 :type loc_logger: _SfaLogger
42 self.auth_hierarchy = auth_hierarchy
43 self.logger = loc_logger
44 self.logger.setLevelDebug()
45 #retrieve all existing SFA objects
46 self.all_records = dbsession.query(RegRecord).all()
48 # initialize record.stale to True by default,
49 # then mark stale=False on the ones that are in use
50 for record in self.all_records:
52 #create hash by (type,hrn)
53 #used to know if a given record is already known to SFA
54 self.records_by_type_hrn = \
55 dict([((record.type, record.hrn), record)
56 for record in self.all_records])
58 self.users_rec_by_email = \
59 dict([(record.email, record)
60 for record in self.all_records if record.type == 'user'])
62 # create hash by (type,pointer)
63 self.records_by_type_pointer = \
64 dict([((str(record.type), record.pointer), record)
65 for record in self.all_records if record.pointer != -1])
69 def hostname_to_hrn_escaped(root_auth, hostname):
72 Returns a node's hrn based on its hostname and the root authority and by
73 removing special caracters from the hostname.
75 :param root_auth: root authority name
76 :param hostname: nodes's hostname
77 :type root_auth: string
78 :type hostname: string
81 return '.'.join([root_auth, Xrn.escape(hostname)])
85 def slicename_to_hrn(person_hrn):
88 Returns the slicename associated to a given person's hrn.
90 :param person_hrn: user's hrn
91 :type person_hrn: string
94 return (person_hrn + '_slice')
96 def add_options(self, parser):
100 # we don't have any options for now
103 def find_record_by_type_hrn(self, record_type, hrn):
105 Finds the record associated with the hrn and its type given in parameter
106 if the tuple (hrn, type hrn) is an existing key in the dictionary.
108 :param record_type: the record's type (slice, node, authority...)
109 :type record_type: string
110 :param hrn: Human readable name of the object's record
112 :returns: Returns the record associated with a given hrn and hrn type.
113 Returns None if the key tuple is not in the dictionary.
114 :rtype: RegUser if user, RegSlice if slice, RegNode if node...or None if
115 record does not exist.
118 return self.records_by_type_hrn.get((record_type, hrn), None)
120 def locate_by_type_pointer(self, record_type, pointer):
123 Returns the record corresponding to the key pointer and record
124 type. Returns None if the record does not exist and is not in the
125 records_by_type_pointer dictionnary.
127 :param record_type: the record's type (slice, node, authority...)
128 :type record_type: string
129 :param pointer:Pointer to where the record is in the origin db,
130 used in case the record comes from a trusted authority.
131 :type pointer: integer
132 :rtype: RegUser if user, RegSlice if slice, RegNode if node...
133 or None if record does not exist.
135 return self.records_by_type_pointer.get((record_type, pointer), None)
138 def update_just_added_records_dict(self, record):
141 Updates the records_by_type_hrn dictionnary if the record has
144 :param record: Record to add in the records_by_type_hrn dict.
145 :type record: dictionary
147 rec_tuple = (record.type, record.hrn)
148 if rec_tuple in self.records_by_type_hrn:
149 self.logger.warning("IotlabImporter.update_just_added_records_dict:\
150 duplicate (%s,%s)" % rec_tuple)
152 self.records_by_type_hrn[rec_tuple] = record
154 def import_sites_and_nodes(self, iotlabdriver):
157 Gets all the sites and nodes from OAR, process the information,
158 creates hrns and RegAuthority for sites, and feed them to the database.
159 For each site, import the site's nodes to the DB by calling
162 :param iotlabdriver: IotlabDriver object, used to have access to
163 iotlabdriver methods and fetching info on sites and nodes.
164 :type iotlabdriver: IotlabDriver
167 sites_listdict = iotlabdriver.iotlab_api.GetSites()
168 nodes_listdict = iotlabdriver.iotlab_api.GetNodes()
169 nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict])
170 for site in sites_listdict:
171 site_hrn = site['name']
172 site_record = self.find_record_by_type_hrn ('authority', site_hrn)
175 urn = hrn_to_urn(site_hrn, 'authority')
176 if not self.auth_hierarchy.auth_exists(urn):
177 self.auth_hierarchy.create_auth(urn)
179 auth_info = self.auth_hierarchy.get_auth_info(urn)
181 RegAuthority(hrn=site_hrn,
182 gid=auth_info.get_gid_object(),
184 authority=get_authority(site_hrn))
185 site_record.just_created()
186 dbsession.add(site_record)
188 self.logger.info("IotlabImporter: imported authority \
189 (site) %s" % site_record)
190 self.update_just_added_records_dict(site_record)
191 except SQLAlchemyError:
192 # if the site import fails then there is no point in
193 # trying to import the
194 # site's child records(node, slices, persons), so skip them.
195 self.logger.log_exc("IotlabImporter: failed to import \
196 site. Skipping child records")
199 # xxx update the record ...
202 site_record.stale = False
203 self.import_nodes(site['node_ids'], nodes_by_id, iotlabdriver)
207 def import_nodes(self, site_node_ids, nodes_by_id, iotlabdriver):
210 Creates appropriate hostnames and RegNode records for each node in
211 site_node_ids, based on the information given by the dict nodes_by_id
212 that was made from data from OAR. Saves the records to the DB.
214 :param site_node_ids: site's node ids
215 :type site_node_ids: list of integers
216 :param nodes_by_id: dictionary , key is the node id, value is the a dict
217 with node information.
218 :type nodes_by_id: dictionary
219 :param iotlabdriver: IotlabDriver object, used to have access to
220 iotlabdriver attributes.
221 :type iotlabdriver: IotlabDriver
228 for node_id in site_node_ids:
230 node = nodes_by_id[node_id]
232 self.logger.warning("IotlabImporter: cannot find node_id %s \
233 - ignored" % (node_id))
236 self.hostname_to_hrn_escaped(iotlabdriver.iotlab_api.root_auth,
238 self.logger.info("IOTLABIMPORTER node %s " % (node))
241 # xxx this sounds suspicious
244 node_record = self.find_record_by_type_hrn('node', hrn)
246 pkey = Keypair(create=True)
247 urn = hrn_to_urn(escaped_hrn, 'node')
249 self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
251 def iotlab_get_authority(hrn):
252 """ Gets the authority part in the hrn.
253 :param hrn: hrn whose authority we are looking for.
255 :returns: splits the hrn using the '.' separator and returns
256 the authority part of the hrn.
260 return hrn.split(".")[0]
262 node_record = RegNode(hrn=hrn, gid=node_gid,
264 authority=iotlab_get_authority(hrn))
267 node_record.just_created()
268 dbsession.add(node_record)
270 self.logger.info("IotlabImporter: imported node: %s"
272 self.update_just_added_records_dict(node_record)
273 except SQLAlchemyError:
274 self.logger.log_exc("IotlabImporter: failed to import node")
276 #TODO: xxx update the record ...
278 node_record.stale = False
280 def init_person_key(self, person, iotlab_key):
283 Returns a tuple pubkey and pkey.
285 :param person Person's data.
287 :param iotlab_key: SSH public key, from LDAP user's data.
289 :type iotlab_key: string
290 :rtype (string, Keypair)
295 # randomly pick first key in set
299 pkey = convert_public_key(pubkey)
301 #key not good. create another pkey
302 self.logger.warn("IotlabImporter: \
303 unable to convert public \
304 key for %s" % person['hrn'])
305 pkey = Keypair(create=True)
308 # the user has no keys.
309 #Creating a random keypair for the user's gid
310 self.logger.warn("IotlabImporter: person %s does not have a \
311 public key" % (person['hrn']))
312 pkey = Keypair(create=True)
313 return (pubkey, pkey)
315 def import_persons_and_slices(self, iotlabdriver):
318 Gets user data from LDAP, process the information.
319 Creates hrn for the user's slice, the user's gid, creates
320 the RegUser record associated with user. Creates the RegKey record
321 associated nwith the user's key.
322 Saves those records into the SFA DB.
323 import the user's slice onto the database as well by calling
326 :param iotlabdriver: IotlabDriver object, used to have access to
327 iotlabdriver attributes.
328 :type iotlabdriver: IotlabDriver
330 ldap_person_listdict = iotlabdriver.iotlab_api.GetPersons()
331 self.logger.info("IOTLABIMPORT \t ldap_person_listdict %s \r\n"
332 % (ldap_person_listdict))
335 for person in ldap_person_listdict:
337 self.logger.info("IotlabImporter: person :" % (person))
338 if 'ssh-rsa' not in person['pkey']:
339 #people with invalid ssh key (ssh-dss, empty, bullshit keys...)
342 person_hrn = person['hrn']
343 slice_hrn = self.slicename_to_hrn(person['hrn'])
345 # xxx suspicious again
346 if len(person_hrn) > 64:
347 person_hrn = person_hrn[:64]
348 person_urn = hrn_to_urn(person_hrn, 'user')
351 self.logger.info("IotlabImporter: users_rec_by_email %s "
352 % (self.users_rec_by_email))
354 #Check if user using person['email'] from LDAP is already registered
355 #in SFA. One email = one person. In this case, do not create another
356 #record for this person
357 #person_hrn returned by GetPerson based on iotlab root auth +
359 user_record = self.find_record_by_type_hrn('user', person_hrn)
361 if not user_record and person['email'] in self.users_rec_by_email:
362 user_record = self.users_rec_by_email[person['email']]
363 person_hrn = user_record.hrn
364 person_urn = hrn_to_urn(person_hrn, 'user')
367 slice_record = self.find_record_by_type_hrn('slice', slice_hrn)
369 iotlab_key = person['pkey']
372 (pubkey, pkey) = self.init_person_key(person, iotlab_key)
373 if pubkey is not None and pkey is not None:
375 self.auth_hierarchy.create_gid(person_urn,
378 self.logger.debug("IOTLAB IMPORTER \
379 PERSON EMAIL OK email %s " % (person['email']))
380 person_gid.set_email(person['email'])
382 RegUser(hrn=person_hrn,
385 authority=get_authority(person_hrn),
386 email=person['email'])
389 RegUser(hrn=person_hrn,
392 authority=get_authority(person_hrn))
395 user_record.reg_keys = [RegKey(pubkey)]
397 self.logger.warning("No key found for user %s"
401 user_record.just_created()
402 dbsession.add (user_record)
404 self.logger.info("IotlabImporter: imported person \
406 self.update_just_added_records_dict(user_record)
408 except SQLAlchemyError:
409 self.logger.log_exc("IotlabImporter: \
410 failed to import person %s" % (person))
412 # update the record ?
413 # if user's primary key has changed then we need to update
414 # the users gid by forcing an update here
415 sfa_keys = user_record.reg_keys
418 if iotlab_key is not sfa_keys:
421 self.logger.info("IotlabImporter: \t \t USER UPDATE \
422 person: %s" % (person['hrn']))
423 (pubkey, pkey) = self.init_person_key(person, iotlab_key)
425 self.auth_hierarchy.create_gid(person_urn,
428 user_record.reg_keys = []
430 user_record.reg_keys = [RegKey(pubkey)]
431 self.logger.info("IotlabImporter: updated person: %s"
435 user_record.email = person['email']
439 user_record.stale = False
440 except SQLAlchemyError:
441 self.logger.log_exc("IotlabImporter: \
442 failed to update person %s"% (person))
444 self.import_slice(slice_hrn, slice_record, user_record)
447 def import_slice(self, slice_hrn, slice_record, user_record):
450 Create RegSlice record according to the slice hrn if the slice
451 does not exist yet.Creates a relationship with the user record
452 associated with the slice.
453 Commit the record to the database.
456 :param slice_hrn: Human readable name of the slice.
457 :type slice_hrn: string
458 :param slice_record: record of the slice found in the DB, if any.
459 :type slice_record: RegSlice or None
460 :param user_record: user record found in the DB if any.
461 :type user_record: RegUser
463 .. todo::Update the record if a slice record already exists.
466 pkey = Keypair(create=True)
467 urn = hrn_to_urn(slice_hrn, 'slice')
469 self.auth_hierarchy.create_gid(urn,
471 slice_record = RegSlice(hrn=slice_hrn, gid=slice_gid,
473 authority=get_authority(slice_hrn))
475 slice_record.just_created()
476 dbsession.add(slice_record)
480 self.update_just_added_records_dict(slice_record)
482 except SQLAlchemyError:
483 self.logger.log_exc("IotlabImporter: failed to import slice")
485 #No slice update upon import in iotlab
487 # xxx update the record ...
488 self.logger.warning("Slice update not yet implemented")
490 # record current users affiliated with the slice
493 slice_record.reg_researchers = [user_record]
496 slice_record.stale = False
497 except SQLAlchemyError:
498 self.logger.log_exc("IotlabImporter: failed to update slice")
501 def run(self, options):
503 Create the special iotlab table, iotlab_xp, in the iotlab database.
504 Import everything (users, slices, nodes and sites from OAR
505 and LDAP) into the SFA database.
506 Delete stale records that are no longer in OAR or LDAP.
512 iotlabdriver = IotlabDriver(config)
513 iotlab_db = IotlabDB(config)
514 #Create special slice table for iotlab
516 if not iotlab_db.exists('iotlab_xp'):
517 iotlab_db.createtable()
518 self.logger.info("IotlabImporter.run: iotlab_xp table created ")
520 # import site and node records in site into the SFA db.
521 self.import_sites_and_nodes(iotlabdriver)
522 #import users and slice into the SFA DB.
523 self.import_persons_and_slices(iotlabdriver)
525 ### remove stale records
526 # special records must be preserved
527 system_hrns = [iotlabdriver.hrn, iotlabdriver.iotlab_api.root_auth,
528 iotlabdriver.hrn + '.slicemanager']
529 for record in self.all_records:
530 if record.hrn in system_hrns:
532 if record.peer_authority:
535 for record in self.all_records:
536 if record.type == 'user':
537 self.logger.info("IotlabImporter: stale records: hrn %s %s"
538 % (record.hrn, record.stale))
543 self.logger.warning("stale not found with %s" % record)
545 self.logger.info("IotlabImporter: deleting stale record: %s"
549 dbsession.delete(record)
551 except SQLAlchemyError:
552 self.logger.log_exc("IotlabImporter: failed to delete \
553 stale record %s" % (record))