1 """ File defining the importer class and all the methods needed to import
2 the nodes, users and slices from OAR and LDAP to the SFA database.
3 Also creates the iotlab specific databse and table to keep track
4 of which slice hrn contains which job.
6 from sfa.util.config import Config
7 from sfa.util.xrn import Xrn, get_authority, hrn_to_urn
9 from sfa.iotlab.iotlabdriver import IotlabDriver
10 from sfa.iotlab.iotlabpostgres import IotlabDB
11 from sfa.trust.certificate import Keypair, convert_public_key
12 from sfa.trust.gid import create_uuid
14 from sfa.storage.alchemy import dbsession
15 from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, \
19 from sqlalchemy.exc import SQLAlchemyError
24 IotlabImporter class, generic importer_class. Used to populate the SFA DB
25 with iotlab resources' records.
26 Used to update records when new resources, users or nodes, are added
30 def __init__(self, auth_hierarchy, loc_logger):
32 Sets and defines import logger and the authority name. Gathers all the
33 records already registerd in the SFA DB, broke them into 3 dicts, by
34 type and hrn, by email and by type and pointer.
36 :param auth_hierarchy: authority name
37 :type auth_hierarchy: string
38 :param loc_logger: local logger
39 :type loc_logger: _SfaLogger
42 self.auth_hierarchy = auth_hierarchy
43 self.logger = loc_logger
44 self.logger.setLevelDebug()
45 #retrieve all existing SFA objects
46 self.all_records = dbsession.query(RegRecord).all()
48 # initialize record.stale to True by default,
49 # then mark stale=False on the ones that are in use
50 for record in self.all_records:
52 #create hash by (type,hrn)
53 #used to know if a given record is already known to SFA
54 self.records_by_type_hrn = \
55 dict([((record.type, record.hrn), record)
56 for record in self.all_records])
58 self.users_rec_by_email = \
59 dict([(record.email, record)
60 for record in self.all_records if record.type == 'user'])
62 # create hash by (type,pointer)
63 self.records_by_type_pointer = \
64 dict([((str(record.type), record.pointer), record)
65 for record in self.all_records if record.pointer != -1])
69 def hostname_to_hrn_escaped(root_auth, hostname):
72 Returns a node's hrn based on its hostname and the root authority and by
73 removing special caracters from the hostname.
75 :param root_auth: root authority name
76 :param hostname: nodes's hostname
77 :type root_auth: string
78 :type hostname: string
81 return '.'.join([root_auth, Xrn.escape(hostname)])
85 def slicename_to_hrn(person_hrn):
88 Returns the slicename associated to a given person's hrn.
90 :param person_hrn: user's hrn
91 :type person_hrn: string
94 return (person_hrn + '_slice')
96 def add_options(self, parser):
100 # we don't have any options for now
103 def find_record_by_type_hrn(self, record_type, hrn):
105 Finds the record associated with the hrn and its type given in parameter
106 if the tuple (hrn, type hrn) is an existing key in the dictionary.
108 :param record_type: the record's type (slice, node, authority...)
109 :type record_type: string
110 :param hrn: Human readable name of the object's record
112 :returns: Returns the record associated with a given hrn and hrn type.
113 Returns None if the key tuple is not in the dictionary.
114 :rtype: RegUser if user, RegSlice if slice, RegNode if node...or None if
115 record does not exist.
118 return self.records_by_type_hrn.get((record_type, hrn), None)
120 def locate_by_type_pointer(self, record_type, pointer):
123 Returns the record corresponding to the key pointer and record
124 type. Returns None if the record does not exist and is not in the
125 records_by_type_pointer dictionnary.
127 :param record_type: the record's type (slice, node, authority...)
128 :type record_type: string
129 :param pointer:Pointer to where the record is in the origin db,
130 used in case the record comes from a trusted authority.
131 :type pointer: integer
132 :rtype: RegUser if user, RegSlice if slice, RegNode if node...
133 or None if record does not exist.
135 return self.records_by_type_pointer.get((record_type, pointer), None)
138 def update_just_added_records_dict(self, record):
141 Updates the records_by_type_hrn dictionnary if the record has
144 :param record: Record to add in the records_by_type_hrn dict.
145 :type record: dictionary
147 rec_tuple = (record.type, record.hrn)
148 if rec_tuple in self.records_by_type_hrn:
149 self.logger.warning("IotlabImporter.update_just_added_records_dict:\
150 duplicate (%s,%s)" % rec_tuple)
152 self.records_by_type_hrn[rec_tuple] = record
155 def import_nodes(self, site_node_ids, nodes_by_id, iotlabdriver):
158 Creates appropriate hostnames and RegNode records for each node in
159 site_node_ids, based on the information given by the dict nodes_by_id
160 that was made from data from OAR. Saves the records to the DB.
162 :param site_node_ids: site's node ids
163 :type site_node_ids: list of integers
164 :param nodes_by_id: dictionary , key is the node id, value is the a dict
165 with node information.
166 :type nodes_by_id: dictionary
167 :param iotlabdriver: IotlabDriver object, used to have access to
168 iotlabdriver attributes.
169 :type iotlabdriver: IotlabDriver
176 for node_id in site_node_ids:
178 node = nodes_by_id[node_id]
180 self.logger.warning("IotlabImporter: cannot find node_id %s \
181 - ignored" % (node_id))
184 self.hostname_to_hrn_escaped(iotlabdriver.iotlab_api.root_auth,
186 self.logger.info("IOTLABIMPORTER node %s " % (node))
189 # xxx this sounds suspicious
192 node_record = self.find_record_by_type_hrn('node', hrn)
194 pkey = Keypair(create=True)
195 urn = hrn_to_urn(escaped_hrn, 'node')
197 self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
199 def iotlab_get_authority(hrn):
200 """ Gets the authority part in the hrn.
201 :param hrn: hrn whose authority we are looking for.
203 :returns: splits the hrn using the '.' separator and returns
204 the authority part of the hrn.
208 return hrn.split(".")[0]
210 node_record = RegNode(hrn=hrn, gid=node_gid,
212 authority=iotlab_get_authority(hrn))
215 node_record.just_created()
216 dbsession.add(node_record)
218 self.logger.info("IotlabImporter: imported node: %s"
220 self.update_just_added_records_dict(node_record)
221 except SQLAlchemyError:
222 self.logger.log_exc("IotlabImporter: failed to import node")
224 #TODO: xxx update the record ...
226 node_record.stale = False
228 def import_sites_and_nodes(self, iotlabdriver):
231 Gets all the sites and nodes from OAR, process the information,
232 creates hrns and RegAuthority for sites, and feed them to the database.
233 For each site, import the site's nodes to the DB by calling
236 :param iotlabdriver: IotlabDriver object, used to have access to
237 iotlabdriver methods and fetching info on sites and nodes.
238 :type iotlabdriver: IotlabDriver
241 sites_listdict = iotlabdriver.iotlab_api.GetSites()
242 nodes_listdict = iotlabdriver.iotlab_api.GetNodes()
243 nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict])
244 for site in sites_listdict:
245 site_hrn = site['name']
246 site_record = self.find_record_by_type_hrn ('authority', site_hrn)
247 self.logger.info("IotlabImporter: import_sites_and_nodes \
248 (site) %s \r\n " % site_record)
251 urn = hrn_to_urn(site_hrn, 'authority')
252 if not self.auth_hierarchy.auth_exists(urn):
253 self.auth_hierarchy.create_auth(urn)
255 auth_info = self.auth_hierarchy.get_auth_info(urn)
257 RegAuthority(hrn=site_hrn,
258 gid=auth_info.get_gid_object(),
260 authority=get_authority(site_hrn))
261 site_record.just_created()
262 dbsession.add(site_record)
264 self.logger.info("IotlabImporter: imported authority \
265 (site) %s" % site_record)
266 self.update_just_added_records_dict(site_record)
267 except SQLAlchemyError:
268 # if the site import fails then there is no point in
269 # trying to import the
270 # site's child records(node, slices, persons), so skip them.
271 self.logger.log_exc("IotlabImporter: failed to import \
272 site. Skipping child records")
275 # xxx update the record ...
278 site_record.stale = False
279 self.import_nodes(site['node_ids'], nodes_by_id, iotlabdriver)
285 def init_person_key(self, person, iotlab_key):
288 Returns a tuple pubkey and pkey.
290 :param person Person's data.
292 :param iotlab_key: SSH public key, from LDAP user's data.
294 :type iotlab_key: string
295 :rtype (string, Keypair)
300 # randomly pick first key in set
304 pkey = convert_public_key(pubkey)
306 #key not good. create another pkey
307 self.logger.warn("IotlabImporter: \
308 unable to convert public \
309 key for %s" % person['hrn'])
310 pkey = Keypair(create=True)
313 # the user has no keys.
314 #Creating a random keypair for the user's gid
315 self.logger.warn("IotlabImporter: person %s does not have a \
316 public key" % (person['hrn']))
317 pkey = Keypair(create=True)
318 return (pubkey, pkey)
320 def import_persons_and_slices(self, iotlabdriver):
323 Gets user data from LDAP, process the information.
324 Creates hrn for the user's slice, the user's gid, creates
325 the RegUser record associated with user. Creates the RegKey record
326 associated nwith the user's key.
327 Saves those records into the SFA DB.
328 import the user's slice onto the database as well by calling
331 :param iotlabdriver: IotlabDriver object, used to have access to
332 iotlabdriver attributes.
333 :type iotlabdriver: IotlabDriver
335 ldap_person_listdict = iotlabdriver.iotlab_api.GetPersons()
336 self.logger.info("IOTLABIMPORT \t ldap_person_listdict %s \r\n"
337 % (ldap_person_listdict))
340 for person in ldap_person_listdict:
342 self.logger.info("IotlabImporter: person :" % (person))
343 if 'ssh-rsa' not in person['pkey']:
344 #people with invalid ssh key (ssh-dss, empty, bullshit keys...)
347 person_hrn = person['hrn']
348 slice_hrn = self.slicename_to_hrn(person['hrn'])
350 # xxx suspicious again
351 if len(person_hrn) > 64:
352 person_hrn = person_hrn[:64]
353 person_urn = hrn_to_urn(person_hrn, 'user')
356 self.logger.info("IotlabImporter: users_rec_by_email %s "
357 % (self.users_rec_by_email))
359 #Check if user using person['email'] from LDAP is already registered
360 #in SFA. One email = one person. In this case, do not create another
361 #record for this person
362 #person_hrn returned by GetPerson based on iotlab root auth +
364 user_record = self.find_record_by_type_hrn('user', person_hrn)
366 if not user_record and person['email'] in self.users_rec_by_email:
367 user_record = self.users_rec_by_email[person['email']]
368 person_hrn = user_record.hrn
369 person_urn = hrn_to_urn(person_hrn, 'user')
372 slice_record = self.find_record_by_type_hrn('slice', slice_hrn)
374 iotlab_key = person['pkey']
377 (pubkey, pkey) = self.init_person_key(person, iotlab_key)
378 if pubkey is not None and pkey is not None:
380 self.auth_hierarchy.create_gid(person_urn,
383 self.logger.debug("IOTLAB IMPORTER \
384 PERSON EMAIL OK email %s " % (person['email']))
385 person_gid.set_email(person['email'])
387 RegUser(hrn=person_hrn,
390 authority=get_authority(person_hrn),
391 email=person['email'])
394 RegUser(hrn=person_hrn,
397 authority=get_authority(person_hrn))
400 user_record.reg_keys = [RegKey(pubkey)]
402 self.logger.warning("No key found for user %s"
406 user_record.just_created()
407 dbsession.add (user_record)
409 self.logger.info("IotlabImporter: imported person \
411 self.update_just_added_records_dict(user_record)
413 except SQLAlchemyError:
414 self.logger.log_exc("IotlabImporter: \
415 failed to import person %s" % (person))
417 # update the record ?
418 # if user's primary key has changed then we need to update
419 # the users gid by forcing an update here
420 sfa_keys = user_record.reg_keys
423 if iotlab_key is not sfa_keys:
426 self.logger.info("IotlabImporter: \t \t USER UPDATE \
427 person: %s" % (person['hrn']))
428 (pubkey, pkey) = self.init_person_key(person, iotlab_key)
430 self.auth_hierarchy.create_gid(person_urn,
433 user_record.reg_keys = []
435 user_record.reg_keys = [RegKey(pubkey)]
436 self.logger.info("IotlabImporter: updated person: %s"
440 user_record.email = person['email']
444 user_record.stale = False
445 except SQLAlchemyError:
446 self.logger.log_exc("IotlabImporter: \
447 failed to update person %s"% (person))
449 self.import_slice(slice_hrn, slice_record, user_record)
452 def import_slice(self, slice_hrn, slice_record, user_record):
455 Create RegSlice record according to the slice hrn if the slice
456 does not exist yet.Creates a relationship with the user record
457 associated with the slice.
458 Commit the record to the database.
461 :param slice_hrn: Human readable name of the slice.
462 :type slice_hrn: string
463 :param slice_record: record of the slice found in the DB, if any.
464 :type slice_record: RegSlice or None
465 :param user_record: user record found in the DB if any.
466 :type user_record: RegUser
468 .. todo::Update the record if a slice record already exists.
471 pkey = Keypair(create=True)
472 urn = hrn_to_urn(slice_hrn, 'slice')
474 self.auth_hierarchy.create_gid(urn,
476 slice_record = RegSlice(hrn=slice_hrn, gid=slice_gid,
478 authority=get_authority(slice_hrn))
480 slice_record.just_created()
481 dbsession.add(slice_record)
485 self.update_just_added_records_dict(slice_record)
487 except SQLAlchemyError:
488 self.logger.log_exc("IotlabImporter: failed to import slice")
490 #No slice update upon import in iotlab
492 # xxx update the record ...
493 self.logger.warning("Slice update not yet implemented")
495 # record current users affiliated with the slice
498 slice_record.reg_researchers = [user_record]
501 slice_record.stale = False
502 except SQLAlchemyError:
503 self.logger.log_exc("IotlabImporter: failed to update slice")
506 def run(self, options):
508 Create the special iotlab table, iotlab_xp, in the iotlab database.
509 Import everything (users, slices, nodes and sites from OAR
510 and LDAP) into the SFA database.
511 Delete stale records that are no longer in OAR or LDAP.
517 iotlabdriver = IotlabDriver(config)
518 iotlab_db = IotlabDB(config)
519 #Create special slice table for iotlab
521 if not iotlab_db.exists('iotlab_xp'):
522 iotlab_db.createtable()
523 self.logger.info("IotlabImporter.run: iotlab_xp table created ")
525 # import site and node records in site into the SFA db.
526 self.import_sites_and_nodes(iotlabdriver)
527 #import users and slice into the SFA DB.
528 self.import_persons_and_slices(iotlabdriver)
530 ### remove stale records
531 # special records must be preserved
532 system_hrns = [iotlabdriver.hrn, iotlabdriver.iotlab_api.root_auth,
533 iotlabdriver.hrn + '.slicemanager']
534 for record in self.all_records:
535 if record.hrn in system_hrns:
537 if record.peer_authority:
540 for record in self.all_records:
541 if record.type == 'user':
542 self.logger.info("IotlabImporter: stale records: hrn %s %s"
543 % (record.hrn, record.stale))
548 self.logger.warning("stale not found with %s" % record)
550 self.logger.info("IotlabImporter: deleting stale record: %s"
554 dbsession.delete(record)
556 except SQLAlchemyError:
557 self.logger.log_exc("IotlabImporter: failed to delete \
558 stale record %s" % (record))