1 """ File defining the importer class and all the methods needed to import
2 the nodes, users and slices from OAR and LDAP to the SFA database.
3 Also creates the iotlab specific databse and table to keep track
4 of which slice hrn contains which job.
6 from sfa.util.config import Config
7 from sfa.util.xrn import Xrn, get_authority, hrn_to_urn
9 from sfa.iotlab.iotlabdriver import IotlabDriver
10 from sfa.iotlab.iotlabpostgres import IotlabDB
11 from sfa.trust.certificate import Keypair, convert_public_key
12 from sfa.trust.gid import create_uuid
14 from sfa.storage.alchemy import dbsession
15 from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, \
19 from sqlalchemy.exc import SQLAlchemyError
24 IotlabImporter class, generic importer_class. Used to populate the SFA DB
25 with iotlab resources' records.
26 Used to update records when new resources, users or nodes, are added
30 def __init__(self, auth_hierarchy, loc_logger):
32 Sets and defines import logger and the authority name. Gathers all the
33 records already registerd in the SFA DB, broke them into 3 dicts, by
34 type and hrn, by email and by type and pointer.
36 :param auth_hierarchy: authority name
37 :type auth_hierarchy: string
38 :param loc_logger: local logger
39 :type loc_logger: _SfaLogger
42 self.auth_hierarchy = auth_hierarchy
43 self.logger = loc_logger
44 self.logger.setLevelDebug()
45 #retrieve all existing SFA objects
46 self.all_records = dbsession.query(RegRecord).all()
48 # initialize record.stale to True by default,
49 # then mark stale=False on the ones that are in use
50 for record in self.all_records:
52 #create hash by (type,hrn)
53 #used to know if a given record is already known to SFA
54 self.records_by_type_hrn = \
55 dict([((record.type, record.hrn), record)
56 for record in self.all_records])
58 self.users_rec_by_email = \
59 dict([(record.email, record)
60 for record in self.all_records if record.type == 'user'])
62 # create hash by (type,pointer)
63 self.records_by_type_pointer = \
64 dict([((str(record.type), record.pointer), record)
65 for record in self.all_records if record.pointer != -1])
69 def hostname_to_hrn_escaped(root_auth, hostname):
72 Returns a node's hrn based on its hostname and the root authority and by
73 removing special caracters from the hostname.
75 :param root_auth: root authority name
76 :param hostname: nodes's hostname
77 :type root_auth: string
78 :type hostname: string
81 return '.'.join([root_auth, Xrn.escape(hostname)])
85 def slicename_to_hrn(person_hrn):
88 Returns the slicename associated to a given person's hrn.
90 :param person_hrn: user's hrn
91 :type person_hrn: string
94 return (person_hrn + '_slice')
96 def add_options(self, parser):
100 # we don't have any options for now
103 def find_record_by_type_hrn(self, record_type, hrn):
105 Finds the record associated with the hrn and its type given in parameter
106 if the tuple (hrn, type hrn) is an existing key in the dictionary.
108 :param record_type: the record's type (slice, node, authority...)
109 :type record_type: string
110 :param hrn: Human readable name of the object's record
112 :returns: Returns the record associated with a given hrn and hrn type.
113 Returns None if the key tuple is not in the dictionary.
114 :rtype: RegUser if user, RegSlice if slice, RegNode if node...or None if
115 record does not exist.
118 return self.records_by_type_hrn.get((record_type, hrn), None)
120 def locate_by_type_pointer(self, record_type, pointer):
123 Returns the record corresponding to the key pointer and record
124 type. Returns None if the record does not exist and is not in the
125 records_by_type_pointer dictionnary.
127 :param record_type: the record's type (slice, node, authority...)
128 :type record_type: string
129 :param pointer:Pointer to where the record is in the origin db,
130 used in case the record comes from a trusted authority.
131 :type pointer: integer
132 :rtype: RegUser if user, RegSlice if slice, RegNode if node...
133 or None if record does not exist.
135 return self.records_by_type_pointer.get((record_type, pointer), None)
138 def update_just_added_records_dict(self, record):
141 Updates the records_by_type_hrn dictionnary if the record has
144 :param record: Record to add in the records_by_type_hrn dict.
145 :type record: dictionary
147 rec_tuple = (record.type, record.hrn)
148 if rec_tuple in self.records_by_type_hrn:
149 self.logger.warning("IotlabImporter.update_just_added_records_dict:\
150 duplicate (%s,%s)" % rec_tuple)
152 self.records_by_type_hrn[rec_tuple] = record
154 def import_sites_and_nodes(self, iotlabdriver):
157 Gets all the sites and nodes from OAR, process the information,
158 creates hrns and RegAuthority for sites, and feed them to the database.
159 For each site, import the site's nodes to the DB by calling
162 :param iotlabdriver: IotlabDriver object, used to have access to
163 iotlabdriver methods and fetching info on sites and nodes.
164 :type iotlabdriver: IotlabDriver
167 sites_listdict = iotlabdriver.iotlab_api.GetSites()
168 nodes_listdict = iotlabdriver.iotlab_api.GetNodes()
169 nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict])
170 for site in sites_listdict:
171 site_hrn = site['name']
172 site_record = self.find_record_by_type_hrn ('authority', site_hrn)
175 urn = hrn_to_urn(site_hrn, 'authority')
176 if not self.auth_hierarchy.auth_exists(urn):
177 self.auth_hierarchy.create_auth(urn)
179 auth_info = self.auth_hierarchy.get_auth_info(urn)
181 RegAuthority(hrn=site_hrn,
182 gid=auth_info.get_gid_object(),
184 authority=get_authority(site_hrn))
185 site_record.just_created()
186 dbsession.add(site_record)
188 self.logger.info("IotlabImporter: imported authority \
189 (site) %s" % site_record)
190 self.update_just_added_records_dict(site_record)
191 except SQLAlchemyError:
192 # if the site import fails then there is no point in
193 # trying to import the
194 # site's child records(node, slices, persons), so skip them.
195 self.logger.log_exc("IotlabImporter: failed to import \
196 site. Skipping child records")
199 # xxx update the record ...
202 site_record.stale = False
203 self.import_nodes(site['node_ids'], nodes_by_id, iotlabdriver)
207 def import_nodes(self, site_node_ids, nodes_by_id, iotlabdriver):
210 Creates appropriate hostnames and RegNode records for
211 each node in site_node_ids, based on the information given by the
212 dict nodes_by_id that was made from data from OAR.
213 Saves the records to the DB.
215 :param site_node_ids: site's node ids
216 :type site_node_ids: list of integers
217 :param nodes_by_id: dictionary , key is the node id, value is the a dict
218 with node information.
219 :type nodes_by_id: dictionary
220 :param iotlabdriver:IotlabDriver object, used to have access to
221 iotlabdriver attributes.
222 :type iotlabdriver:IotlabDriver
226 for node_id in site_node_ids:
228 node = nodes_by_id[node_id]
230 self.logger.warning("IotlabImporter: cannot find node_id %s \
231 - ignored" % (node_id))
234 self.hostname_to_hrn_escaped(iotlabdriver.iotlab_api.root_auth,
236 self.logger.info("IOTLABIMPORTER node %s " % (node))
239 # xxx this sounds suspicious
242 node_record = self.find_record_by_type_hrn('node', hrn)
244 pkey = Keypair(create=True)
245 urn = hrn_to_urn(escaped_hrn, 'node')
247 self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
249 def iotlab_get_authority(hrn):
250 """ Gets the authority part in the hrn.
251 :param hrn: hrn whose authority we are looking for.
253 :returns: splits the hrn using the '.' separator and returns
254 the authority part of the hrn.
258 return hrn.split(".")[0]
260 node_record = RegNode(hrn=hrn, gid=node_gid,
262 authority=iotlab_get_authority(hrn))
265 node_record.just_created()
266 dbsession.add(node_record)
268 self.logger.info("IotlabImporter: imported node: %s"
270 self.update_just_added_records_dict(node_record)
271 except SQLAlchemyError:
272 self.logger.log_exc("IotlabImporter: failed to import node")
274 #TODO: xxx update the record ...
276 node_record.stale = False
278 def init_person_key(self, person, iotlab_key):
281 Returns a tuple pubkey and pkey.
283 :param person Person's data.
285 :param iotlab_key: SSH public key, from LDAP user's data.
287 :type iotlab_key: string
288 :rtype (string, Keypair)
293 # randomly pick first key in set
297 pkey = convert_public_key(pubkey)
299 #key not good. create another pkey
300 self.logger.warn("IotlabImporter: \
301 unable to convert public \
302 key for %s" % person['hrn'])
303 pkey = Keypair(create=True)
306 # the user has no keys.
307 #Creating a random keypair for the user's gid
308 self.logger.warn("IotlabImporter: person %s does not have a \
309 public key" % (person['hrn']))
310 pkey = Keypair(create=True)
311 return (pubkey, pkey)
313 def import_persons_and_slices(self, iotlabdriver):
316 Gets user data from LDAP, process the information.
317 Creates hrn for the user's slice, the user's gid, creates
318 the RegUser record associated with user. Creates the RegKey record
319 associated nwith the user's key.
320 Saves those records into the SFA DB.
321 import the user's slice onto the database as well by calling
324 :param iotlabdriver:IotlabDriver object, used to have access to
325 iotlabdriver attributes.
326 :type iotlabdriver:IotlabDriver
328 ldap_person_listdict = iotlabdriver.iotlab_api.GetPersons()
329 self.logger.info("IOTLABIMPORT \t ldap_person_listdict %s \r\n"
330 % (ldap_person_listdict))
333 for person in ldap_person_listdict:
335 self.logger.info("IotlabImporter: person :" % (person))
336 if 'ssh-rsa' not in person['pkey']:
337 #people with invalid ssh key (ssh-dss, empty, bullshit keys...)
340 person_hrn = person['hrn']
341 slice_hrn = self.slicename_to_hrn(person['hrn'])
343 # xxx suspicious again
344 if len(person_hrn) > 64:
345 person_hrn = person_hrn[:64]
346 person_urn = hrn_to_urn(person_hrn, 'user')
349 self.logger.info("IotlabImporter: users_rec_by_email %s "
350 % (self.users_rec_by_email))
352 #Check if user using person['email'] from LDAP is already registered
353 #in SFA. One email = one person. In this case, do not create another
354 #record for this person
355 #person_hrn returned by GetPerson based on iotlab root auth +
357 user_record = self.find_record_by_type_hrn('user', person_hrn)
359 if not user_record and person['email'] in self.users_rec_by_email:
360 user_record = self.users_rec_by_email[person['email']]
361 person_hrn = user_record.hrn
362 person_urn = hrn_to_urn(person_hrn, 'user')
365 slice_record = self.find_record_by_type_hrn('slice', slice_hrn)
367 iotlab_key = person['pkey']
370 (pubkey, pkey) = self.init_person_key(person, iotlab_key)
371 if pubkey is not None and pkey is not None:
373 self.auth_hierarchy.create_gid(person_urn,
376 self.logger.debug("IOTLAB IMPORTER \
377 PERSON EMAIL OK email %s " % (person['email']))
378 person_gid.set_email(person['email'])
380 RegUser(hrn=person_hrn,
383 authority=get_authority(person_hrn),
384 email=person['email'])
387 RegUser(hrn=person_hrn,
390 authority=get_authority(person_hrn))
393 user_record.reg_keys = [RegKey(pubkey)]
395 self.logger.warning("No key found for user %s"
399 user_record.just_created()
400 dbsession.add (user_record)
402 self.logger.info("IotlabImporter: imported person \
404 self.update_just_added_records_dict(user_record)
406 except SQLAlchemyError:
407 self.logger.log_exc("IotlabImporter: \
408 failed to import person %s" % (person))
410 # update the record ?
411 # if user's primary key has changed then we need to update
412 # the users gid by forcing an update here
413 sfa_keys = user_record.reg_keys
416 if iotlab_key is not sfa_keys:
419 self.logger.info("IotlabImporter: \t \t USER UPDATE \
420 person: %s" % (person['hrn']))
421 (pubkey, pkey) = self.init_person_key(person, iotlab_key)
423 self.auth_hierarchy.create_gid(person_urn,
426 user_record.reg_keys = []
428 user_record.reg_keys = [RegKey(pubkey)]
429 self.logger.info("IotlabImporter: updated person: %s"
433 user_record.email = person['email']
437 user_record.stale = False
438 except SQLAlchemyError:
439 self.logger.log_exc("IotlabImporter: \
440 failed to update person %s"% (person))
442 self.import_slice(slice_hrn, slice_record, user_record)
445 def import_slice(self, slice_hrn, slice_record, user_record):
448 Create RegSlice record according to the slice hrn if the slice
449 does not exist yet.Creates a relationship with the user record
450 associated with the slice.
451 Commit the record to the database.
454 :param slice_hrn: Human readable name of the slice.
455 :type slice_hrn: string
456 :param slice_record: record of the slice found in the DB, if any.
457 :type slice_record: RegSlice or None
458 :param user_record: user record found in the DB if any.
459 :type user_record: RegUser
461 .. todo::Update the record if a slice record already exists.
464 pkey = Keypair(create=True)
465 urn = hrn_to_urn(slice_hrn, 'slice')
467 self.auth_hierarchy.create_gid(urn,
469 slice_record = RegSlice(hrn=slice_hrn, gid=slice_gid,
471 authority=get_authority(slice_hrn))
473 slice_record.just_created()
474 dbsession.add(slice_record)
478 self.update_just_added_records_dict(slice_record)
480 except SQLAlchemyError:
481 self.logger.log_exc("IotlabImporter: failed to import slice")
483 #No slice update upon import in iotlab
485 # xxx update the record ...
486 self.logger.warning("Slice update not yet implemented")
488 # record current users affiliated with the slice
491 slice_record.reg_researchers = [user_record]
494 slice_record.stale = False
495 except SQLAlchemyError:
496 self.logger.log_exc("IotlabImporter: failed to update slice")
499 def run(self, options):
501 Create the special iotlab table, iotlab_xp, in the iotlab database.
502 Import everything (users, slices, nodes and sites from OAR
503 and LDAP) into the SFA database.
504 Delete stale records that are no longer in OAR or LDAP.
510 iotlabdriver = IotlabDriver(config)
511 iotlab_db = IotlabDB(config)
512 #Create special slice table for iotlab
514 if not iotlab_db.exists('iotlab_xp'):
515 iotlab_db.createtable()
516 self.logger.info("IotlabImporter.run: iotlab_xp table created ")
518 # import site and node records in site into the SFA db.
519 self.import_sites_and_nodes(iotlabdriver)
520 #import users and slice into the SFA DB.
521 self.import_persons_and_slices(iotlabdriver)
523 ### remove stale records
524 # special records must be preserved
525 system_hrns = [iotlabdriver.hrn, iotlabdriver.iotlab_api.root_auth,
526 iotlabdriver.hrn + '.slicemanager']
527 for record in self.all_records:
528 if record.hrn in system_hrns:
530 if record.peer_authority:
533 for record in self.all_records:
534 if record.type == 'user':
535 self.logger.info("IotlabImporter: stale records: hrn %s %s"
536 % (record.hrn, record.stale))
541 self.logger.warning("stale not found with %s" % record)
543 self.logger.info("IotlabImporter: deleting stale record: %s"
547 dbsession.delete(record)
549 except SQLAlchemyError:
550 self.logger.log_exc("IotlabImporter: failed to delete \
551 stale record %s" % (record))