From fb54f03086be76e9b5a2c34ec63a75c13752394b Mon Sep 17 00:00:00 2001 From: Sandrine Avakian Date: Thu, 8 Aug 2013 17:27:05 +0200 Subject: [PATCH] Cleaning and comenting. --- sfa/importer/iotlabimporter.py | 277 +++++++++++++++++---------------- 1 file changed, 143 insertions(+), 134 deletions(-) diff --git a/sfa/importer/iotlabimporter.py b/sfa/importer/iotlabimporter.py index 0817dad8..3212de04 100644 --- a/sfa/importer/iotlabimporter.py +++ b/sfa/importer/iotlabimporter.py @@ -1,20 +1,24 @@ +""" File defining the importer class and all the methods needed to import +the nodes, users and slices from OAR and LDAP to the SFA database. +Also creates the iotlab specific databse and table to keep track +of which slice hrn contains which job. +""" from sfa.util.config import Config from sfa.util.xrn import Xrn, get_authority, hrn_to_urn from sfa.iotlab.iotlabdriver import IotlabDriver -from sfa.iotlab.iotlabpostgres import IotlabDB, Iotlab_xp +from sfa.iotlab.iotlabpostgres import IotlabDB from sfa.trust.certificate import Keypair, convert_public_key from sfa.trust.gid import create_uuid from sfa.storage.alchemy import dbsession from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, \ - RegUser, RegKey + RegUser, RegKey from sqlalchemy.exc import SQLAlchemyError - class IotlabImporter: """ IotlabImporter class, generic importer_class. Used to populate the SFA DB @@ -23,11 +27,11 @@ class IotlabImporter: or deleted. """ - def __init__ (self, auth_hierarchy, loc_logger): + def __init__(self, auth_hierarchy, loc_logger): """ Sets and defines import logger and the authority name. Gathers all the - records already registerd in the SFA DB, broke them into 3 dicts, - by type and hrn, by email and by type and pointer. + records already registerd in the SFA DB, broke them into 3 dicts, by + type and hrn, by email and by type and pointer. :param auth_hierarchy: authority name :type auth_hierarchy: string @@ -48,26 +52,25 @@ class IotlabImporter: #create hash by (type,hrn) #used to know if a given record is already known to SFA self.records_by_type_hrn = \ - dict([( (record.type,record.hrn), record) \ - for record in self.all_records]) + dict([((record.type, record.hrn), record) + for record in self.all_records]) self.users_rec_by_email = \ - dict([ (record.email, record) \ - for record in self.all_records if record.type == 'user']) + dict([(record.email, record) + for record in self.all_records if record.type == 'user']) # create hash by (type,pointer) self.records_by_type_pointer = \ - dict([ ( (str(record.type), record.pointer) , record) \ - for record in self.all_records if record.pointer != -1]) - + dict([((str(record.type), record.pointer), record) + for record in self.all_records if record.pointer != -1]) @staticmethod def hostname_to_hrn_escaped(root_auth, hostname): """ - Returns a node's hrn based on its hostname and the root - authority and by removing special caracters from the hostname. + Returns a node's hrn based on its hostname and the root authority and by + removing special caracters from the hostname. :param root_auth: root authority name :param hostname: nodes's hostname @@ -75,7 +78,7 @@ class IotlabImporter: :type hostname: string :rtype: string """ - return '.'.join( [root_auth, Xrn.escape(hostname)] ) + return '.'.join([root_auth, Xrn.escape(hostname)]) @staticmethod @@ -88,29 +91,33 @@ class IotlabImporter: :type person_hrn: string :rtype: string """ - return (person_hrn +'_slice') + return (person_hrn + '_slice') - def add_options (self, parser): + def add_options(self, parser): + """ + .. warning:: not used + """ # we don't have any options for now pass def find_record_by_type_hrn(self, record_type, hrn): """ - - Returns the record associated with a given hrn and hrn type. - Returns None if the key tuple is not in dictionary. + Finds the record associated with the hrn and its type given in parameter + if the tuple (hrn, type hrn) is an existing key in the dictionary. :param record_type: the record's type (slice, node, authority...) :type record_type: string :param hrn: Human readable name of the object's record :type hrn: string - :rtype: RegUser if user, RegSlice if slice, RegNode if node... - or None if record does not exist. + :returns: Returns the record associated with a given hrn and hrn type. + Returns None if the key tuple is not in the dictionary. + :rtype: RegUser if user, RegSlice if slice, RegNode if node...or None if + record does not exist. """ - return self.records_by_type_hrn.get ( (record_type, hrn), None) + return self.records_by_type_hrn.get((record_type, hrn), None) - def locate_by_type_pointer (self, record_type, pointer): + def locate_by_type_pointer(self, record_type, pointer): """ Returns the record corresponding to the key pointer and record @@ -120,15 +127,15 @@ class IotlabImporter: :param record_type: the record's type (slice, node, authority...) :type record_type: string :param pointer:Pointer to where the record is in the origin db, - used in case the record comes from a trusted authority. + used in case the record comes from a trusted authority. :type pointer: integer :rtype: RegUser if user, RegSlice if slice, RegNode if node... - or None if record does not exist. + or None if record does not exist. """ - return self.records_by_type_pointer.get ( (record_type, pointer), None) + return self.records_by_type_pointer.get((record_type, pointer), None) - def update_just_added_records_dict (self, record): + def update_just_added_records_dict(self, record): """ Updates the records_by_type_hrn dictionnary if the record has @@ -139,10 +146,10 @@ class IotlabImporter: """ rec_tuple = (record.type, record.hrn) if rec_tuple in self.records_by_type_hrn: - self.logger.warning ("IotlabImporter.update_just_added_records_dict:\ - duplicate (%s,%s)"%rec_tuple) + self.logger.warning("IotlabImporter.update_just_added_records_dict:\ + duplicate (%s,%s)" % rec_tuple) return - self.records_by_type_hrn [ rec_tuple ] = record + self.records_by_type_hrn[rec_tuple] = record def import_sites_and_nodes(self, iotlabdriver): """ @@ -152,13 +159,13 @@ class IotlabImporter: For each site, import the site's nodes to the DB by calling import_nodes. - :param iotlabdriver: IotlabDriver object, used to have access to iotlabdriver - methods and fetching info on sites and nodes. + :param iotlabdriver: IotlabDriver object, used to have access to + iotlabdriver methods and fetching info on sites and nodes. :type iotlabdriver: IotlabDriver """ - sites_listdict = iotlabdriver.iotlab_api.GetSites() - nodes_listdict = iotlabdriver.iotlab_api.GetNodes() + sites_listdict = iotlabdriver.iotlab_api.GetSites() + nodes_listdict = iotlabdriver.iotlab_api.GetNodes() nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict]) for site in sites_listdict: site_hrn = site['name'] @@ -170,28 +177,28 @@ class IotlabImporter: self.auth_hierarchy.create_auth(urn) auth_info = self.auth_hierarchy.get_auth_info(urn) - site_record = RegAuthority(hrn=site_hrn, \ - gid=auth_info.get_gid_object(), - pointer='-1', - authority=get_authority(site_hrn)) + site_record = \ + RegAuthority(hrn=site_hrn, + gid=auth_info.get_gid_object(), + pointer='-1', + authority=get_authority(site_hrn)) site_record.just_created() dbsession.add(site_record) dbsession.commit() - self.logger.info("IotlabImporter: imported authority (site) \ - %s" % site_record) + self.logger.info("IotlabImporter: imported authority \ + (site) %s" % site_record) self.update_just_added_records_dict(site_record) except SQLAlchemyError: # if the site import fails then there is no point in # trying to import the # site's child records(node, slices, persons), so skip them. - self.logger.log_exc("IotlabImporter: failed to import site. \ - Skipping child records") + self.logger.log_exc("IotlabImporter: failed to import \ + site. Skipping child records") continue else: # xxx update the record ... pass - site_record.stale = False self.import_nodes(site['node_ids'], nodes_by_id, iotlabdriver) @@ -210,8 +217,8 @@ class IotlabImporter: :param nodes_by_id: dictionary , key is the node id, value is the a dict with node information. :type nodes_by_id: dictionary - :param iotlabdriver:IotlabDriver object, used to have access to iotlabdriver - attributes. + :param iotlabdriver:IotlabDriver object, used to have access to + iotlabdriver attributes. :type iotlabdriver:IotlabDriver """ @@ -220,51 +227,55 @@ class IotlabImporter: try: node = nodes_by_id[node_id] except KeyError: - self.logger.warning ("IotlabImporter: cannot find node_id %s \ - - ignored" %(node_id)) + self.logger.warning("IotlabImporter: cannot find node_id %s \ + - ignored" % (node_id)) continue escaped_hrn = \ - self.hostname_to_hrn_escaped(iotlabdriver.iotlab_api.root_auth, \ - node['hostname']) - self.logger.info("IOTLABIMPORTER node %s " %(node)) - hrn = node['hrn'] - + self.hostname_to_hrn_escaped(iotlabdriver.iotlab_api.root_auth, + node['hostname']) + self.logger.info("IOTLABIMPORTER node %s " % (node)) + hrn = node['hrn'] # xxx this sounds suspicious if len(hrn) > 64: hrn = hrn[:64] - node_record = self.find_record_by_type_hrn( 'node', hrn ) + node_record = self.find_record_by_type_hrn('node', hrn) if not node_record: pkey = Keypair(create=True) urn = hrn_to_urn(escaped_hrn, 'node') node_gid = \ - self.auth_hierarchy.create_gid(urn, \ - create_uuid(), pkey) + self.auth_hierarchy.create_gid(urn, create_uuid(), pkey) def iotlab_get_authority(hrn): + """ Gets the authority part in the hrn. + :param hrn: hrn whose authority we are looking for. + :type hrn: string + :returns: splits the hrn using the '.' separator and returns + the authority part of the hrn. + :rtype: string + + """ return hrn.split(".")[0] node_record = RegNode(hrn=hrn, gid=node_gid, - pointer = '-1', - authority=iotlab_get_authority(hrn)) + pointer='-1', + authority=iotlab_get_authority(hrn)) try: node_record.just_created() dbsession.add(node_record) dbsession.commit() - self.logger.info("IotlabImporter: imported node: %s" \ - % node_record) + self.logger.info("IotlabImporter: imported node: %s" + % node_record) self.update_just_added_records_dict(node_record) except SQLAlchemyError: - self.logger.log_exc("IotlabImporter: \ - failed to import node") + self.logger.log_exc("IotlabImporter: failed to import node") else: #TODO: xxx update the record ... pass node_record.stale = False - - def init_person_key (self, person, iotlab_key): + def init_person_key(self, person, iotlab_key): """ Returns a tuple pubkey and pkey. @@ -272,12 +283,13 @@ class IotlabImporter: :param person Person's data. :type person: dict :param iotlab_key: SSH public key, from LDAP user's data. - RSA type supported. + RSA type supported. :type iotlab_key: string :rtype (string, Keypair) + """ pubkey = None - if person['pkey']: + if person['pkey']: # randomly pick first key in set pubkey = iotlab_key @@ -285,20 +297,19 @@ class IotlabImporter: pkey = convert_public_key(pubkey) except TypeError: #key not good. create another pkey - self.logger.warn('IotlabImporter: \ + self.logger.warn("IotlabImporter: \ unable to convert public \ - key for %s' %person['hrn']) + key for %s" % person['hrn']) pkey = Keypair(create=True) else: # the user has no keys. #Creating a random keypair for the user's gid self.logger.warn("IotlabImporter: person %s does not have a \ - public key" %(person['hrn'])) + public key" % (person['hrn'])) pkey = Keypair(create=True) return (pubkey, pkey) - def import_persons_and_slices(self, iotlabdriver): """ @@ -310,18 +321,18 @@ class IotlabImporter: import the user's slice onto the database as well by calling import_slice. - :param iotlabdriver:IotlabDriver object, used to have access to iotlabdriver - attributes. + :param iotlabdriver:IotlabDriver object, used to have access to + iotlabdriver attributes. :type iotlabdriver:IotlabDriver """ ldap_person_listdict = iotlabdriver.iotlab_api.GetPersons() - self.logger.info("IOTLABIMPORT \t ldap_person_listdict %s \r\n" \ - %(ldap_person_listdict)) + self.logger.info("IOTLABIMPORT \t ldap_person_listdict %s \r\n" + % (ldap_person_listdict)) # import persons - for person in ldap_person_listdict : + for person in ldap_person_listdict: - self.logger.info("IotlabImporter: person :" %(person)) + self.logger.info("IotlabImporter: person :" % (person)) if 'ssh-rsa' not in person['pkey']: #people with invalid ssh key (ssh-dss, empty, bullshit keys...) #won't be imported @@ -335,8 +346,8 @@ class IotlabImporter: person_urn = hrn_to_urn(person_hrn, 'user') - self.logger.info("IotlabImporter: users_rec_by_email %s " \ - %(self.users_rec_by_email)) + self.logger.info("IotlabImporter: users_rec_by_email %s " + % (self.users_rec_by_email)) #Check if user using person['email'] from LDAP is already registered #in SFA. One email = one person. In this case, do not create another @@ -345,54 +356,56 @@ class IotlabImporter: #uid ldap user_record = self.find_record_by_type_hrn('user', person_hrn) - if not user_record and person['email'] in self.users_rec_by_email: + if not user_record and person['email'] in self.users_rec_by_email: user_record = self.users_rec_by_email[person['email']] person_hrn = user_record.hrn person_urn = hrn_to_urn(person_hrn, 'user') - slice_record = self.find_record_by_type_hrn ('slice', slice_hrn) + slice_record = self.find_record_by_type_hrn('slice', slice_hrn) iotlab_key = person['pkey'] # new person if not user_record: (pubkey, pkey) = self.init_person_key(person, iotlab_key) - if pubkey is not None and pkey is not None : + if pubkey is not None and pkey is not None: person_gid = \ - self.auth_hierarchy.create_gid(person_urn, \ - create_uuid(), pkey) + self.auth_hierarchy.create_gid(person_urn, + create_uuid(), pkey) if person['email']: - self.logger.debug( "IOTLAB IMPORTER \ - PERSON EMAIL OK email %s " %(person['email'])) + self.logger.debug("IOTLAB IMPORTER \ + PERSON EMAIL OK email %s " % (person['email'])) person_gid.set_email(person['email']) - user_record = RegUser(hrn=person_hrn, \ - gid=person_gid, - pointer='-1', - authority=get_authority(person_hrn), - email=person['email']) + user_record = \ + RegUser(hrn=person_hrn, + gid=person_gid, + pointer='-1', + authority=get_authority(person_hrn), + email=person['email']) else: - user_record = RegUser(hrn=person_hrn, \ - gid=person_gid, - pointer='-1', - authority=get_authority(person_hrn)) + user_record = \ + RegUser(hrn=person_hrn, + gid=person_gid, + pointer='-1', + authority=get_authority(person_hrn)) if pubkey: user_record.reg_keys = [RegKey(pubkey)] else: - self.logger.warning("No key found for user %s" \ - %(user_record)) + self.logger.warning("No key found for user %s" + % (user_record)) try: user_record.just_created() dbsession.add (user_record) dbsession.commit() - self.logger.info("IotlabImporter: imported person %s"\ - %(user_record)) - self.update_just_added_records_dict( user_record ) + self.logger.info("IotlabImporter: imported person \ + %s" % (user_record)) + self.update_just_added_records_dict(user_record) except SQLAlchemyError: self.logger.log_exc("IotlabImporter: \ - failed to import person %s"%(person)) + failed to import person %s" % (person)) else: # update the record ? # if user's primary key has changed then we need to update @@ -400,21 +413,21 @@ class IotlabImporter: sfa_keys = user_record.reg_keys new_key = False - if iotlab_key is not sfa_keys : + if iotlab_key is not sfa_keys: new_key = True if new_key: self.logger.info("IotlabImporter: \t \t USER UPDATE \ - person: %s" %(person['hrn'])) - (pubkey, pkey) = self.init_person_key (person, iotlab_key) + person: %s" % (person['hrn'])) + (pubkey, pkey) = self.init_person_key(person, iotlab_key) person_gid = \ - self.auth_hierarchy.create_gid(person_urn, \ - create_uuid(), pkey) + self.auth_hierarchy.create_gid(person_urn, + create_uuid(), pkey) if not pubkey: user_record.reg_keys = [] else: user_record.reg_keys = [RegKey(pubkey)] - self.logger.info("IotlabImporter: updated person: %s" \ - % (user_record)) + self.logger.info("IotlabImporter: updated person: %s" + % (user_record)) if person['email']: user_record.email = person['email'] @@ -424,7 +437,7 @@ class IotlabImporter: user_record.stale = False except SQLAlchemyError: self.logger.log_exc("IotlabImporter: \ - failed to update person %s"%(person)) + failed to update person %s"% (person)) self.import_slice(slice_hrn, slice_record, user_record) @@ -447,22 +460,22 @@ class IotlabImporter: .. todo::Update the record if a slice record already exists. """ - if not slice_record : + if not slice_record: pkey = Keypair(create=True) urn = hrn_to_urn(slice_hrn, 'slice') slice_gid = \ - self.auth_hierarchy.create_gid(urn, \ - create_uuid(), pkey) - slice_record = RegSlice (hrn=slice_hrn, gid=slice_gid, - pointer='-1', - authority=get_authority(slice_hrn)) + self.auth_hierarchy.create_gid(urn, + create_uuid(), pkey) + slice_record = RegSlice(hrn=slice_hrn, gid=slice_gid, + pointer='-1', + authority=get_authority(slice_hrn)) try: slice_record.just_created() dbsession.add(slice_record) dbsession.commit() - self.update_just_added_records_dict ( slice_record ) + self.update_just_added_records_dict(slice_record) except SQLAlchemyError: self.logger.log_exc("IotlabImporter: failed to import slice") @@ -470,12 +483,12 @@ class IotlabImporter: #No slice update upon import in iotlab else: # xxx update the record ... - self.logger.warning ("Slice update not yet implemented") + self.logger.warning("Slice update not yet implemented") pass # record current users affiliated with the slice - slice_record.reg_researchers = [user_record] + slice_record.reg_researchers = [user_record] try: dbsession.commit() slice_record.stale = False @@ -483,7 +496,7 @@ class IotlabImporter: self.logger.log_exc("IotlabImporter: failed to update slice") - def run (self, options): + def run(self, options): """ Create the special iotlab table, iotlab_xp, in the iotlab database. Import everything (users, slices, nodes and sites from OAR @@ -500,8 +513,7 @@ class IotlabImporter: if not iotlab_db.exists('iotlab_xp'): iotlab_db.createtable() - self.logger.info ("IotlabImporter.run: iotlab_xp table created ") - + self.logger.info("IotlabImporter.run: iotlab_xp table created ") # import site and node records in site into the SFA db. self.import_sites_and_nodes(iotlabdriver) @@ -510,33 +522,30 @@ class IotlabImporter: ### remove stale records # special records must be preserved - system_hrns = [iotlabdriver.hrn, iotlabdriver.iotlab_api.root_auth, \ - iotlabdriver.hrn+ '.slicemanager'] + system_hrns = [iotlabdriver.hrn, iotlabdriver.iotlab_api.root_auth, + iotlabdriver.hrn + '.slicemanager'] for record in self.all_records: if record.hrn in system_hrns: record.stale = False if record.peer_authority: record.stale = False - for record in self.all_records: if record.type == 'user': - self.logger.info("IotlabImporter: stale records: hrn %s %s" \ - %(record.hrn,record.stale) ) + self.logger.info("IotlabImporter: stale records: hrn %s %s" + % (record.hrn, record.stale)) try: stale = record.stale - except : + except: stale = True - self.logger.warning("stale not found with %s"%record) + self.logger.warning("stale not found with %s" % record) if stale: - self.logger.info("IotlabImporter: deleting stale record: %s" \ - %(record)) + self.logger.info("IotlabImporter: deleting stale record: %s" + % (record)) try: dbsession.delete(record) dbsession.commit() except SQLAlchemyError: - self.logger.log_exc("IotlabImporter: failed to delete stale \ - record %s" %(record) ) - - + self.logger.log_exc("IotlabImporter: failed to delete \ + stale record %s" % (record)) -- 2.43.0