X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=sfa%2Fimporter%2Fiotlabimporter.py;h=a794a40ff596c59d9a59098a95d7e27d534571bc;hb=052e281f8c79237b6759cbb259407f071adda1cc;hp=f97cd3aca287f0a240077b7e99d57fd87afa1879;hpb=b0995f45712c5ea9dd26fdf0c4347ccba2d77fce;p=sfa.git diff --git a/sfa/importer/iotlabimporter.py b/sfa/importer/iotlabimporter.py index f97cd3ac..a794a40f 100644 --- a/sfa/importer/iotlabimporter.py +++ b/sfa/importer/iotlabimporter.py @@ -1,542 +1,40 @@ -from sfa.util.config import Config -from sfa.util.xrn import Xrn, get_authority, hrn_to_urn +# -*- coding:utf-8 -*- +""" Iot-LAB importer class management """ -from sfa.iotlab.iotlabdriver import IotlabDriver +from sfa.storage.alchemy import engine +from sfa.storage.model import init_tables +from sqlalchemy import Table, MetaData +from sqlalchemy.exc import NoSuchTableError -from sfa.trust.certificate import Keypair, convert_public_key -from sfa.trust.gid import create_uuid -from sfa.storage.alchemy import dbsession -from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, \ - RegUser, RegKey - - -from sqlalchemy.exc import SQLAlchemyError - - - -class IotlabImporter: +class IotLabImporter: """ - IotlabImporter class, generic importer_class. Used to populate the SFA DB - with iotlab resources' records. - Used to update records when new resources, users or nodes, are added - or deleted. + Creates the iotlab specific lease table to keep track + of which slice hrn match OAR job """ - def __init__ (self, auth_hierarchy, loc_logger): - """ - Sets and defines import logger and the authority name. Gathers all the - records already registerd in the SFA DB, broke them into 3 dicts, - by type and hrn, by email and by type and pointer. - - :param auth_hierarchy: authority name - :type auth_hierarchy: string - :param loc_logger: local logger - :type loc_logger: _SfaLogger - - """ - self.auth_hierarchy = auth_hierarchy + def __init__(self, auth_hierarchy, loc_logger): self.logger = loc_logger self.logger.setLevelDebug() - #retrieve all existing SFA objects - self.all_records = dbsession.query(RegRecord).all() - - # initialize record.stale to True by default, - # then mark stale=False on the ones that are in use - for record in self.all_records: - record.stale = True - #create hash by (type,hrn) - #used to know if a given record is already known to SFA - self.records_by_type_hrn = \ - dict([( (record.type,record.hrn), record) \ - for record in self.all_records]) - - self.users_rec_by_email = \ - dict([ (record.email, record) \ - for record in self.all_records if record.type == 'user']) - - # create hash by (type,pointer) - self.records_by_type_pointer = \ - dict([ ( (str(record.type), record.pointer) , record) \ - for record in self.all_records if record.pointer != -1]) - - - - @staticmethod - def hostname_to_hrn_escaped(root_auth, hostname): - """ - - Returns a node's hrn based on its hostname and the root - authority and by removing special caracters from the hostname. - - :param root_auth: root authority name - :param hostname: nodes's hostname - :type root_auth: string - :type hostname: string - :rtype: string - """ - return '.'.join( [root_auth, Xrn.escape(hostname)] ) - - - @staticmethod - def slicename_to_hrn(person_hrn): - """ - - Returns the slicename associated to a given person's hrn. - :param person_hrn: user's hrn - :type person_hrn: string - :rtype: string - """ - return (person_hrn +'_slice') - - def add_options (self, parser): - # we don't have any options for now + def add_options(self, parser): + """ Not used and need by SFA """ pass - def find_record_by_type_hrn(self, record_type, hrn): - """ - - Returns the record associated with a given hrn and hrn type. - Returns None if the key tuple is not in dictionary. - - :param record_type: the record's type (slice, node, authority...) - :type record_type: string - :param hrn: Human readable name of the object's record - :type hrn: string - :rtype: RegUser if user, RegSlice if slice, RegNode if node... - or None if record does not exist. - - """ - return self.records_by_type_hrn.get ( (record_type, hrn), None) - - def locate_by_type_pointer (self, record_type, pointer): - """ - - Returns the record corresponding to the key pointer and record - type. Returns None if the record does not exist and is not in the - records_by_type_pointer dictionnary. - - :param record_type: the record's type (slice, node, authority...) - :type record_type: string - :param pointer:Pointer to where the record is in the origin db, - used in case the record comes from a trusted authority. - :type pointer: integer - :rtype: RegUser if user, RegSlice if slice, RegNode if node... - or None if record does not exist. - """ - return self.records_by_type_pointer.get ( (record_type, pointer), None) - - - def update_just_added_records_dict (self, record): - """ - - Updates the records_by_type_hrn dictionnary if the record has - just been created. - - :param record: Record to add in the records_by_type_hrn dict. - :type record: dictionary - """ - rec_tuple = (record.type, record.hrn) - if rec_tuple in self.records_by_type_hrn: - self.logger.warning ("IotlabImporter.update_just_added_records_dict:\ - duplicate (%s,%s)"%rec_tuple) - return - self.records_by_type_hrn [ rec_tuple ] = record - - def import_sites_and_nodes(self, iotlabdriver): - """ - - Gets all the sites and nodes from OAR, process the information, - creates hrns and RegAuthority for sites, and feed them to the database. - For each site, import the site's nodes to the DB by calling - import_nodes. - - :param iotlabdriver: IotlabDriver object, used to have access to iotlabdriver - methods and fetching info on sites and nodes. - :type iotlabdriver: IotlabDriver + def _exists(self, tablename): """ - - sites_listdict = iotlabdriver.iotlab_api.GetSites() - nodes_listdict = iotlabdriver.iotlab_api.GetNodes() - nodes_by_id = dict([(node['node_id'], node) for node in nodes_listdict]) - for site in sites_listdict: - site_hrn = site['name'] - site_record = self.find_record_by_type_hrn ('authority', site_hrn) - if not site_record: - try: - urn = hrn_to_urn(site_hrn, 'authority') - if not self.auth_hierarchy.auth_exists(urn): - self.auth_hierarchy.create_auth(urn) - - auth_info = self.auth_hierarchy.get_auth_info(urn) - site_record = RegAuthority(hrn=site_hrn, \ - gid=auth_info.get_gid_object(), - pointer='-1', - authority=get_authority(site_hrn)) - site_record.just_created() - dbsession.add(site_record) - dbsession.commit() - self.logger.info("IotlabImporter: imported authority (site) \ - %s" % site_record) - self.update_just_added_records_dict(site_record) - except SQLAlchemyError: - # if the site import fails then there is no point in - # trying to import the - # site's child records(node, slices, persons), so skip them. - self.logger.log_exc("IotlabImporter: failed to import site. \ - Skipping child records") - continue - else: - # xxx update the record ... - pass - - - site_record.stale = False - self.import_nodes(site['node_ids'], nodes_by_id, iotlabdriver) - - return - - def import_nodes(self, site_node_ids, nodes_by_id, iotlabdriver): + Checks if the table exists in SFA database. """ - - Creates appropriate hostnames and RegNode records for - each node in site_node_ids, based on the information given by the - dict nodes_by_id that was made from data from OAR. - Saves the records to the DB. - - :param site_node_ids: site's node ids - :type site_node_ids: list of integers - :param nodes_by_id: dictionary , key is the node id, value is the a dict - with node information. - :type nodes_by_id: dictionary - :param iotlabdriver:IotlabDriver object, used to have access to iotlabdriver - attributes. - :type iotlabdriver:IotlabDriver - - """ - - for node_id in site_node_ids: - try: - node = nodes_by_id[node_id] - except KeyError: - self.logger.warning ("IotlabImporter: cannot find node_id %s \ - - ignored" %(node_id)) - continue - escaped_hrn = \ - self.hostname_to_hrn_escaped(iotlabdriver.iotlab_api.root_auth, \ - node['hostname']) - self.logger.info("IOTLABIMPORTER node %s " %(node)) - hrn = node['hrn'] - - - # xxx this sounds suspicious - if len(hrn) > 64: - hrn = hrn[:64] - node_record = self.find_record_by_type_hrn( 'node', hrn ) - if not node_record: - pkey = Keypair(create=True) - urn = hrn_to_urn(escaped_hrn, 'node') - node_gid = \ - self.auth_hierarchy.create_gid(urn, \ - create_uuid(), pkey) - - def iotlab_get_authority(hrn): - return hrn.split(".")[0] - - node_record = RegNode(hrn=hrn, gid=node_gid, - pointer = '-1', - authority=iotlab_get_authority(hrn)) - try: - - node_record.just_created() - dbsession.add(node_record) - dbsession.commit() - self.logger.info("IotlabImporter: imported node: %s" \ - % node_record) - self.update_just_added_records_dict(node_record) - except SQLAlchemyError: - self.logger.log_exc("IotlabImporter: \ - failed to import node") - else: - #TODO: xxx update the record ... - pass - node_record.stale = False - - - def init_person_key (self, person, iotlab_key): - """ - - Returns a tuple pubkey and pkey. - - :param person Person's data. - :type person: dict - :param iotlab_key: SSH public key, from LDAP user's data. - RSA type supported. - :type iotlab_key: string - :rtype (string, Keypair) - """ - pubkey = None - if person['pkey']: - # randomly pick first key in set - pubkey = iotlab_key - - try: - pkey = convert_public_key(pubkey) - except TypeError: - #key not good. create another pkey - self.logger.warn('IotlabImporter: \ - unable to convert public \ - key for %s' %person['hrn']) - pkey = Keypair(create=True) - - else: - # the user has no keys. - #Creating a random keypair for the user's gid - self.logger.warn("IotlabImporter: person %s does not have a \ - public key" %(person['hrn'])) - pkey = Keypair(create=True) - return (pubkey, pkey) - - - def import_persons_and_slices(self, iotlabdriver): - """ - - Gets user data from LDAP, process the information. - Creates hrn for the user's slice, the user's gid, creates - the RegUser record associated with user. Creates the RegKey record - associated nwith the user's key. - Saves those records into the SFA DB. - import the user's slice onto the database as well by calling - import_slice. - - :param iotlabdriver:IotlabDriver object, used to have access to iotlabdriver - attributes. - :type iotlabdriver:IotlabDriver - """ - ldap_person_listdict = iotlabdriver.iotlab_api.GetPersons() - self.logger.info("IOTLABIMPORT \t ldap_person_listdict %s \r\n" \ - %(ldap_person_listdict)) - - # import persons - for person in ldap_person_listdict : - - self.logger.info("IotlabImporter: person :" %(person)) - if 'ssh-rsa' not in person['pkey']: - #people with invalid ssh key (ssh-dss, empty, bullshit keys...) - #won't be imported - continue - person_hrn = person['hrn'] - slice_hrn = self.slicename_to_hrn(person['hrn']) - - # xxx suspicious again - if len(person_hrn) > 64: - person_hrn = person_hrn[:64] - person_urn = hrn_to_urn(person_hrn, 'user') - - - self.logger.info("IotlabImporter: users_rec_by_email %s " \ - %(self.users_rec_by_email)) - - #Check if user using person['email'] from LDAP is already registered - #in SFA. One email = one person. In this case, do not create another - #record for this person - #person_hrn returned by GetPerson based on iotlab root auth + - #uid ldap - user_record = self.find_record_by_type_hrn('user', person_hrn) - - if not user_record and person['email'] in self.users_rec_by_email: - user_record = self.users_rec_by_email[person['email']] - person_hrn = user_record.hrn - person_urn = hrn_to_urn(person_hrn, 'user') - - - slice_record = self.find_record_by_type_hrn ('slice', slice_hrn) - - iotlab_key = person['pkey'] - # new person - if not user_record: - (pubkey, pkey) = self.init_person_key(person, iotlab_key) - if pubkey is not None and pkey is not None : - person_gid = \ - self.auth_hierarchy.create_gid(person_urn, \ - create_uuid(), pkey) - if person['email']: - self.logger.debug( "SLAB IMPORTER \ - PERSON EMAIL OK email %s " %(person['email'])) - person_gid.set_email(person['email']) - user_record = RegUser(hrn=person_hrn, \ - gid=person_gid, - pointer='-1', - authority=get_authority(person_hrn), - email=person['email']) - else: - user_record = RegUser(hrn=person_hrn, \ - gid=person_gid, - pointer='-1', - authority=get_authority(person_hrn)) - - if pubkey: - user_record.reg_keys = [RegKey(pubkey)] - else: - self.logger.warning("No key found for user %s" \ - %(user_record)) - - try: - user_record.just_created() - dbsession.add (user_record) - dbsession.commit() - self.logger.info("IotlabImporter: imported person %s"\ - %(user_record)) - self.update_just_added_records_dict( user_record ) - - except SQLAlchemyError: - self.logger.log_exc("IotlabImporter: \ - failed to import person %s"%(person)) - else: - # update the record ? - # if user's primary key has changed then we need to update - # the users gid by forcing an update here - sfa_keys = user_record.reg_keys - - new_key = False - if iotlab_key is not sfa_keys : - new_key = True - if new_key: - self.logger.info("IotlabImporter: \t \t USER UPDATE \ - person: %s" %(person['hrn'])) - (pubkey, pkey) = self.init_person_key (person, iotlab_key) - person_gid = \ - self.auth_hierarchy.create_gid(person_urn, \ - create_uuid(), pkey) - if not pubkey: - user_record.reg_keys = [] - else: - user_record.reg_keys = [RegKey(pubkey)] - self.logger.info("IotlabImporter: updated person: %s" \ - % (user_record)) - - if person['email']: - user_record.email = person['email'] - - try: - dbsession.commit() - user_record.stale = False - except SQLAlchemyError: - self.logger.log_exc("IotlabImporter: \ - failed to update person %s"%(person)) - - self.import_slice(slice_hrn, slice_record, user_record) - - - def import_slice(self, slice_hrn, slice_record, user_record): - """ - - Create RegSlice record according to the slice hrn if the slice - does not exist yet.Creates a relationship with the user record - associated with the slice. - Commit the record to the database. - - - :param slice_hrn: Human readable name of the slice. - :type slice_hrn: string - :param slice_record: record of the slice found in the DB, if any. - :type slice_record: RegSlice or None - :param user_record: user record found in the DB if any. - :type user_record: RegUser - - .. todo::Update the record if a slice record already exists. - """ - if not slice_record : - pkey = Keypair(create=True) - urn = hrn_to_urn(slice_hrn, 'slice') - slice_gid = \ - self.auth_hierarchy.create_gid(urn, \ - create_uuid(), pkey) - slice_record = RegSlice (hrn=slice_hrn, gid=slice_gid, - pointer='-1', - authority=get_authority(slice_hrn)) - try: - slice_record.just_created() - dbsession.add(slice_record) - dbsession.commit() - - - self.update_just_added_records_dict ( slice_record ) - - except SQLAlchemyError: - self.logger.log_exc("IotlabImporter: failed to import slice") - - #No slice update upon import in iotlab - else: - # xxx update the record ... - self.logger.warning ("Slice update not yet implemented") - pass - # record current users affiliated with the slice - - - slice_record.reg_researchers = [user_record] + metadata = MetaData(bind=engine) try: - dbsession.commit() - slice_record.stale = False - except SQLAlchemyError: - self.logger.log_exc("IotlabImporter: failed to update slice") - - - def run (self, options): - """ - Create the special iotlab table, iotlab_xp, in the iotlab database. - Import everything (users, slices, nodes and sites from OAR - and LDAP) into the SFA database. - Delete stale records that are no longer in OAR or LDAP. - :param options: - :type options: - """ - config = Config() - - iotlabdriver = IotlabDriver(config) - - #Create special slice table for iotlab - - if not iotlabdriver.db.exists('iotlab_xp'): - iotlabdriver.db.createtable() - self.logger.info ("IotlabImporter.run: iotlab_xp table created ") - - - # import site and node records in site into the SFA db. - self.import_sites_and_nodes(iotlabdriver) - #import users and slice into the SFA DB. - self.import_persons_and_slices(iotlabdriver) - - ### remove stale records - # special records must be preserved - system_hrns = [iotlabdriver.hrn, iotlabdriver.iotlab_api.root_auth, \ - iotlabdriver.hrn+ '.slicemanager'] - for record in self.all_records: - if record.hrn in system_hrns: - record.stale = False - if record.peer_authority: - record.stale = False - - - for record in self.all_records: - if record.type == 'user': - self.logger.info("IotlabImporter: stale records: hrn %s %s" \ - %(record.hrn,record.stale) ) - try: - stale = record.stale - except : - stale = True - self.logger.warning("stale not found with %s"%record) - if stale: - self.logger.info("IotlabImporter: deleting stale record: %s" \ - %(record)) - - try: - dbsession.delete(record) - dbsession.commit() - except SQLAlchemyError: - self.logger.log_exc("IotlabImporter: failed to delete stale \ - record %s" %(record) ) + Table(tablename, metadata, autoload=True) + return True + except NoSuchTableError: + return False + def run(self, options): + """ Run importer""" + if not self._exists('lease_table'): + init_tables(engine) + self.logger.info("iotlabimporter run lease_table created")