From 7ca8555b412e798405d3412ae1ec536712c0eb43 Mon Sep 17 00:00:00 2001 From: Sandrine Avakian Date: Mon, 29 Apr 2013 10:23:54 +0200 Subject: [PATCH] cleaning slabimporter.py --- sfa/importer/slabimporter.py | 312 ++++++++++++++++++++--------------- 1 file changed, 180 insertions(+), 132 deletions(-) diff --git a/sfa/importer/slabimporter.py b/sfa/importer/slabimporter.py index e66f083a..1bb70819 100644 --- a/sfa/importer/slabimporter.py +++ b/sfa/importer/slabimporter.py @@ -11,7 +11,7 @@ from sfa.trust.gid import create_uuid from sfa.storage.alchemy import dbsession from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, \ RegUser, RegKey -from sfa.util.sfalogging import logger + from sqlalchemy.exc import SQLAlchemyError @@ -26,13 +26,14 @@ class SlabImporter: self.auth_hierarchy = auth_hierarchy self.logger = loc_logger self.logger.setLevelDebug() - - def hostname_to_hrn_escaped(self, root_auth, hostname): + + @staticmethod + def hostname_to_hrn_escaped(root_auth, hostname): return '.'.join( [root_auth, Xrn.escape(hostname)] ) - - def slicename_to_hrn(self, person_hrn): + @staticmethod + def slicename_to_hrn(person_hrn): return (person_hrn +'_slice') def add_options (self, parser): @@ -43,18 +44,19 @@ class SlabImporter: return self.records_by_type_hrn.get ( (record_type, hrn), None) def locate_by_type_pointer (self, record_type, pointer): - print >>sys.stderr, " \r\n \r\n \t SLABPOSTGRES locate_by_type_pointer .........................." ret = self.records_by_type_pointer.get ( (record_type, pointer), None) - print >>sys.stderr, " \r\n \r\n \t SLABPOSTGRES locate_by_type_pointer " return ret def update_just_added_records_dict (self, record): rec_tuple = (record.type, record.hrn) if rec_tuple in self.records_by_type_hrn: - self.logger.warning ("SlabImporter.update_just_added_records_dict: duplicate (%s,%s)"%rec_tuple) + self.logger.warning ("SlabImporter.update_just_added_records_dict:\ + duplicate (%s,%s)"%rec_tuple) return self.records_by_type_hrn [ rec_tuple ] = record + def import_sites(self) : + def run (self, options): config = Config() @@ -69,21 +71,25 @@ class SlabImporter: #retrieve all existing SFA objects all_records = dbsession.query(RegRecord).all() - # initialize record.stale to True by default, then mark stale=False on the ones that are in use + # initialize record.stale to True by default, + # then mark stale=False on the ones that are in use for record in all_records: record.stale = True #create hash by (type,hrn) #used to know if a given record is already known to SFA self.records_by_type_hrn = \ - dict ( [ ( (record.type,record.hrn) , record ) for record in all_records ] ) - print>>sys.stderr,"\r\n SLABIMPORT \t all_records[0] %s all_records[0].email %s \r\n" %(all_records[0].type, all_records[0]) + dict([( (record.type,record.hrn), record) \ + for record in all_records]) + self.users_rec_by_email = \ - dict ( [ (record.email, record) for record in all_records if record.type == 'user' ] ) + dict([ (record.email, record) \ + for record in all_records if record.type == 'user']) # create hash by (type,pointer) self.records_by_type_pointer = \ - dict ( [ ( (str(record.type),record.pointer) , record ) for record in all_records if record.pointer != -1] ) + dict([ ( (str(record.type), record.pointer) , record) \ + for record in all_records if record.pointer != -1]) nodes_listdict = slabdriver.slab_api.GetNodes() @@ -91,13 +97,17 @@ class SlabImporter: sites_listdict = slabdriver.slab_api.GetSites() ldap_person_listdict = slabdriver.slab_api.GetPersons() - print>>sys.stderr,"\r\n SLABIMPORT \t ldap_person_listdict %s \r\n" %(ldap_person_listdict) + print>>sys.stderr,"\r\n SLABIMPORT \t ldap_person_listdict %s \r\n" \ + %(ldap_person_listdict) slices_listdict = slabdriver.slab_api.GetSlices() - try: - slices_by_userid = dict ( [ (one_slice['reg_researchers']['record_id'], one_slice ) for one_slice in slices_listdict ] ) - except TypeError: - self.logger.log_exc("SlabImporter: failed to create list of slices by user id.") - pass + #try: + #slices_by_userid = \ + #dict([(one_slice['reg_researchers']['record_id'], one_slice ) \ + #for one_slice in slices_listdict ]) + #except TypeError: + #self.logger.log_exc("SlabImporter: failed to create list \ + #of slices by user id.") + #pass for site in sites_listdict: site_hrn = _get_site_hrn(site) @@ -107,19 +117,24 @@ class SlabImporter: urn = hrn_to_urn(site_hrn, 'authority') if not self.auth_hierarchy.auth_exists(urn): self.auth_hierarchy.create_auth(urn) + auth_info = self.auth_hierarchy.get_auth_info(urn) - site_record = RegAuthority(hrn=site_hrn, gid=auth_info.get_gid_object(), - pointer='-1', - authority=get_authority(site_hrn)) + site_record = RegAuthority(hrn=site_hrn, \ + gid=auth_info.get_gid_object(), + pointer='-1', + authority=get_authority(site_hrn)) site_record.just_created() dbsession.add(site_record) dbsession.commit() - self.logger.info("SlabImporter: imported authority (site) : %s" % site_record) + self.logger.info("SlabImporter: imported authority (site) \ + %s" % site_record) self.update_just_added_records_dict(site_record) except SQLAlchemyError: - # if the site import fails then there is no point in trying to import the + # if the site import fails then there is no point in + # trying to import the # site's child records (node, slices, persons), so skip them. - self.logger.log_exc("SlabImporter: failed to import site. Skipping child records") + self.logger.log_exc("SlabImporter: failed to import site. \ + Skipping child records") continue else: # xxx update the record ... @@ -130,12 +145,13 @@ class SlabImporter: for node_id in site['node_ids']: try: node = nodes_by_id[node_id] - except: - self.logger.warning ("SlabImporter: cannot find node_id %s - ignored"%node_id) - continue - site_auth = get_authority(site_hrn) - site_name = site['name'] - escaped_hrn = self.hostname_to_hrn_escaped(slabdriver.slab_api.root_auth, node['hostname']) + except KeyError: + self.logger.warning ("SlabImporter: cannot find node_id %s \ + - ignored" %(node_id)) + continue + escaped_hrn = \ + self.hostname_to_hrn_escaped(slabdriver.slab_api.root_auth, \ + node['hostname']) print>>sys.stderr, "\r\n \r\n SLABIMPORTER node %s " %(node) hrn = node['hrn'] @@ -144,23 +160,29 @@ class SlabImporter: if len(hrn) > 64: hrn = hrn[:64] node_record = self.find_record_by_type_hrn( 'node', hrn ) if not node_record: + pkey = Keypair(create=True) + urn = hrn_to_urn(escaped_hrn, 'node') + node_gid = \ + self.auth_hierarchy.create_gid(urn, \ + create_uuid(), pkey) + + def slab_get_authority(hrn): + return hrn.split(".")[0] + + node_record = RegNode(hrn=hrn, gid=node_gid, + pointer = '-1', + authority=slab_get_authority(hrn)) try: - pkey = Keypair(create=True) - urn = hrn_to_urn(escaped_hrn, 'node') - node_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey) - def slab_get_authority(hrn): - return hrn.split(".")[0] - - node_record = RegNode (hrn=hrn, gid=node_gid, - pointer = '-1', - authority=slab_get_authority(hrn)) + node_record.just_created() dbsession.add(node_record) dbsession.commit() - #self.logger.info("SlabImporter: imported node: %s" % node_record) + self.logger.info("SlabImporter: imported node: %s" \ + % node_record) self.update_just_added_records_dict(node_record) - except: - self.logger.log_exc("SlabImporter: failed to import node") + except SQLAlchemyError: + self.logger.log_exc("SlabImporter: \ + failed to import node") else: # xxx update the record ... pass @@ -170,8 +192,7 @@ class SlabImporter: # import persons for person in ldap_person_listdict : - - print>>sys.stderr,"SlabImporter: person: %s" %(person) + self.logger.info("SlabImporter: person :" %(person)) if 'ssh-rsa' not in person['pkey']: #people with invalid ssh key (ssh-dss, empty, bullshit keys...) #won't be imported @@ -184,13 +205,15 @@ class SlabImporter: person_urn = hrn_to_urn(person_hrn, 'user') - print>>sys.stderr," \r\n SlabImporter: HEYYYYYYYYYY" , self.users_rec_by_email + self.logger.info("SlabImporter: users_rec_by_email %s " \ + %(self.users_rec_by_email)) #Check if user using person['email'] form LDAP is already registered - #in SFA. One email = one person. Inb this case, do not create another + #in SFA. One email = one person. In this case, do not create another #record for this person - #person_hrn returned by GetPErson based on senslab root auth + uid ldap + #person_hrn returned by GetPerson based on senslab root auth + uid ldap user_record = self.find_record_by_type_hrn('user', person_hrn) + if not user_record and person['email'] in self.users_rec_by_email: user_record = self.users_rec_by_email[person['email']] person_hrn = user_record.hrn @@ -216,82 +239,107 @@ class SlabImporter: pkey = Keypair(create=True) else: - # the user has no keys. Creating a random keypair for the user's gid - self.logger.warn("SlabImporter: person %s does not have a public key"%person_hrn) + # the user has no keys. + #Creating a random keypair for the user's gid + self.logger.warn("SlabImporter: person %s does not have a \ + public key" %(person_hrn)) pkey = Keypair(create=True) return (pubkey, pkey) - try: - slab_key = person['pkey'] - # new person - if not user_record: - (pubkey,pkey) = init_person_key (person, slab_key ) - if pubkey is not None and pkey is not None : - person_gid = self.auth_hierarchy.create_gid(person_urn, create_uuid(), pkey) - if person['email']: - print>>sys.stderr, "\r\n \r\n SLAB IMPORTER PERSON EMAIL OK email %s " %(person['email']) - person_gid.set_email(person['email']) - user_record = RegUser (hrn=person_hrn, gid=person_gid, - pointer='-1', - authority=get_authority(person_hrn), - email=person['email']) - else: - user_record = RegUser (hrn=person_hrn, gid=person_gid, - pointer='-1', - authority=get_authority(person_hrn)) - - if pubkey: - user_record.reg_keys = [RegKey (pubkey)] - else: - self.logger.warning("No key found for user %s"%user_record) - user_record.just_created() - dbsession.add (user_record) - dbsession.commit() - self.logger.info("SlabImporter: imported person: %s" % user_record) - self.update_just_added_records_dict( user_record ) - else: - # update the record ? - # if user's primary key has changed then we need to update the - # users gid by forcing an update here - sfa_keys = user_record.reg_keys - - new_key=False - if slab_key is not sfa_keys : - new_key = True - if new_key: - print>>sys.stderr,"SlabImporter: \t \t USER UPDATE person: %s" %(person['hrn']) - (pubkey,pkey) = init_person_key (person, slab_key) - person_gid = self.auth_hierarchy.create_gid(person_urn, create_uuid(), pkey) - if not pubkey: - user_record.reg_keys=[] - else: - user_record.reg_keys=[ RegKey (pubkey)] - self.logger.info("SlabImporter: updated person: %s" % user_record) - + + slab_key = person['pkey'] + # new person + if not user_record: + (pubkey,pkey) = init_person_key (person, slab_key ) + if pubkey is not None and pkey is not None : + person_gid = \ + self.auth_hierarchy.create_gid(person_urn, \ + create_uuid(), pkey) if person['email']: - user_record.email = person['email'] + print>>sys.stderr, "\r\n \r\n SLAB IMPORTER \ + PERSON EMAIL OK email %s " %(person['email']) + person_gid.set_email(person['email']) + user_record = RegUser(hrn=person_hrn, \ + gid=person_gid, + pointer='-1', + authority=get_authority(person_hrn), + email=person['email']) + else: + user_record = RegUser(hrn=person_hrn, \ + gid=person_gid, + pointer='-1', + authority=get_authority(person_hrn)) + if pubkey: + user_record.reg_keys = [RegKey(pubkey)] + else: + self.logger.warning("No key found for user %s" \ + %(user_record)) + + try: + user_record.just_created() + dbsession.add (user_record) + dbsession.commit() + self.logger.info("SlabImporter: imported person: %s"\ + %(user_record)) + self.update_just_added_records_dict( user_record ) + + except SQLAlchemyError: + self.logger.log_exc("SlabImporter: \ + failed to import person %s"%(person)) + else: + # update the record ? + # if user's primary key has changed then we need to update + # the users gid by forcing an update here + sfa_keys = user_record.reg_keys + + new_key=False + if slab_key is not sfa_keys : + new_key = True + if new_key: + print>>sys.stderr,"SlabImporter: \t \t USER UPDATE \ + person: %s" %(person['hrn']) + (pubkey,pkey) = init_person_key (person, slab_key) + person_gid = \ + self.auth_hierarchy.create_gid(person_urn, \ + create_uuid(), pkey) + if not pubkey: + user_record.reg_keys = [] + else: + user_record.reg_keys = [RegKey(pubkey)] + self.logger.info("SlabImporter: updated person: %s" \ + % (user_record)) + + if person['email']: + user_record.email = person['email'] + + try: dbsession.commit() - user_record.stale = False - except: - self.logger.log_exc("SlabImporter: failed to import person %s"%(person) ) + except SQLAlchemyError: + self.logger.log_exc("SlabImporter: \ + failed to update person %s"%(person)) - try: - single_slice = slices_by_userid[user_record.record_id] - except: - self.logger.warning ("SlabImporter: cannot locate slices_by_userid[user_record.record_id] %s - ignored"%user_record) + + #try: + #single_slice = slices_by_userid[user_record.record_id] + #except KeyError: + #self.logger.warning ("SlabImporter: \ + #cannot locate slices_by_userid[user_record.record_id] %s - \ + #ignored" %(user_record)) if not slice_record : + + pkey = Keypair(create=True) + urn = hrn_to_urn(slice_hrn, 'slice') + slice_gid = \ + self.auth_hierarchy.create_gid(urn, \ + create_uuid(), pkey) + slice_record = RegSlice (hrn=slice_hrn, gid=slice_gid, + pointer='-1', + authority=get_authority(slice_hrn)) try: - pkey = Keypair(create=True) - urn = hrn_to_urn(slice_hrn, 'slice') - slice_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey) - slice_record = RegSlice (hrn=slice_hrn, gid=slice_gid, - pointer='-1', - authority=get_authority(slice_hrn)) - slice_record.just_created() dbsession.add(slice_record) dbsession.commit() @@ -300,14 +348,10 @@ class SlabImporter: #Get it sl_rec = dbsession.query(RegSlice).filter(RegSlice.hrn.match(slice_hrn)).all() - #slab_slice = SenslabXP( slice_hrn = slice_hrn, record_id_slice=sl_rec[0].record_id, record_id_user= user_record.record_id) - #print>>sys.stderr, "\r\n \r\n SLAB IMPORTER SLICE IMPORT NOTslice_record %s \r\n slab_slice %s" %(sl_rec,slab_slice) - #slab_dbsession.add(slab_slice) - #slab_dbsession.commit() - #self.logger.info("SlabImporter: imported slice: %s" % slice_record) + self.update_just_added_records_dict ( slice_record ) - except: + except SQLAlchemyError: self.logger.log_exc("SlabImporter: failed to import slice") #No slice update upon import in senslab @@ -319,8 +363,11 @@ class SlabImporter: slice_record.reg_researchers = [user_record] - dbsession.commit() - slice_record.stale=False + try: + dbsession.commit() + slice_record.stale = False + except SQLAlchemyError: + self.logger.log_exc("SlabImporter: failed to update slice") @@ -330,9 +377,9 @@ class SlabImporter: slabdriver.hrn+ '.slicemanager'] for record in all_records: if record.hrn in system_hrns: - record.stale=False + record.stale = False if record.peer_authority: - record.stale=False + record.stale = False for record in all_records: @@ -341,17 +388,18 @@ class SlabImporter: %(record.hrn,record.stale) try: stale = record.stale - except: - stale=True + except : + stale = True self.logger.warning("stale not found with %s"%record) if stale: - self.logger.info("SlabImporter: deleting stale record: %s" % record) - #if record.type == 'user': - #rec = slab_dbsession.query(SenslabXP).filter_by(record_id_user = record.record_id).first() - #slab_dbsession.delete(rec) - #slab_dbsession.commit() - dbsession.delete(record) - dbsession.commit() - + self.logger.info("SlabImporter: deleting stale record: %s" \ + %(record)) + + try: + dbsession.delete(record) + dbsession.commit() + except SQLAlchemyError: + self.logger.log_exc("SlabImporter: failed to delete stale \ + record %s" %(record) ) -- 2.43.0