From ab8c1d0f6adc51e0135caf1eb6d7cb421d3c29f7 Mon Sep 17 00:00:00 2001 From: Sandrine Avakian Date: Fri, 9 Mar 2012 15:57:51 +0100 Subject: [PATCH] Some emails from LDAP are missing = set field to None. Created slab_sfa db, and senslab_slice table. Using SQLalchemy in slabpostgres instead of psycopg ( as in sfa master). Removed unused files. TODO : use Query objects from SQLalchemy to make the filtering when selecting on senslab_slice. --- sfa/generic/slab.py | 18 +- sfa/importer/slabimporter.py | 430 +++++++++++++++++++---------------- sfa/senslab/LDAPapi.py | 11 +- sfa/senslab/sfaImport.py | 272 ---------------------- sfa/senslab/slab-import.py | 297 ------------------------ sfa/senslab/slabdriver.py | 192 ++++++++++------ sfa/senslab/slabpostgres.py | 214 +++++++++-------- sfa/senslab/table_slab.py | 182 --------------- 8 files changed, 488 insertions(+), 1128 deletions(-) delete mode 100644 sfa/senslab/sfaImport.py delete mode 100644 sfa/senslab/slab-import.py delete mode 100644 sfa/senslab/table_slab.py diff --git a/sfa/generic/slab.py b/sfa/generic/slab.py index 16c8a13b..7923af08 100644 --- a/sfa/generic/slab.py +++ b/sfa/generic/slab.py @@ -1,11 +1,9 @@ from sfa.generic import Generic import sfa.server.sfaapi -import sfa.senslab.slabdriver -import sfa.managers.registry_manager -import sfa.managers.slice_manager -#import sfa.managers.aggregate_manager_slab -import sfa.managers.aggregate_manager + + + class slab (Generic): # use the standard api class @@ -18,15 +16,21 @@ class slab (Generic): return sfa.importer.slabimporter.SlabImporter # the manager classes for the server-side services - def registry_manager_class (self) : + def registry_manager_class (self) : + import sfa.managers.registry_manager return sfa.managers.registry_manager.RegistryManager - def slicemgr_manager_class (self) : + + def slicemgr_manager_class (self) : + import sfa.managers.slice_manager return sfa.managers.slice_manager.SliceManager + def aggregate_manager_class (self) : + import sfa.managers.aggregate_manager return sfa.managers.aggregate_manager.AggregateManager # driver class for server-side services, talk to the whole testbed def driver_class (self): + import sfa.senslab.slabdriver return sfa.senslab.slabdriver.SlabDriver # slab does not have a component manager yet diff --git a/sfa/importer/slabimporter.py b/sfa/importer/slabimporter.py index 0c4740ee..f4468855 100644 --- a/sfa/importer/slabimporter.py +++ b/sfa/importer/slabimporter.py @@ -9,7 +9,7 @@ from sfa.util.plxrn import PlXrn, slicename_to_hrn, email_to_hrn, hrn_to_pl_slic from sfa.senslab.LDAPapi import LDAPapi from sfa.senslab.slabdriver import SlabDriver -from sfa.senslab.slabpostgres import SlabDB +from sfa.senslab.slabpostgres import SlabSliceDB, slab_dbsession from sfa.trust.certificate import Keypair,convert_public_key from sfa.trust.gid import create_uuid @@ -19,6 +19,7 @@ from sfa.storage.model import RegRecord, RegAuthority, RegSlice, RegNode, RegUse from sfa.storage.dbschema import DBSchema + def _get_site_hrn(site): hrn = site['login_base'] return hrn @@ -28,6 +29,7 @@ class SlabImporter: def __init__ (self, auth_hierarchy, logger): self.auth_hierarchy = auth_hierarchy self.logger=logger + def hostname_to_hrn(self,root_auth,login_base,hostname): return PlXrn(auth=root_auth,hostname=login_base+'_'+hostname).get_hrn() @@ -42,6 +44,12 @@ class SlabImporter: def find_record_by_type_hrn(self,type,hrn): return self.records_by_type_hrn.get ( (type, hrn), None) + def locate_by_type_pointer (self, type, pointer): + print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES locate_by_type_pointer .........................." + ret = self.records_by_type_pointer.get ( (type, pointer), None) + print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES locate_by_type_pointer " + return ret + def update_just_added_records_dict (self, record): tuple = (record.type, record.hrn) if tuple in self.records_by_type_hrn: @@ -51,226 +59,258 @@ class SlabImporter: def run (self, options): config = Config() - interface_hrn = config.SFA_INTERFACE_HRN - root_auth = config.SFA_REGISTRY_ROOT_AUTH + slabdriver = SlabDriver(config) #Create special slice table for senslab - db = SlabDB() - if not db.exists('slice_senslab'): - db.createtable('slice_senslab') + if not slabdriver.db.exists('slice_senslab'): + slabdriver.db.createtable('slice_senslab') + print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES CREATETABLE YAAAAAAAAAAY" ######## retrieve all existing SFA objects all_records = dbsession.query(RegRecord).all() - # create hash by (type,hrn) - # used to know if a given record is already known to SFA + #print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES all_records %s" %(all_records) + #create hash by (type,hrn) + #used to know if a given record is already known to SFA self.records_by_type_hrn = \ - dict ( [ ( (record.type, record.hrn) , record ) for record in all_records ] ) - - ## initialize record.stale to True by default, then mark stale=False on the ones that are in use - #for record in all_records: - #record.stale=True + dict ( [ ( (record.type,record.hrn) , record ) for record in all_records ] ) + + # create hash by (type,pointer) + self.records_by_type_pointer = \ + dict ( [ ( (str(record.type),record.pointer) , record ) for record in all_records if record.pointer != -1] ) + print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES self.records_by_type_pointer %s" %( self.records_by_type_pointer) + # initialize record.stale to True by default, then mark stale=False on the ones that are in use + for record in all_records: + record.stale=True - #nodes_listdict = slabdriver.GetNodes() - #nodes_by_id = dict([(node['node_id'],node) for node in nodes_listdict]) - #sites_listdict = slabdriver.GetSites() - - #ldap_person_listdict = slabdriver.GetPersons() - #slices_listdict = slabdriver.GetSlices() - #for site in sites_listdict: - #site_hrn = _get_site_hrn(site) - #site_record = self.find_record_by_type_hrn ('authority', site_hrn) - #if not site_record: - #try: - #urn = hrn_to_urn(site_hrn, 'authority') - #if not self.auth_hierarchy.auth_exists(urn): - #self.auth_hierarchy.create_auth(urn) - #auth_info = self.auth_hierarchy.get_auth_info(urn) - #site_record = RegAuthority(hrn=site_hrn, gid=auth_info.get_gid_object(), - #pointer='-1', - #authority=get_authority(site_hrn)) - #site_record.just_created() - #dbsession.add(site_record) - #dbsession.commit() - #self.logger.info("SlabImporter: imported authority (site) : %s" % site_record) - #self.update_just_added_records_dict(site_record) - #except: - ## if the site import fails then there is no point in trying to import the - ## site's child records (node, slices, persons), so skip them. - #self.logger.log_exc("SlabImporter: failed to import site. Skipping child records") - #continue - #else: - ## xxx update the record ... - #pass - #site_record.stale=False + nodes_listdict = slabdriver.GetNodes() + nodes_by_id = dict([(node['node_id'],node) for node in nodes_listdict]) + sites_listdict = slabdriver.GetSites() + + ldap_person_listdict = slabdriver.GetPersons() + slices_listdict = slabdriver.GetSlices() + try: + slices_by_userid = dict ( [ (slice.record_id_user, slice ) for slice in slices_listdict ] ) + except TypeError: + print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES slices_listdict EMPTY " + pass + #print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES slices_by_userid %s" %( slices_by_userid) + for site in sites_listdict: + site_hrn = _get_site_hrn(site) + site_record = self.find_record_by_type_hrn ('authority', site_hrn) + if not site_record: + try: + urn = hrn_to_urn(site_hrn, 'authority') + if not self.auth_hierarchy.auth_exists(urn): + self.auth_hierarchy.create_auth(urn) + auth_info = self.auth_hierarchy.get_auth_info(urn) + site_record = RegAuthority(hrn=site_hrn, gid=auth_info.get_gid_object(), + pointer='-1', + authority=get_authority(site_hrn)) + site_record.just_created() + dbsession.add(site_record) + dbsession.commit() + self.logger.info("SlabImporter: imported authority (site) : %s" % site_record) + self.update_just_added_records_dict(site_record) + except: + # if the site import fails then there is no point in trying to import the + # site's child records (node, slices, persons), so skip them. + self.logger.log_exc("SlabImporter: failed to import site. Skipping child records") + continue + else: + # xxx update the record ... + pass + site_record.stale=False - ## import node records in site - #for node_id in site['node_ids']: - #try: - #node = nodes_by_id[node_id] - #except: - #self.logger.warning ("SlabImporter: cannot find node_id %s - ignored"%node_id) - #continue - #site_auth = get_authority(site_hrn) - #site_name = site['login_base'] - #hrn = self.hostname_to_hrn(root_auth, site_name, node['hostname']) - ## xxx this sounds suspicious - #if len(hrn) > 64: hrn = hrn[:64] - #node_record = self.find_record_by_type_hrn( 'node', hrn ) + # import node records in site + for node_id in site['node_ids']: + try: + node = nodes_by_id[node_id] + except: + self.logger.warning ("SlabImporter: cannot find node_id %s - ignored"%node_id) + continue + site_auth = get_authority(site_hrn) + site_name = site['login_base'] + hrn = self.hostname_to_hrn(slabdriver.root_auth, site_name, node['hostname']) + # xxx this sounds suspicious + if len(hrn) > 64: hrn = hrn[:64] + node_record = self.find_record_by_type_hrn( 'node', hrn ) #print >>sys.stderr, " \r\n \r\n SLAB IMPORTER node_record %s " %(node_record) - #if not node_record: - #try: - #pkey = Keypair(create=True) - #urn = hrn_to_urn(hrn, 'node') + if not node_record: + try: + pkey = Keypair(create=True) + urn = hrn_to_urn(hrn, 'node') #print>>sys.stderr, "\r\n \r\n SLAB IMPORTER NODE IMPORT urn %s hrn %s" %(urn, hrn) - #node_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey) - #node_record = RegNode (hrn=hrn, gid=node_gid, - #pointer =node['node_id'], - #authority=get_authority(hrn)) - #node_record.just_created() - #dbsession.add(node_record) - #dbsession.commit() - #self.logger.info("SlabImporter: imported node: %s" % node_record) - #print>>sys.stderr, "\r\n \t\t\t SLAB IMPORTER NODE IMPORT NOTnode_record %s " %(node_record) - #self.update_just_added_records_dict(node_record) - #except: - #self.logger.log_exc("SlabImporter: failed to import node") - #else: - ## xxx update the record ... - #pass - #node_record.stale=False + node_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey) + node_record = RegNode (hrn=hrn, gid=node_gid, + pointer =node['node_id'], + authority=get_authority(hrn)) + node_record.just_created() + dbsession.add(node_record) + dbsession.commit() + self.logger.info("SlabImporter: imported node: %s" % node_record) + print>>sys.stderr, "\r\n \t\t\t SLAB IMPORTER NODE IMPORT NOTnode_record %s " %(node_record) + self.update_just_added_records_dict(node_record) + except: + self.logger.log_exc("SlabImporter: failed to import node") + else: + # xxx update the record ... + pass + node_record.stale=False - ## import persons - #for person in ldap_person_listdict : + # import persons + for person in ldap_person_listdict : - #person_hrn = person['hrn'] - #slice_hrn = self.slicename_to_hrn(person['hrn'],'_slice') + person_hrn = person['hrn'] + slice_hrn = self.slicename_to_hrn(person['hrn']) - ## xxx suspicious again - #if len(person_hrn) > 64: person_hrn = person_hrn[:64] - #person_urn = hrn_to_urn(person_hrn, 'user') + # xxx suspicious again + if len(person_hrn) > 64: person_hrn = person_hrn[:64] + person_urn = hrn_to_urn(person_hrn, 'user') - #user_record = self.find_record_by_type_hrn( 'user', person_hrn) - #slice_record = self.find_record_by_type_hrn ('slice', slice_hrn) - #print>>sys.stderr, "\r\n \r\n SLAB IMPORTER PERSON IMPORT user_record %s " %(user_record) + user_record = self.find_record_by_type_hrn( 'user', person_hrn) + slice_record = self.find_record_by_type_hrn ('slice', slice_hrn) + print>>sys.stderr, "\r\n \r\n SLAB IMPORTER FROM LDAP LIST PERSON IMPORT user_record %s " %(user_record) - ## return a tuple pubkey (a plc key object) and pkey (a Keypair object) - #def init_person_key (person, slab_key): - #pubkey=None - #if person['pkey']: - ## randomly pick first key in set - #pubkey = slab_key - #try: - #pkey = convert_public_key(pubkey) - #except: - #self.logger.warn('SlabImporter: unable to convert public key for %s' % person_hrn) - #pkey = Keypair(create=True) - #else: - ## the user has no keys. Creating a random keypair for the user's gid - #self.logger.warn("SlabImporter: person %s does not have a PL public key"%person_hrn) - #pkey = Keypair(create=True) - #return (pubkey, pkey) + # return a tuple pubkey (a plc key object) and pkey (a Keypair object) + def init_person_key (person, slab_key): + pubkey=None + if person['pkey']: + # randomly pick first key in set + pubkey = slab_key + try: + pkey = convert_public_key(pubkey) + except: + self.logger.warn('SlabImporter: unable to convert public key for %s' % person_hrn) + pkey = Keypair(create=True) + else: + # the user has no keys. Creating a random keypair for the user's gid + self.logger.warn("SlabImporter: person %s does not have a PL public key"%person_hrn) + pkey = Keypair(create=True) + return (pubkey, pkey) - #try: - #slab_key = person['pkey'] - ## new person - #if not user_record: - #(pubkey,pkey) = init_person_key (person, slab_key ) - #person_gid = self.auth_hierarchy.create_gid(person_urn, create_uuid(), pkey) - #person_gid.set_email(person['email']) - #user_record = RegUser (hrn=person_hrn, gid=person_gid, - #pointer='-1', - #authority=get_authority(person_hrn), - #email=person['email']) - #if pubkey: - #user_record.reg_keys=[RegKey (pubkey)] - #else: - #self.logger.warning("No key found for user %s"%user_record) - #user_record.just_created() - #dbsession.add (user_record) - #dbsession.commit() - #self.logger.info("SlabImporter: imported person: %s" % user_record) - #print>>sys.stderr, "\r\n \r\n SLAB IMPORTER PERSON IMPORT NOTuser_record %s " %(user_record) - #self.update_just_added_records_dict( user_record ) - #else: - ## update the record ? - ## if user's primary key has changed then we need to update the - ## users gid by forcing an update here - #sfa_keys = user_record.reg_keys - ##def key_in_list (key,sfa_keys): - ##for reg_key in sfa_keys: - ##if reg_key.key==key['key']: return True - ##return False - ## is there a new key in myplc ? - #new_keys=False - #if key is not sfa_keys : - #new_keys = True - #if new_keys: - #(pubkey,pkey) = init_person_key (person, slab_key) - #person_gid = self.auth_hierarchy.create_gid(person_urn, create_uuid(), pkey) - #if not pubkey: - #user_record.reg_keys=[] - #else: - #user_record.reg_keys=[ RegKey (pubkey)] - #self.logger.info("SlabImporter: updated person: %s" % user_record) - #user_record.email = person['email'] - #dbsession.commit() - #user_record.stale=False - #except: - #self.logger.log_exc("SlabImporter: failed to import person %s"%(person) ) - - #if not slice_record: - #try: - #pkey = Keypair(create=True) - #urn = hrn_to_urn(slice_hrn, 'slice') - #slice_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey) - #slice_record = RegSlice (hrn=slice_hrn, gid=slice_gid, - #pointer='-1', - #authority=get_authority(slice_hrn)) - #print>>sys.stderr, "\r\n \r\n SLAB IMPORTER SLICE IMPORT NOTslice_record%s " %(slice_record) - #slice_record.just_created() - #dbsession.add(slice_record) - #dbsession.commit() - #self.logger.info("SlabImporter: imported slice: %s" % slice_record) - #self.remember_record ( slice_record ) - #except: - #self.logger.log_exc("SlabImporter: failed to import slice") - #else: - ## xxx update the record ... - #self.logger.warning ("Slice update not yet implemented") - #pass - ## record current users affiliated with the slice - #slice_record.reg_researchers = \ - #[ self.locate_by_type_pointer ('user',user_id) for user_id in slice['person_ids'] ] - #dbsession.commit() - #slice_record.stale=False + try: + slab_key = person['pkey'] + # new person + if not user_record: + (pubkey,pkey) = init_person_key (person, slab_key ) + person_gid = self.auth_hierarchy.create_gid(person_urn, create_uuid(), pkey) + if person['email']: + print>>sys.stderr, "\r\n \r\n SLAB IMPORTER PERSON EMAIL OK email %s " %(person['email']) + person_gid.set_email(person['email']) + user_record = RegUser (hrn=person_hrn, gid=person_gid, + pointer='-1', + authority=get_authority(person_hrn), + email=person['email']) + else: + user_record = RegUser (hrn=person_hrn, gid=person_gid, + pointer='-1', + authority=get_authority(person_hrn)) + + if pubkey: + user_record.reg_keys=[RegKey (pubkey)] + else: + self.logger.warning("No key found for user %s"%user_record) + user_record.just_created() + dbsession.add (user_record) + dbsession.commit() + self.logger.info("SlabImporter: imported person: %s" % user_record) + print>>sys.stderr, "\r\n \r\n SLAB IMPORTER PERSON IMPORT NOTuser_record %s " %(user_record) + self.update_just_added_records_dict( user_record ) + else: + # update the record ? + # if user's primary key has changed then we need to update the + # users gid by forcing an update here + sfa_keys = user_record.reg_keys + #def key_in_list (key,sfa_keys): + #for reg_key in sfa_keys: + #if reg_key.key==key['key']: return True + #return False + # is there a new key in myplc ? + new_keys=False + if slab_key is not sfa_keys : + new_keys = True + if new_keys: + (pubkey,pkey) = init_person_key (person, slab_key) + person_gid = self.auth_hierarchy.create_gid(person_urn, create_uuid(), pkey) + if not pubkey: + user_record.reg_keys=[] + else: + user_record.reg_keys=[ RegKey (pubkey)] + self.logger.info("SlabImporter: updated person: %s" % user_record) + if person['email']: + user_record.email = person['email'] + dbsession.commit() + user_record.stale=False + except: + self.logger.log_exc("SlabImporter: failed to import person %s"%(person) ) + + try: + slice = slices_by_userid[user_record.record_id] + except: + self.logger.warning ("SlabImporter: cannot locate slices_by_userid[user_record.record_id] %s - ignored"%user_record.record_id ) + if not slice_record: + + try: + pkey = Keypair(create=True) + urn = hrn_to_urn(slice_hrn, 'slice') + slice_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey) + slice_record = RegSlice (hrn=slice_hrn, gid=slice_gid, + pointer='-1', + authority=get_authority(slice_hrn)) + + slice_record.just_created() + dbsession.add(slice_record) + dbsession.commit() + + #Serial id created after commit + #Get it + sl_rec = dbsession.query(RegSlice).filter(RegSlice.hrn.match(slice_hrn)).all() + + slab_slice = SlabSliceDB( slice_hrn = slice_hrn, record_id_slice=sl_rec[0].record_id, record_id_user= user_record.record_id) + print>>sys.stderr, "\r\n \r\n SLAB IMPORTER SLICE IMPORT NOTslice_record %s \r\n slab_slice %s" %(sl_rec,slab_slice) + slab_dbsession.add(slab_slice) + slab_dbsession.commit() + self.logger.info("SlabImporter: imported slice: %s" % slice_record) + self.update_just_added_records_dict ( slice_record ) + except: + self.logger.log_exc("SlabImporter: failed to import slice") + else: + # xxx update the record ... + self.logger.warning ("Slice update not yet implemented") + pass + # record current users affiliated with the slice + + slice_record.reg_researchers = [user_record] + dbsession.commit() + slice_record.stale=False - #### remove stale records - ## special records must be preserved - #system_hrns = [interface_hrn, root_auth, interface_hrn + '.slicemanager'] - #for record in all_records: - #if record.hrn in system_hrns: - #record.stale=False - #if record.peer_authority: - #record.stale=False + ### remove stale records + # special records must be preserved + system_hrns = [slabdriver.hrn, slabdriver.root_auth, slabdriver.hrn+ '.slicemanager'] + for record in all_records: + if record.hrn in system_hrns: + record.stale=False + if record.peer_authority: + record.stale=False - #for record in all_records: - #try: - #stale=record.stale - #except: - #stale=True - #self.logger.warning("stale not found with %s"%record) - #if stale: - #self.logger.info("SlabImporter: deleting stale record: %s" % record) - #dbsession.delete(record) - #dbsession.commit() + for record in all_records: + try: + stale=record.stale + except: + stale=True + self.logger.warning("stale not found with %s"%record) + if stale: + self.logger.info("SlabImporter: deleting stale record: %s" % record) + dbsession.delete(record) + dbsession.commit() \ No newline at end of file diff --git a/sfa/senslab/LDAPapi.py b/sfa/senslab/LDAPapi.py index 8b2976ce..55d86711 100644 --- a/sfa/senslab/LDAPapi.py +++ b/sfa/senslab/LDAPapi.py @@ -68,7 +68,7 @@ class LDAPapi : rindex=self.ldapserv.search("ou=people,dc=senslab,dc=info",ldap.SCOPE_SUBTREE,ldapfilter, ['mail','givenName', 'sn', 'uid','sshPublicKey']) ldapresponse=self.ldapserv.result(rindex,1) for ldapentry in ldapresponse[1]: - print>>sys.stderr, " \r\n \t LDAP : ! hrn ldapentry[1]['uid'][0]%s " %(ldapentry[1]['uid'][0]) + #print>>sys.stderr, " \r\n \t LDAP : ! mail ldapentry[1]['mail'][0] %s " %(ldapentry[1]['mail'][0]) tmpname = ldapentry[1]['uid'][0] @@ -76,8 +76,10 @@ class LDAPapi : tmpname = 'avakian' hrn=self.authname+"."+ tmpname - - + + tmpemail = ldapentry[1]['mail'][0] + if ldapentry[1]['mail'][0] == "unknown": + tmpemail = None # uuid=create_uuid() # RSA_KEY_STRING=ldapentry[1]['sshPublicKey'][0] @@ -94,7 +96,8 @@ class LDAPapi : 'pkey': ldapentry[1]['sshPublicKey'][0], #'uid': ldapentry[1]['uid'][0], 'uid': tmpname , - 'email': ldapentry[1]['mail'][0], + 'email':tmpemail, + #'email': ldapentry[1]['mail'][0], 'first_name': ldapentry[1]['givenName'][0], 'last_name': ldapentry[1]['sn'][0], # 'phone': 'none', diff --git a/sfa/senslab/sfaImport.py b/sfa/senslab/sfaImport.py deleted file mode 100644 index 3f85f4bb..00000000 --- a/sfa/senslab/sfaImport.py +++ /dev/null @@ -1,272 +0,0 @@ -# -# The import tool assumes that the existing PLC hierarchy should all be part -# of "planetlab.us" (see the root_auth and level1_auth variables below). -# -# Public keys are extracted from the users' SSH keys automatically and used to -# create GIDs. This is relatively experimental as a custom tool had to be -# written to perform conversion from SSH to OpenSSL format. It only supports -# RSA keys at this time, not DSA keys. -## - -import getopt -import sys -import tempfile - -from sfa.util.sfalogging import sfa_logger_goes_to_import,sfa_logger - -from sfa.util.record import * -from sfa.util.table import SfaTable -from sfa.util.xrn import get_authority, hrn_to_urn -from sfa.util.plxrn import email_to_hrn -from sfa.util.config import Config -from sfa.trust.certificate import convert_public_key, Keypair -from sfa.trust.trustedroot import * -from sfa.trust.hierarchy import * -from sfa.trust.gid import create_uuid - - -def _un_unicode(str): - if isinstance(str, unicode): - return str.encode("ascii", "ignore") - else: - return str - -def _cleanup_string(str): - # pgsql has a fit with strings that have high ascii in them, so filter it - # out when generating the hrns. - tmp = "" - for c in str: - if ord(c) < 128: - tmp = tmp + c - str = tmp - - str = _un_unicode(str) - str = str.replace(" ", "_") - str = str.replace(".", "_") - str = str.replace("(", "_") - str = str.replace("'", "_") - str = str.replace(")", "_") - str = str.replace('"', "_") - return str - -class sfaImport: - - def __init__(self): - sfa_logger_goes_to_import() - self.logger = sfa_logger() - self.AuthHierarchy = Hierarchy() - self.config = Config() - self.TrustedRoots = TrustedRootList(Config.get_trustedroots_dir(self.config)) - - self.plc_auth = self.config.get_plc_auth() - self.root_auth = self.config.SFA_REGISTRY_ROOT_AUTH - print>>sys.stderr, "\r\n ========= \t\t sfaImport plc_auth %s root_auth %s \r\n" %( self.plc_auth, self.root_auth ) - # connect to planetlab - self.shell = None - if "Url" in self.plc_auth: - from sfa.plc.remoteshell import RemoteShell - self.shell = RemoteShell(self.logger) - else: - import PLC.Shell - self.shell = PLC.Shell.Shell(globals = globals()) - - def create_top_level_auth_records(self, hrn): - """ - Create top level records (includes root and sub authorities (local/remote) - """ - - urn = hrn_to_urn(hrn, 'authority') - # make sure parent exists - parent_hrn = get_authority(hrn) - if not parent_hrn: - parent_hrn = hrn - if not parent_hrn == hrn: - self.create_top_level_auth_records(parent_hrn) - print>>sys.stderr, "\r\n =========create_top_level_auth_records parent_hrn \r\n", parent_hrn - - # create the authority if it doesnt already exist - if not self.AuthHierarchy.auth_exists(urn): - self.logger.info("Import: creating top level authorities") - self.AuthHierarchy.create_auth(urn) - - # create the db record if it doesnt already exist - auth_info = self.AuthHierarchy.get_auth_info(hrn) - table = SfaTable() - auth_record = table.find({'type': 'authority', 'hrn': hrn}) - - if not auth_record: - auth_record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=-1) - auth_record['authority'] = get_authority(auth_record['hrn']) - self.logger.info("Import: inserting authority record for %s"%hrn) - table.insert(auth_record) - print>>sys.stderr, "\r\n ========= \t\t NO AUTH RECORD \r\n" ,auth_record['authority'] - - - def create_interface_records(self): - """ - Create a record for each SFA interface - """ - # just create certs for all sfa interfaces even if they - # arent enabled - interface_hrn = self.config.SFA_INTERFACE_HRN - interfaces = ['authority+sa', 'authority+am', 'authority+sm'] - table = SfaTable() - auth_info = self.AuthHierarchy.get_auth_info(interface_hrn) - pkey = auth_info.get_pkey_object() - for interface in interfaces: - interface_record = table.find({'type': interface, 'hrn': interface_hrn}) - if not interface_record: - self.logger.info("Import: interface %s %s " % (interface_hrn, interface)) - urn = hrn_to_urn(interface_hrn, interface) - gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey) - record = SfaRecord(hrn=interface_hrn, gid=gid, type=interface, pointer=-1) - record['authority'] = get_authority(interface_hrn) - print>>sys.stderr,"\r\n ==========create_interface_records", record['authority'] - table.insert(record) - - def import_person(self, parent_hrn, person): - """ - Register a user record - """ - hrn = email_to_hrn(parent_hrn, person['email']) - - print >>sys.stderr , "\r\n_____00______SfaImport : person", person - # ASN.1 will have problems with hrn's longer than 64 characters - if len(hrn) > 64: - hrn = hrn[:64] - print >>sys.stderr , "\r\n_____0______SfaImport : parent_hrn", parent_hrn - self.logger.info("Import: person %s"%hrn) - key_ids = [] - if 'key_ids' in person and person['key_ids']: - key_ids = person["key_ids"] - # get the user's private key from the SSH keys they have uploaded - # to planetlab - keys = self.shell.GetKeys(self.plc_auth, key_ids) - print >>sys.stderr , "\r\n_____1______SfaImport : self.plc_auth %s \r\n \t keys %s " %(self.plc_auth,keys) - key = keys[0]['key'] - pkey = convert_public_key(key) - print >>sys.stderr , "\r\n_____2______SfaImport : key %s pkey %s"% (key,pkey.as_pem()) - if not pkey: - pkey = Keypair(create=True) - else: - # the user has no keys - self.logger.warning("Import: person %s does not have a PL public key"%hrn) - # if a key is unavailable, then we still need to put something in the - # user's GID. So make one up. - pkey = Keypair(create=True) - print >>sys.stderr , "\r\n___ELSE________SfaImport pkey : %s \r\n \t pkey.key.bits%s "%(dir(pkey.key), pkey.as_pem()) - # create the gid - urn = hrn_to_urn(hrn, 'user') - print >>sys.stderr , "\r\n \t\t : urn ", urn - person_gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey) - table = SfaTable() - person_record = SfaRecord(hrn=hrn, gid=person_gid, type="user", pointer=person['person_id']) - person_record['authority'] = get_authority(person_record['hrn']) - existing_records = table.find({'hrn': hrn, 'type': 'user', 'pointer': person['person_id']}) - if not existing_records: - table.insert(person_record) - else: - self.logger.info("Import: %s exists, updating " % hrn) - existing_record = existing_records[0] - person_record['record_id'] = existing_record['record_id'] - table.update(person_record) - - def import_slice(self, parent_hrn, slice): - slicename = slice['name'].split("_",1)[-1] - slicename = _cleanup_string(slicename) - - if not slicename: - self.logger.error("Import: failed to parse slice name %s" %slice['name']) - return - - hrn = parent_hrn + "." + slicename - self.logger.info("Import: slice %s"%hrn) - - pkey = Keypair(create=True) - urn = hrn_to_urn(hrn, 'slice') - slice_gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey) - slice_record = SfaRecord(hrn=hrn, gid=slice_gid, type="slice", pointer=slice['slice_id']) - slice_record['authority'] = get_authority(slice_record['hrn']) - table = SfaTable() - existing_records = table.find({'hrn': hrn, 'type': 'slice', 'pointer': slice['slice_id']}) - if not existing_records: - table.insert(slice_record) - else: - self.logger.info("Import: %s exists, updating " % hrn) - existing_record = existing_records[0] - slice_record['record_id'] = existing_record['record_id'] - table.update(slice_record) - - def import_node(self, hrn, node): - self.logger.info("Import: node %s" % hrn) - # ASN.1 will have problems with hrn's longer than 64 characters - if len(hrn) > 64: - hrn = hrn[:64] - - table = SfaTable() - node_record = table.find({'type': 'node', 'hrn': hrn}) - pkey = Keypair(create=True) - urn = hrn_to_urn(hrn, 'node') - node_gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey) - node_record = SfaRecord(hrn=hrn, gid=node_gid, type="node", pointer=node['node_id']) - node_record['authority'] = get_authority(node_record['hrn']) - existing_records = table.find({'hrn': hrn, 'type': 'node', 'pointer': node['node_id']}) - if not existing_records: - table.insert(node_record) - else: - self.logger.info("Import: %s exists, updating " % hrn) - existing_record = existing_records[0] - node_record['record_id'] = existing_record['record_id'] - table.update(node_record) - - - def import_site(self, parent_hrn, site): - shell = self.shell - plc_auth = self.plc_auth - print >>sys.stderr , " \r\n !!!!!!!!! import_site plc_shell %s \r\n \t type %s dir %s" %(shell, type(shell),dir(shell)) - sitename = site['login_base'] - sitename = _cleanup_string(sitename) - hrn = parent_hrn + "." + sitename - - # Hardcode 'internet2' into the hrn for sites hosting - # internet2 nodes. This is a special operation for some vini - # sites only - if ".vini" in parent_hrn and parent_hrn.endswith('vini'): - if sitename.startswith("i2"): - #sitename = sitename.replace("ii", "") - hrn = ".".join([parent_hrn, "internet2", sitename]) - elif sitename.startswith("nlr"): - #sitename = sitename.replace("nlr", "") - hrn = ".".join([parent_hrn, "internet2", sitename]) - - urn = hrn_to_urn(hrn, 'authority') - self.logger.info("Import: site %s"%hrn) - print >>sys.stderr , " \r\n !!!!!!!!! import_site sitename %s sitename %s \r\n \t hrn %s urn %s" %(site['login_base'],sitename, hrn,urn) - # create the authority - if not self.AuthHierarchy.auth_exists(urn): - self.AuthHierarchy.create_auth(urn) - - auth_info = self.AuthHierarchy.get_auth_info(urn) - - table = SfaTable() - auth_record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=site['site_id']) - auth_record['authority'] = get_authority(auth_record['hrn']) - existing_records = table.find({'hrn': hrn, 'type': 'authority', 'pointer': site['site_id']}) - if not existing_records: - table.insert(auth_record) - else: - self.logger.info("Import: %s exists, updating " % hrn) - existing_record = existing_records[0] - auth_record['record_id'] = existing_record['record_id'] - table.update(auth_record) - - return hrn - - - def delete_record(self, hrn, type): - # delete the record - table = SfaTable() - record_list = table.find({'type': type, 'hrn': hrn}) - for record in record_list: - self.logger.info("Import: removing record %s %s" % (type, hrn)) - table.remove(record) diff --git a/sfa/senslab/slab-import.py b/sfa/senslab/slab-import.py deleted file mode 100644 index 888dcccc..00000000 --- a/sfa/senslab/slab-import.py +++ /dev/null @@ -1,297 +0,0 @@ - -import sys -import datetime -import time -#from sfa.senslab.OARrestapi import OARapi -from sfa.senslab.LDAPapi import LDAPapi -from sfa.senslab.slabdriver import SlabDriver -from sfa.senslab.slabpostgres import SlabDB -from sfa.util.config import Config -from sfa.util.plxrn import PlXrn -from sfa.util.xrn import hrn_to_urn, get_authority,Xrn,get_leaf -from sfa.storage.table import SfaTable -from sfa.storage.record import SfaRecord -from sfa.trust.hierarchy import Hierarchy -from sfa.trust.certificate import Keypair,convert_public_key -from sfa.trust.gid import create_uuid -from sfa.trust.trustedroots import TrustedRoots - -config = Config() -interface_hrn = config.SFA_INTERFACE_HRN -TrustedR = TrustedRoots(Config.get_trustedroots_dir(config)) -AuthHierarchy = Hierarchy() -table = SfaTable() -db = SlabDB() - - - -def create_sm_client_record(): - """ - Create a user record for the Slicemanager service. - """ - hrn = config.SFA_INTERFACE_HRN + '.slicemanager' - urn = hrn_to_urn(hrn, 'user') - if not AuthHierarchy.auth_exists(urn): - AuthHierarchy.create_auth(urn) - - auth_info = AuthHierarchy.get_auth_info(hrn) - table = SfaTable() - sm_user_record = table.find({'type': 'user', 'hrn': hrn}) - if not sm_user_record: - record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="user", pointer=-1) - record['authority'] = get_authority(record['hrn']) - table.insert(record) - -def create_interface_records(): - """ - Create a record for each SFA interface - """ - # just create certs for all sfa interfaces even if they - # arent enabled - interface_hrn = config.SFA_INTERFACE_HRN - interfaces = ['authority+sa', 'authority+am', 'authority+sm'] - - auth_info = AuthHierarchy.get_auth_info(interface_hrn) - pkey = auth_info.get_pkey_object() - for interface in interfaces: - interface_record = table.find({'type': interface, 'hrn': interface_hrn}) - if not interface_record: - urn = hrn_to_urn(interface_hrn, interface) - gid = AuthHierarchy.create_gid(urn, create_uuid(), pkey) - record = SfaRecord(hrn=interface_hrn, gid=gid, type=interface, pointer=-1) - record['authority'] = get_authority(interface_hrn) - print>>sys.stderr,"\r\n ==========create_interface_records", record['authority'] - table.insert(record) - -def create_top_level_auth_records(hrn): - """ - Create top level records (includes root and sub authorities (local/remote) - """ - - urn = hrn_to_urn(hrn, 'authority') - # make sure parent exists - parent_hrn = get_authority(hrn) - print>>sys.stderr, "\r\n =========slab-import create_top_level_auth_records hrn %s urn %s parent_hrn %s \r\n" %(hrn, urn, parent_hrn) - if not parent_hrn: - parent_hrn = hrn - if not parent_hrn == hrn: - create_top_level_auth_records(parent_hrn) - - # enxure key and cert exists: - AuthHierarchy.create_top_level_auth(hrn) - - # create the authority if it doesnt already exist - #if not AuthHierarchy.auth_exists(urn): - #AuthHierarchy.create_auth(urn) - - # create the db record if it doesnt already exist - auth_info = AuthHierarchy.get_auth_info(hrn) - - auth_record = table.find({'type': 'authority', 'hrn': hrn}) - - if not auth_record: - auth_record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=-1) - auth_record['authority'] = get_authority(auth_record['hrn']) - print sys.stderr, " \r\n \t slab-import : auth record %s inserted record %s " %(auth_record['hrn'], auth_record) - table.insert(auth_record) - - - -def import_node(hrn, node): - - # ASN.1 will have problems with hrn's longer than 64 characters - if len(hrn) > 64: - hrn = hrn[:64] - - node_record = table.find({'type': 'node', 'hrn': hrn}) - pkey = Keypair(create=True) - - urn = hrn_to_urn(hrn, 'node') - node_gid = AuthHierarchy.create_gid(urn, create_uuid(), pkey) - node_record = SfaRecord(hrn=hrn, gid=node_gid, type="node", pointer=node['node_id']) - node_record['authority'] = get_authority(node_record['hrn']) - extime = datetime.datetime.utcnow() - node_record['date_created'] = int(time.mktime(extime.timetuple())) - existing_records = table.find({'hrn': hrn, 'type': 'node', 'pointer': node['node_id']}) - if not existing_records: - print>>sys.stderr, " \r\n \t slab-import : node record[hrn]%s inserted" %(node['hrn']) - table.insert(node_record) - else: - existing_record = existing_records[0] - node_record['record_id'] = existing_record['record_id'] - table.update(node_record) - -# person is already a sfa record -def import_person(authname,person): - existing_records = table.find({'hrn': person['hrn'], 'type': 'user'}) - extime = datetime.datetime.utcnow() - person['date_created'] = int(time.mktime(extime.timetuple())) - - - if not existing_records: - print>>sys.stderr, " \r\n \t slab-import : person record %s inserted" %(person['hrn']) - uuid=create_uuid() - RSA_KEY_STRING=person['pkey'] - pkey=convert_public_key(RSA_KEY_STRING) - person['gid']=AuthHierarchy.create_gid("urn:publicid:IDN+"+authname+"+user+"+person['uid'], uuid, pkey, CA=False).save_to_string() - table.insert(person) - else: - existing_record = existing_records[0] - person['record_id'] = existing_record['record_id'] - # handle key change ??? - table.update(person) - -def import_slice(person): - - hrn = person['hrn']+'_slice' - pkey = Keypair(create=True) - urn = hrn_to_urn(hrn, 'slice') - gid = AuthHierarchy.create_gid(urn, create_uuid(), pkey) - slice_record= SfaRecord(hrn=hrn, gid=gid, type="slice", pointer=-1) - slice_record['authority'] = get_authority(slice_record['hrn']) - - extime = datetime.datetime.utcnow() - slice_record['date_created'] = int(time.mktime(extime.timetuple())) - #special slice table for Senslab, to store nodes info (OAR) - - existing_records = table.find({'hrn': slice_record['hrn'], 'type': 'slice'}) - if not existing_records: - print>>sys.stderr, " \r\n \t slab-import : slice record %s inserted" %(slice_record['hrn']) - table.insert(slice_record) - db.insert_slab_slice(person) - - else: - print>>sys.stderr, " \r\n \t slab-import : slice record %s updated" %(slice_record['hrn']) - existing_record = existing_records[0] - slice_record['record_id'] = existing_record['record_id'] - table.update(slice_record) - db.update_senslab_slice(slice_record) - -def delete_record( hrn, type): - # delete the record - record_list = table.find({'type': type, 'hrn': hrn}) - for record in record_list: - print>>sys.stderr, " \r\n \t slab-import : record %s deleted" %(record['hrn']) - table.remove(record) - -def hostname_to_hrn(root_auth,login_base,hostname): - return PlXrn(auth=root_auth,hostname=login_base+'_'+hostname).get_hrn() - - -def main(): - - if not db.exists('slice'): - db.createtable('slice') - - if not config.SFA_REGISTRY_ENABLED: - sys.exit(0) - root_auth = config.SFA_REGISTRY_ROOT_AUTH - interface_hrn = config.SFA_INTERFACE_HRN - - - #Get all records in the sfa table - # create dict of all existing sfa records - existing_records = {} - existing_hrns = [] - key_ids = [] - results = table.find() - - for result in results: - existing_records[(result['hrn'], result['type'])] = result - existing_hrns.append(result['hrn']) - - # create root authority if it doesn't exist - if root_auth not in existing_hrns or \ - (root_auth, 'authority') not in existing_records: - create_top_level_auth_records(root_auth) - if not root_auth == interface_hrn: - create_top_level_auth_records(interface_hrn) - - # create s user record for the slice manager Do we need this? - create_sm_client_record() - - # create interface records ADDED 18 nov 11 Do we need this? - - create_interface_records() - - # add local root authority's cert to trusted list ADDED 18 nov 11 Do we need this? - - authority = AuthHierarchy.get_auth_info(interface_hrn) - TrustedR.add_gid(authority.get_gid_object()) - - - #Get Senslab nodes - - Driver = SlabDriver(config) - nodes_dict = Driver.GetNodes() - #print "\r\n NODES8DICT ",nodes_dict - - ldap_person_list = Driver.GetPersons() - - # import node records - for node in nodes_dict: - # Sandrine - # A changer pour l utilisation du nouveau OAR de prod, le site etant contenu dans le hostname - hrn = node['hrn'] - #hrn = hostname_to_hrn( root_auth,node['site_login_base'], node['hostname']) - if hrn not in existing_hrns or \ - (hrn, 'node') not in existing_records: - import_node(hrn, node) - - # import persons and slices - for person in ldap_person_list: - if person['hrn'] not in existing_hrns or \ - (person['hrn'], 'user') not in existing_records : - import_person(root_auth,person) - import_slice(person) - - - # remove stale records - system_records = [interface_hrn, root_auth, interface_hrn + '.slicemanager'] - - for (record_hrn, type) in existing_records.keys(): - if record_hrn in system_records: - continue - - record = existing_records[(record_hrn, type)] - if record['peer_authority']: - continue - - - - found = False - - if type == 'authority': - found = True - print "\t \t Found :", found - break - - elif type == 'user': - for person in ldap_person_list: - if person['hrn'] == record_hrn: - found = True - break - - elif type == 'node': - login_base = get_leaf(get_authority(record_hrn)) - nodename = Xrn.unescape(get_leaf(record_hrn)) - for node in nodes_dict: - if node['hostname'] == nodename : - found = True - break - - elif type == 'slice': - for person in ldap_person_list: - if person['hrn']+'_slice' == record_hrn: - found = True - break - else: - continue - - if not found: - record_object = existing_records[(record_hrn, type)] - print "\t\t NOT FOUND ! ", record_hrn - delete_record(record_hrn, type) - -if __name__ == "__main__": - main() diff --git a/sfa/senslab/slabdriver.py b/sfa/senslab/slabdriver.py index d7c68780..894dada3 100644 --- a/sfa/senslab/slabdriver.py +++ b/sfa/senslab/slabdriver.py @@ -6,9 +6,14 @@ from time import gmtime, strftime from sfa.util.faults import MissingSfaInfo , SliverDoesNotExist #from sfa.util.sfatime import datetime_to_string from sfa.util.sfalogging import logger -from sfa.storage.table import SfaTable +#from sfa.storage.table import SfaTable from sfa.util.defaultdict import defaultdict +from sfa.storage.record import Record +from sfa.storage.alchemy import dbsession +from sfa.storage.model import RegRecord + + from sfa.trust.certificate import * from sfa.trust.credential import * from sfa.trust.gid import GID @@ -18,7 +23,7 @@ from sfa.rspecs.version_manager import VersionManager from sfa.rspecs.rspec import RSpec from sfa.util.xrn import hrn_to_urn, urn_to_sliver_id -from sfa.util.plxrn import slicename_to_hrn, hostname_to_hrn, hrn_to_pl_slicename, hrn_to_pl_login_base +from sfa.util.plxrn import slicename_to_hrn, hostname_to_hrn, hrn_to_pl_slicename ## thierry: everything that is API-related (i.e. handling incoming requests) # is taken care of @@ -29,7 +34,7 @@ from sfa.senslab.OARrestapi import OARrestapi from sfa.senslab.LDAPapi import LDAPapi #from sfa.senslab.SenslabImportUsers import SenslabImportUsers from sfa.senslab.parsing import parse_filter -from sfa.senslab.slabpostgres import SlabDB +from sfa.senslab.slabpostgres import SlabDB, slab_dbsession,SlabSliceDB from sfa.senslab.slabaggregate import SlabAggregate from sfa.senslab.slabslices import SlabSlices @@ -66,7 +71,7 @@ class SlabDriver(Driver): self.ldap = LDAPapi() #self.users = SenslabImportUsers() self.time_format = "%Y-%m-%d %H:%M:%S" - self.db = SlabDB() + self.db = SlabDB(config) #self.logger=sfa_logger() self.cache=None @@ -97,16 +102,16 @@ class SlabDriver(Driver): if nodes: top_level_status = 'ready' result['geni_urn'] = slice_urn - result['pl_login'] = sl['job_user'] + result['slab_login'] = sl['job_user'] timestamp = float(sl['startTime']) + float(sl['walltime']) - result['pl_expires'] = strftime(self.time_format, gmtime(float(timestamp))) + result['slab_expires'] = strftime(self.time_format, gmtime(float(timestamp))) resources = [] for node in nodes: res = {} - res['pl_hostname'] = node['hostname'] - res['pl_boot_state'] = node['boot_state'] + res['slab_hostname'] = node['hostname'] + res['slab_boot_state'] = node['boot_state'] sliver_id = urn_to_sliver_id(slice_urn, sl['record_id_slice'], node['node_id']) res['geni_urn'] = sliver_id @@ -265,24 +270,24 @@ class SlabDriver(Driver): #No site or node register supported def register (self, sfa_record, hrn, pub_key): type = sfa_record['type'] - pl_record = self.sfa_fields_to_pl_fields(type, hrn, sfa_record) + slab_record = self.sfa_fields_to_slab_fields(type, hrn, sfa_record) #if type == 'authority': - #sites = self.shell.GetSites([pl_record['login_base']]) + #sites = self.shell.GetSites([slab_record['login_base']]) #if not sites: - #pointer = self.shell.AddSite(pl_record) + #pointer = self.shell.AddSite(slab_record) #else: #pointer = sites[0]['site_id'] if type == 'slice': acceptable_fields=['url', 'instantiation', 'name', 'description'] - for key in pl_record.keys(): + for key in slab_record.keys(): if key not in acceptable_fields: - pl_record.pop(key) + slab_record.pop(key) print>>sys.stderr, " \r\n \t\t SLABDRIVER.PY register" - slices = self.GetSlices([pl_record['hrn']]) + slices = self.GetSlices([slab_record['hrn']]) if not slices: - pointer = self.AddSlice(pl_record) + pointer = self.AddSlice(slab_record) else: pointer = slices[0]['slice_id'] @@ -312,10 +317,10 @@ class SlabDriver(Driver): #No node adding outside OAR #elif type == 'node': - #login_base = hrn_to_pl_login_base(sfa_record['authority']) - #nodes = self.GetNodes([pl_record['hostname']]) + #login_base = hrn_to_slab_login_base(sfa_record['authority']) + #nodes = self.GetNodes([slab_record['hostname']]) #if not nodes: - #pointer = self.AddNode(login_base, pl_record) + #pointer = self.AddNode(login_base, slab_record) #else: #pointer = nodes[0]['node_id'] @@ -334,10 +339,10 @@ class SlabDriver(Driver): #self.shell.UpdateSite(pointer, new_sfa_record) if type == "slice": - pl_record=self.sfa_fields_to_pl_fields(type, hrn, new_sfa_record) - if 'name' in pl_record: - pl_record.pop('name') - self.UpdateSlice(pointer, pl_record) + slab_record=self.sfa_fields_to_slab_fields(type, hrn, new_sfa_record) + if 'name' in slab_record: + slab_record.pop('name') + self.UpdateSlice(pointer, slab_record) elif type == "user": update_fields = {} @@ -395,9 +400,28 @@ class SlabDriver(Driver): return True def GetPeers (self,auth = None, peer_filter=None, return_fields=None): - table = SfaTable() + + existing_records = {} + existing_hrns_by_types= {} + all_records = dbsession.query(RegRecord).all + for record in all_records: + existing_records[record.hrn] = record + if record.type not in existing_hrns_by_types: + existing_hrns_by_types[record.type] = [record.hrn] + else: + existing_hrns_by_types.update({record.type:(existing_hrns_by_types[record.type].append(record.hrn))}) + + print >>sys.stderr, "\r\n \r\n SLABDRIVER GetPeers existing_hrns_by_types %s " %( existing_hrns_by_types) return_records = [] - records_list = table.findObjects({'type':'authority+sa'}) + #records_list = table.findObjects({'type':'authority+sa'}) + try: + for hrn in existing_hrns_by_types['authority+sa']: + records_list.append(existing_records[hrn]) + print >>sys.stderr, "\r\n \r\n SLABDRIVER GetPeers records_list %s " %(records_list) + + except: + pass + if not peer_filter and not return_fields: return records_list return_records = parse_filter(records_list,peer_filter, 'peers', return_fields) @@ -520,21 +544,25 @@ class SlabDriver(Driver): return_site_list = parse_filter(site_dict.values(), site_filter,'site', return_fields) return return_site_list - + #TODO : filtrer au niveau de la query voir sqlalchemy + #http://docs.sqlalchemy.org/en/latest/orm/tutorial.html#returning-lists-and-scalars def GetSlices(self,slice_filter = None, return_fields=None): - sliceslist = self.db.find('slice_senslab',columns = ['oar_job_id', 'slice_hrn', 'record_id_slice','record_id_user'], record_filter=slice_filter) - + #sliceslist = self.db.find('slice_senslab',columns = ['oar_job_id', 'slice_hrn', 'record_id_slice','record_id_user'], record_filter=slice_filter) + sliceslist = slab_dbsession.query(SlabSliceDB).all() + #sliceslist = slices_records.order_by("record_id_slice").all() + print >>sys.stderr, " \r\n \r\n \tSLABDRIVER.PY GetSlices slices %s slice_filter %s " %(sliceslist,slice_filter) - + return_slice_list = parse_filter(sliceslist, slice_filter,'slice', return_fields) if return_slice_list: for sl in return_slice_list: - login = sl['slice_hrn'].split(".")[1].split("_")[0] + #login = sl['slice_hrn'].split(".")[1].split("_")[0] + login = sl.slice_hrn.split(".")[1].split("_")[0] print >>sys.stderr, " \r\n \r\n \tSLABDRIVER.PY GetSlices sl %s " %(sl) - if sl['oar_job_id'] is not -1: - rslt = self.GetJobs( sl['oar_job_id'],resources=False, username = login ) + if sl.oar_job_id is not -1: + rslt = self.GetJobs( sl.oar_job_id,resources=False, username = login ) print >>sys.stderr, " \r\n \r\n \tSLABDRIVER.PY GetSlices GetJobs %s " %(rslt) if rslt : sl.update(rslt) @@ -544,7 +572,7 @@ class SlabDriver(Driver): else : sl['oar_job_id'] = '-1' sl.update({'hrn':str(sl['slice_hrn'])}) - self.db.update_senslab_slice(sl) + #self.db.update_senslab_slice(sl) print >>sys.stderr, " \r\n \r\n \tSLABDRIVER.PY GetSlices return_slice_list %s" %(return_slice_list) return return_slice_list @@ -581,57 +609,58 @@ class SlabDriver(Driver): # @param type type of record (user, slice, ...) # @param hrn human readable name # @param sfa_fields dictionary of SFA fields - # @param pl_fields dictionary of PLC fields (output) + # @param slab_fields dictionary of PLC fields (output) - def sfa_fields_to_pl_fields(self, type, hrn, record): + def sfa_fields_to_slab_fields(self, type, hrn, record): def convert_ints(tmpdict, int_fields): for field in int_fields: if field in tmpdict: tmpdict[field] = int(tmpdict[field]) - pl_record = {} + slab_record = {} #for field in record: - # pl_record[field] = record[field] + # slab_record[field] = record[field] if type == "slice": #instantion used in get_slivers ? - if not "instantiation" in pl_record: - pl_record["instantiation"] = "senslab-instantiated" - pl_record["hrn"] = hrn_to_pl_slicename(hrn) + if not "instantiation" in slab_record: + slab_record["instantiation"] = "senslab-instantiated" + slab_record["hrn"] = hrn_to_pl_slicename(hrn) + print >>sys.stderr, "\r\n \r\n \t SLABDRIVER.PY sfa_fields_to_slab_fields slab_record %s hrn_to_pl_slicename(hrn) hrn %s " %(slab_record['hrn'], hrn) if "url" in record: - pl_record["url"] = record["url"] + slab_record["url"] = record["url"] if "description" in record: - pl_record["description"] = record["description"] + slab_record["description"] = record["description"] if "expires" in record: - pl_record["expires"] = int(record["expires"]) + slab_record["expires"] = int(record["expires"]) #nodes added by OAR only and then imported to SFA #elif type == "node": - #if not "hostname" in pl_record: + #if not "hostname" in slab_record: #if not "hostname" in record: #raise MissingSfaInfo("hostname") - #pl_record["hostname"] = record["hostname"] - #if not "model" in pl_record: - #pl_record["model"] = "geni" + #slab_record["hostname"] = record["hostname"] + #if not "model" in slab_record: + #slab_record["model"] = "geni" #One authority only #elif type == "authority": - #pl_record["login_base"] = hrn_to_pl_login_base(hrn) + #slab_record["login_base"] = hrn_to_slab_login_base(hrn) - #if not "name" in pl_record: - #pl_record["name"] = hrn + #if not "name" in slab_record: + #slab_record["name"] = hrn - #if not "abbreviated_name" in pl_record: - #pl_record["abbreviated_name"] = hrn + #if not "abbreviated_name" in slab_record: + #slab_record["abbreviated_name"] = hrn - #if not "enabled" in pl_record: - #pl_record["enabled"] = True + #if not "enabled" in slab_record: + #slab_record["enabled"] = True - #if not "is_public" in pl_record: - #pl_record["is_public"] = True + #if not "is_public" in slab_record: + #slab_record["is_public"] = True - return pl_record + return slab_record @@ -752,9 +781,21 @@ class SlabDriver(Driver): # we obtain # get the sfa records - table = SfaTable() + #table = SfaTable() + existing_records = {} + all_records = dbsession.query(RegRecord).all + for record in all_records: + existing_records[(record.type,record.pointer)] = record + + print >>sys.stderr, " \r\r\n SLABDRIVER fill_record_sfa_info existing_records %s " %(existing_records) person_list, persons = [], {} - person_list = table.find({'type': 'user', 'pointer': person_ids}) + #person_list = table.find({'type': 'user', 'pointer': person_ids}) + try: + for p_id in person_ids: + person_list.append( existing_records.get(('user',p_id))) + except KeyError: + print >>sys.stderr, " \r\r\n SLABDRIVER fill_record_sfa_info ERRRRRRRRRROR" + # create a hrns keyed on the sfa record's pointer. # Its possible for multiple records to have the same pointer so # the dict's value will be a list of hrns. @@ -763,10 +804,10 @@ class SlabDriver(Driver): persons[person['pointer']].append(person) # get the pl records - pl_person_list, pl_persons = [], {} - pl_person_list = self.GetPersons(person_ids, ['person_id', 'roles']) - pl_persons = list_to_dict(pl_person_list, 'person_id') - #print>>sys.stderr, "\r\n \r\n _fill_record_sfa_info ___ _list %s \r\n \t\t SenslabUsers.GetPersons ['person_id', 'roles'] pl_persons %s \r\n records %s" %(pl_person_list, pl_persons,records) + slab_person_list, slab_persons = [], {} + slab_person_list = self.GetPersons(person_ids, ['person_id', 'roles']) + slab_persons = list_to_dict(slab_person_list, 'person_id') + #print>>sys.stderr, "\r\n \r\n _fill_record_sfa_info ___ _list %s \r\n \t\t SenslabUsers.GetPersons ['person_id', 'roles'] slab_persons %s \r\n records %s" %(slab_person_list, slab_persons,records) # fill sfa info for record in records: @@ -788,8 +829,8 @@ class SlabDriver(Driver): record['researcher'].extend(hrns) # pis at the slice's site - pl_pis = site_pis[record['site_id']] - pi_ids = [pi['person_id'] for pi in pl_pis] + slab_pis = site_pis[record['site_id']] + pi_ids = [pi['person_id'] for pi in slab_pis] for person_id in pi_ids: hrns = [person['hrn'] for person in persons[person_id]] record['PI'].extend(hrns) @@ -801,11 +842,11 @@ class SlabDriver(Driver): record['operator'] = [] record['owner'] = [] for pointer in record['person_ids']: - if pointer not in persons or pointer not in pl_persons: + if pointer not in persons or pointer not in slab_persons: # this means there is not sfa or pl record for this user continue hrns = [person['hrn'] for person in persons[pointer]] - roles = pl_persons[pointer]['roles'] + roles = slab_persons[pointer]['roles'] if 'pi' in roles: record['PI'].extend(hrns) if 'tech' in roles: @@ -837,19 +878,26 @@ class SlabDriver(Driver): print >>sys.stderr, "\r\n \t\t BEFORE fill_record_info %s" %(records) if not isinstance(records, list): records = [records] - #print >>sys.stderr, "\r\n \t\t BEFORE fill_record_pl_info %s" %(records) + parkour = records try: for record in parkour: if str(record['type']) == 'slice': print >>sys.stderr, "\r\n \t\t SLABDRIVER.PY fill_record_info record %s" %(record) - sfatable = SfaTable() + #sfatable = SfaTable() + + existing_records_by_id = {} + all_records = dbsession.query(RegRecord).all + for rec in all_records: + existing_records_by_id[rec.record_id] = rec + print >>sys.stderr, "\r\n \t\t SLABDRIVER.PY fill_record_info existing_records_by_id %s" %(existing_records_by_id) + recslice = self.db.find('slice',str(record['hrn'])) if isinstance(recslice,list) and len(recslice) == 1: recslice = recslice[0] - recuser = sfatable.find( recslice['record_id_user'], ['hrn']) - + #recuser = sfatable.find( recslice['record_id_user'], ['hrn']) + recuser = existing_records_by_id[recslice['record_id_user']]['hrn'] print >>sys.stderr, "\r\n \t\t SLABDRIVER.PY fill_record_info %s" %(recuser) if isinstance(recuser,list) and len(recuser) == 1: @@ -874,8 +922,8 @@ class SlabDriver(Driver): print >>sys.stderr, "\r\n \t\t SLABDRIVER fill_record_info EXCEPTION RECORDS : %s" %(records) return - #self.fill_record_pl_info(records) - ##print >>sys.stderr, "\r\n \t\t after fill_record_pl_info %s" %(records) + #self.fill_record_slab_info(records) + ##print >>sys.stderr, "\r\n \t\t after fill_record_slab_info %s" %(records) #self.fill_record_sfa_info(records) #print >>sys.stderr, "\r\n \t\t after fill_record_sfa_info" diff --git a/sfa/senslab/slabpostgres.py b/sfa/senslab/slabpostgres.py index b589f2bf..a54ff45e 100644 --- a/sfa/senslab/slabpostgres.py +++ b/sfa/senslab/slabpostgres.py @@ -1,24 +1,26 @@ -import psycopg2 -import psycopg2.extensions -psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) -# UNICODEARRAY not exported yet -psycopg2.extensions.register_type(psycopg2._psycopg.UNICODEARRAY) +#import psycopg2 +#import psycopg2.extensions +#psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) +## UNICODEARRAY not exported yet +#psycopg2.extensions.register_type(psycopg2._psycopg.UNICODEARRAY) from sfa.util.config import Config -from sfa.storage.table import SfaTable +#from sfa.storage.table import SfaTable from sfa.util.sfalogging import logger # allow to run sfa2wsdl if this is missing (for mac) import sys -try: import pgdb -except: print >> sys.stderr, "WARNING, could not import pgdb" - +#try: import pgdb +#except: print >> sys.stderr, "WARNING, could not import pgdb" +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker from sqlalchemy import Column, Integer, String, DateTime from sqlalchemy import Table, Column, MetaData, join, ForeignKey -from sfa.storage.model import Base +import sfa.storage.model as model + from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship, backref -from sfa.storage.alchemy import dbsession, engine + from sqlalchemy import MetaData, Table from sqlalchemy.exc import NoSuchTableError @@ -33,14 +35,14 @@ tablenames_dict = {'slice_senslab': slice_table} -SlabBase = declarative_base(metadata= Base.metadata, bind=engine) +SlabBase = declarative_base() class SlabSliceDB (SlabBase): - __tablename__ = 'slice_senslab' - record_id_user = Column(Integer, ForeignKey("records.record_id"), primary_key=True) + __tablename__ = 'slice_senslab' + record_id_user = Column(Integer, primary_key=True) oar_job_id = Column( Integer,default = -1) record_id_slice = Column(Integer) slice_hrn = Column(String,nullable = False) @@ -57,19 +59,59 @@ class SlabSliceDB (SlabBase): if record_id_user: self.record_id_user= record_id_user + def __repr__(self): + result=">sys.stderr, " \r\n \r\n SLAPOSTGRES INIT unix_url %s" %(unix_url) + # the TCP fallback method + tcp_url = "postgresql+psycopg2://%s:%s@%s:%s/%s"%\ + (config.SFA_DB_USER,config.SFA_DB_PASSWORD,config.SFA_DB_HOST,config.SFA_DB_PORT,dbname) + for url in [ unix_url, tcp_url ] : + try: + self.slab_engine = create_engine (url,echo_pool=True,echo=True) + self.check() + self.url=url + return + except: + pass + self.slab_engine=None + raise Exception,"Could not connect to database" + + def check (self): + self.slab_engine.execute ("select 1").scalar() + + + def session (self): + if self.slab_session is None: + Session=sessionmaker () + self.slab_session=Session(bind=self.slab_engine) + return self.slab_session - def init_create_query(self): - sfatable = SfaTable() - slice_table['record_id_user'] = slice_table['record_id_user'].replace("X",sfatable.tablename) - print sys.stderr, " \r\n \r\n slice_table %s ",slice_table def cursor(self): if self.connection is None: @@ -141,8 +183,9 @@ class SlabDB: try: - metadata = MetaData (bind=engine) + metadata = MetaData (bind=self.slab_engine) table=Table (tablename, metadata, autoload=True) + return True except NoSuchTableError: print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES EXISTS NOPE! tablename %s " %(tablename) @@ -167,47 +210,13 @@ class SlabDB: the table schema. """ - print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES CREATETABLE " - SlabBase.metadata.create_all(engine) - print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES CREATETABLE YAAAAAAAAAAY" - #mark = self.cursor() - #tablelist =[] - #if tablename not in tablenames_dict: - #logger.error("Tablename unknown - creation failed") - #return - - #T = tablenames_dict[tablename] - #print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES.PY T %s" %(T) - #for k in T.keys(): - #tmp = str(k) +' ' + T[k] - #tablelist.append(tmp) - - #end_of_statement = ",".join(tablelist) - - #statement = "CREATE TABLE " + tablename + " ("+ end_of_statement +");" - #print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES.PY statement %s" %(statement) - ##template = "CREATE INDEX %s_%s_idx ON %s (%s);" - ##indexes = [template % ( self.tablename, field, self.tablename, field) \ - ##for field in ['hrn', 'type', 'authority', 'peer_authority', 'pointer']] - ##IF EXISTS doenst exist in postgres < 8.2 - #try: - #print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES.PY AVANT LE DROP IF EXISTS" - #mark.execute('DROP TABLE IF EXISTS ' + tablename +';') - - #except: - #try: - #mark.execute('DROP TABLE' + tablename +';') - #except: - #pass - #print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES.PY AVANT EXECUTE statement ",statement - #mark.execute(statement) - #print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES.PY OUEEEEEEEEEEEEEEEEEEEEEE " - ##for index in indexes: - ##self.db.do(index) - #self.connection.commit() - #print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES.PY COMMIT DE OUFGUEDIN " - #mark.close() - #self.close() + #metadata = MetaData (bind=engine) + #table=Table (tablename, metadata, autoload=True) + #records = Table ( 'records', SlabBase.metadata,autoload=True ) + #records = Table ( 'records', SlabBase.metadata,Column ('record_id', Integer, primary_key=True), ) + print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES createtable SlabBase.metadata.sorted_tables %s \r\n engine %s" %(SlabBase.metadata.sorted_tables , slab_engine) + #table.create(bind =engine) + SlabBase.metadata.create_all(slab_engine) return @@ -229,33 +238,33 @@ class SlabDB: self.close() return - def insert_slab_slice(self, person_rec): - """ - Inserts information about a user and his slice into the slice table. + #def insert_slab_slice(self, person_rec): + #""" + #Inserts information about a user and his slice into the slice table. - """ - sfatable = SfaTable() - keys = slice_table.keys() + #""" + #sfatable = SfaTable() + #keys = slice_table.keys() - #returns a list of records from the sfa table (dicts) - #the filters specified will return only one matching record, into a list of dicts - #Finds the slice associated with the user (Senslabs slices hrns contains the user hrn) - - userrecord = sfatable.find({'hrn': person_rec['hrn'], 'type':'user'}) - slicerec = sfatable.find({'hrn': person_rec['hrn']+'_slice', 'type':'slice'}) - if slicerec : - if (isinstance (userrecord, list)): - userrecord = userrecord[0] - if (isinstance (slicerec, list)): - slicerec = slicerec[0] + ##returns a list of records from the sfa table (dicts) + ##the filters specified will return only one matching record, into a list of dicts + ##Finds the slice associated with the user (Senslabs slices hrns contains the user hrn) + + #userrecord = sfatable.find({'hrn': person_rec['hrn'], 'type':'user'}) + #slicerec = sfatable.find({'hrn': person_rec['hrn']+'_slice', 'type':'slice'}) + #if slicerec : + #if (isinstance (userrecord, list)): + #userrecord = userrecord[0] + #if (isinstance (slicerec, list)): + #slicerec = slicerec[0] - oar_dflt_jobid = -1 - values = [ str(oar_dflt_jobid), ' \''+ str(slicerec['hrn']) + '\'', str(userrecord['record_id']), str( slicerec['record_id'])] + #oar_dflt_jobid = -1 + #values = [ str(oar_dflt_jobid), ' \''+ str(slicerec['hrn']) + '\'', str(userrecord['record_id']), str( slicerec['record_id'])] - self.insert('slice_senslab', keys, values) - else : - logger.error("Trying to import a not senslab slice") - return + #self.insert('slice_senslab', keys, values) + #else : + #logger.error("Trying to import a not senslab slice") + #return def update(self, table, column_names, values, whereclause, valueclause): @@ -288,18 +297,18 @@ class SlabDB: return - def update_senslab_slice(self, slice_rec): - sfatable = SfaTable() - hrn = str(slice_rec['hrn']) - userhrn = hrn.rstrip('_slice') - userrecord = sfatable.find({'hrn': userhrn, 'type':'user'}) - print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES.PY update_senslab_slice : userrecord %s slice_rec %s userhrn %s" %( userrecord, slice_rec, userhrn) - if (isinstance (userrecord, list)): - userrecord = userrecord[0] - columns = [ 'record_id_user', 'oar_job_id'] - values = [slice_rec['record_id_user'],slice_rec['oar_job_id']] - self.update('slice',columns, values,'record_id_slice', slice_rec['record_id_slice']) - return + #def update_senslab_slice(self, slice_rec): + #sfatable = SfaTable() + #hrn = str(slice_rec['hrn']) + #userhrn = hrn.rstrip('_slice') + #userrecord = sfatable.find({'hrn': userhrn, 'type':'user'}) + #print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES.PY update_senslab_slice : userrecord %s slice_rec %s userhrn %s" %( userrecord, slice_rec, userhrn) + #if (isinstance (userrecord, list)): + #userrecord = userrecord[0] + #columns = [ 'record_id_user', 'oar_job_id'] + #values = [slice_rec['record_id_user'],slice_rec['oar_job_id']] + #self.update('slice',columns, values,'record_id_slice', slice_rec['record_id_slice']) + #return def find(self, tablename,record_filter = None, columns=None): @@ -343,3 +352,10 @@ class SlabDB: results = [results] return results + + +from sfa.util.config import Config + +slab_alchemy= SlabDB(Config()) +slab_engine=slab_alchemy.slab_engine +slab_dbsession=slab_alchemy.session() \ No newline at end of file diff --git a/sfa/senslab/table_slab.py b/sfa/senslab/table_slab.py deleted file mode 100644 index 9ace4140..00000000 --- a/sfa/senslab/table_slab.py +++ /dev/null @@ -1,182 +0,0 @@ -# -# implements support for SFA records stored in db tables -# -# TODO: Use existing PLC database methods? or keep this separate? - - -from sfa.trust.gid import * -from sfa.util.record import * -from sfa.util.config import * -from sfa.util.filter import * -from sfa.trust.hierarchy import * -from sfa.trust.certificate import * -from sfa.trust.auth import * -from sfa.senslab.OARrestapi import * -from sfa.senslab.LDAPapi import * - -class SfaTable(list): - authname="" - def __init__(self, record_filter = None): - self.oar = OARapi() - self.ldap = LDAPapi() - self.senslabauth=Hierarchy() - config=Config() - self.authname=config.SFA_REGISTRY_ROOT_AUTH - authinfo=self.senslabauth.get_auth_info(self.authname) - - self.auth=Auth() - gid=authinfo.get_gid_object() - - def exists(self): - return True - - def db_fields(self, obj=None): - return dict( [ ] ) - - @staticmethod - def is_writable (key,value,dict): - # if not mentioned, assume it's writable (e.g. deleted ...) - if key not in dict: return True - # if mentioned but not linked to a Parameter object, idem - if not isinstance(dict[key], Parameter): return True - # if not marked ro, it's writable - if not dict[key].ro: return True - - return False - - - def create(self): - return True - - def remove(self, record): - return 0 - - def insert(self, record): - return 0 - - def update(self, record): - return 0 - - def quote_string(self, value): - return str(self.db.quote(value)) - - def quote(self, value): - return self.db.quote(value) - - - def oarFind(self, record_filter = None, columns=None): - results=[] - node_ids=[] - - if 'authority' in record_filter: - # ask for authority - if record_filter['authority']== self.authname : - # which is senslab - print>> sys.stderr , "ET MERDE !!!!" - node_ids="" - else: - # which is NOT senslab - return [] - else : - if not 'hrn' in record_filter: - print >>sys.stderr,"find : don't know how to handle filter ",record_filter - return [] - else: - hrns=[] - h=record_filter['hrn'] - if isinstance(h,list): - hrns=h - else : - hrns.append(h) - - for hrn in hrns: - head,sep,tail=hrn.partition(".") - if head != self.authname : - print >>sys.stderr,"i know nothing about",hrn, " my authname is ", self.authname, " not ", splited_hrn[0] - else : - node_ids.append(tail) - - node_list = self.oar.GetNodes( node_ids) - - for node in node_list: - hrn=self.authname+"."+node['hostname'] - results.append( { - 'type': 'node', -# 'email': ldapentry[1]['mail'][0], -# 'first_name': ldapentry[1]['givenName'][0], -# 'last_name': ldapentry[1]['sn'][0], -# 'phone': 'none', -# 'gid': gid.save_to_string(), -# 'serial': 'none', - 'authority': self.authname, - 'peer_authority': '', - 'pointer' : '', - 'hrn': hrn, - 'date_created' : 'none', - 'last_updated': 'none' - } ) - - return results - - def find(self, record_filter = None, columns=None): - # senslab stores its users in an ldap dictionnary - # and nodes in a oar scheduller database - # both should be interrogated. - print >>sys.stderr,"find : ",record_filter - if not isinstance(record_filter,dict): - print >>sys.stderr,"find : record_filter is not a dict" - print >>sys.stderr,record_filter.__class__ - return [] - allResults=[] - if 'type' in record_filter: - if record_filter['type'] == 'slice': - print >>sys.stderr,"find : don't know how to handle slices yet" - return [] - if record_filter['type'] == 'authority': - if 'hrn' in record_filter and record_filter['hrn']==self.authname: - return [] - else: - print >>sys.stderr,"find which authority ?" - return [] - if record_filter['type'] == 'user': - return self.ldap.ldapFind(record_filter, columns) - if record_filter['type'] == 'node': - return self.ldap.ldapFind(record_filter, columns) - else: - print >>sys.stderr,"unknown type to find : ", record_filter['type'] - return [] - else: - allResults = self.ldap.ldapFind(record_filter, columns) - allResults+= self.oarFind(record_filter, columns) - - return allResults - - def findObjects(self, record_filter = None, columns=None): - - print >>sys.stderr,"find : ",record_filter - # print record_filter['type'] - # if record_filter['type'] in ['authority']: - # print "findObjectAuthority" - results = self.find(record_filter, columns) - result_rec_list = [] - for result in results: - if result['type'] in ['authority']: - result_rec_list.append(AuthorityRecord(dict=result)) - elif result['type'] in ['node']: - result_rec_list.append(NodeRecord(dict=result)) - elif result['type'] in ['slice']: - result_rec_list.append(SliceRecord(dict=result)) - elif result['type'] in ['user']: - result_rec_list.append(UserRecord(dict=result)) - else: - result_rec_list.append(SfaRecord(dict=result)) - - return result_rec_list - - - def drop(self): - return 0 - - def sfa_records_purge(self): - return 0 - -- 2.43.0