From 81386c70035f6a184696bb5d07e01fb42fb3af0d Mon Sep 17 00:00:00 2001 From: Sandrine Avakian Date: Wed, 14 Mar 2012 10:36:20 +0100 Subject: [PATCH] Corrected import site - corrected GetSites - corrected GetNodes Federation still notworking. --- sfa/importer/slabimporter.py | 18 ++- sfa/senslab/OARrestapi.py | 42 ++++--- sfa/senslab/slabaggregate.py | 4 +- sfa/senslab/slabdriver.py | 2 +- sfa/senslab/slabpostgres.py | 234 +++-------------------------------- 5 files changed, 56 insertions(+), 244 deletions(-) diff --git a/sfa/importer/slabimporter.py b/sfa/importer/slabimporter.py index f4468855..166a7c21 100644 --- a/sfa/importer/slabimporter.py +++ b/sfa/importer/slabimporter.py @@ -21,7 +21,7 @@ from sfa.storage.dbschema import DBSchema def _get_site_hrn(site): - hrn = site['login_base'] + hrn = site['name'] return hrn class SlabImporter: @@ -131,7 +131,7 @@ class SlabImporter: self.logger.warning ("SlabImporter: cannot find node_id %s - ignored"%node_id) continue site_auth = get_authority(site_hrn) - site_name = site['login_base'] + site_name = site['name'] hrn = self.hostname_to_hrn(slabdriver.root_auth, site_name, node['hostname']) # xxx this sounds suspicious if len(hrn) > 64: hrn = hrn[:64] @@ -226,15 +226,11 @@ class SlabImporter: # if user's primary key has changed then we need to update the # users gid by forcing an update here sfa_keys = user_record.reg_keys - #def key_in_list (key,sfa_keys): - #for reg_key in sfa_keys: - #if reg_key.key==key['key']: return True - #return False - # is there a new key in myplc ? - new_keys=False + + new_key=False if slab_key is not sfa_keys : - new_keys = True - if new_keys: + new_key = True + if new_key: (pubkey,pkey) = init_person_key (person, slab_key) person_gid = self.auth_hierarchy.create_gid(person_urn, create_uuid(), pkey) if not pubkey: @@ -279,6 +275,8 @@ class SlabImporter: self.update_just_added_records_dict ( slice_record ) except: self.logger.log_exc("SlabImporter: failed to import slice") + + #No slice update upon import in senslab else: # xxx update the record ... self.logger.warning ("Slice update not yet implemented") diff --git a/sfa/senslab/OARrestapi.py b/sfa/senslab/OARrestapi.py index 4d32d8d0..c0d4df65 100644 --- a/sfa/senslab/OARrestapi.py +++ b/sfa/senslab/OARrestapi.py @@ -12,6 +12,8 @@ from sfa.util.config import Config from sfa.util.plxrn import PlXrn from sfa.util.xrn import hrn_to_urn, get_authority,Xrn,get_leaf +from sfa.util.config import Config + #OARIP='10.127.255.254' OARIP='192.168.0.109' @@ -179,7 +181,8 @@ class OARGETParser: def AddNodeSite(self,tuplelist,value): - tuplelist.append(('site_login_base',str(value))) + tuplelist.append(('site',str(value))) + def AddNodeRadio(self,tuplelist,value): tuplelist.append(('radio',str(value))) @@ -330,37 +333,48 @@ class OARGETParser: #Retourne liste de dictionnaires contenant attributs des sites def ParseSites(self): nodes_per_site = {} - + config = Config() # Create a list of nodes per site_id for node_id in self.node_dictlist.keys(): node = self.node_dictlist[node_id] - - if node['site_login_base'] not in nodes_per_site.keys(): - nodes_per_site[node['site_login_base']] = [] - nodes_per_site[node['site_login_base']].append(node['node_id']) + if node['site'] not in nodes_per_site: + nodes_per_site[node['site']] = [] + nodes_per_site[node['site']].append(node['node_id']) else: - if node['node_id'] not in nodes_per_site[node['site_login_base']]: - nodes_per_site[node['site_login_base']].append(node['node_id']) + if node['node_id'] not in nodes_per_site[node['site']]: + nodes_per_site[node['site']].append(node['node_id']) + #Create a site dictionary with key is site_login_base (name of the site) # and value is a dictionary of properties, including the list of the node_ids for node_id in self.node_dictlist.keys(): node = self.node_dictlist[node_id] - node.update({'hrn':self.hostname_to_hrn(self.interface_hrn, node['site_login_base'],node['hostname'])}) + node.update({'hrn':self.hostname_to_hrn(self.interface_hrn, node['site'],node['hostname'])}) #node['hrn'] = self.hostname_to_hrn(self.interface_hrn, node['site_login_base'],node['hostname']) self.node_dictlist.update({node_id:node}) #if node_id is 1: #print>>sys.stderr, " \r\n \r\n \t \t\t\t OARESTAPI Parse Sites self.node_dictlist %s " %(self.node_dictlist) - if node['site_login_base'] not in self.site_dict.keys(): - self.site_dict[node['site_login_base']] = {'login_base':node['site_login_base'], - 'node_ids':nodes_per_site[node['site_login_base']], + if node['site'] not in self.site_dict: + self.site_dict[node['site']] = {'site':node['site'], + 'node_ids':nodes_per_site[node['site']], 'latitude':"48.83726", - 'longitude':"- 2.10336",'name':"senslab", + 'longitude':"- 2.10336",'name':config.SFA_REGISTRY_ROOT_AUTH, 'pcu_ids':[], 'max_slices':None, 'ext_consortium_id':None, 'max_slivers':None, 'is_public':True, 'peer_site_id': None, 'abbreviated_name':"senslab", 'address_ids': [], 'url':"http,//www.senslab.info", 'person_ids':[], 'site_tag_ids':[], 'enabled': True, 'slice_ids':[], - 'date_created': None, 'peer_id': None } + 'date_created': None, 'peer_id': None } + #if node['site_login_base'] not in self.site_dict.keys(): + #self.site_dict[node['site_login_base']] = {'login_base':node['site_login_base'], + #'node_ids':nodes_per_site[node['site_login_base']], + #'latitude':"48.83726", + #'longitude':"- 2.10336",'name':"senslab", + #'pcu_ids':[], 'max_slices':None, 'ext_consortium_id':None, + #'max_slivers':None, 'is_public':True, 'peer_site_id': None, + #'abbreviated_name':"senslab", 'address_ids': [], + #'url':"http,//www.senslab.info", 'person_ids':[], + #'site_tag_ids':[], 'enabled': True, 'slice_ids':[], + #'date_created': None, 'peer_id': None } diff --git a/sfa/senslab/slabaggregate.py b/sfa/senslab/slabaggregate.py index 959fc41f..861ac5eb 100644 --- a/sfa/senslab/slabaggregate.py +++ b/sfa/senslab/slabaggregate.py @@ -198,11 +198,11 @@ class SlabAggregate: # xxx how to retrieve site['login_base'] #site_id=node['site_id'] #site=sites_dict[site_id] - rspec_node['component_id'] = hostname_to_urn(self.driver.root_auth, node['site_login_base'], node['hostname']) + rspec_node['component_id'] = hostname_to_urn(self.driver.root_auth, node['site'], node['hostname']) rspec_node['component_name'] = node['hostname'] rspec_node['component_manager_id'] = hrn_to_urn(self.driver.root_auth, 'authority+sa') #rspec_node['component_manager_id'] = Xrn(self.driver.root_auth, 'authority+sa').get_urn() - rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.root_auth, node['site_login_base']), 'authority+sa') + rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.root_auth, node['site']), 'authority+sa') # do not include boot state ( element) in the manifest rspec if not slice: rspec_node['boot_state'] = node['boot_state'] diff --git a/sfa/senslab/slabdriver.py b/sfa/senslab/slabdriver.py index 894dada3..337c5de8 100644 --- a/sfa/senslab/slabdriver.py +++ b/sfa/senslab/slabdriver.py @@ -917,7 +917,7 @@ class SlabDriver(Driver): records.append(rec) print >>sys.stderr, "\r\n \t\t SLABDRIVER.PY fill_record_info ADDING SLIC EINFO recslice %s" %(recslice) - + print >>sys.stderr, "\r\n \t\t SLABDRIVER.PY fill_record_info OKrecords %s" %(records) except TypeError: print >>sys.stderr, "\r\n \t\t SLABDRIVER fill_record_info EXCEPTION RECORDS : %s" %(records) return diff --git a/sfa/senslab/slabpostgres.py b/sfa/senslab/slabpostgres.py index a54ff45e..fc561cd8 100644 --- a/sfa/senslab/slabpostgres.py +++ b/sfa/senslab/slabpostgres.py @@ -1,18 +1,11 @@ -#import psycopg2 -#import psycopg2.extensions -#psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) -## UNICODEARRAY not exported yet -#psycopg2.extensions.register_type(psycopg2._psycopg.UNICODEARRAY) -from sfa.util.config import Config -#from sfa.storage.table import SfaTable -from sfa.util.sfalogging import logger -# allow to run sfa2wsdl if this is missing (for mac) import sys -#try: import pgdb -#except: print >> sys.stderr, "WARNING, could not import pgdb" + from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +from sfa.util.config import Config +from sfa.util.sfalogging import logger + from sqlalchemy import Column, Integer, String, DateTime from sqlalchemy import Table, Column, MetaData, join, ForeignKey import sfa.storage.model as model @@ -70,11 +63,8 @@ class SlabSliceDB (SlabBase): class SlabDB: def __init__(self,config): - #self.config = Config() - #self.connection = None self.sl_base = SlabBase - #self.init_create_query() - + dbname="slab_sfa" # will be created lazily on-demand self.slab_session = None @@ -113,33 +103,7 @@ class SlabDB: return self.slab_session - def cursor(self): - if self.connection is None: - # (Re)initialize database connection - if psycopg2: - try: - # Try UNIX socket first - self.connection = psycopg2.connect(user = 'sfa', - password = 'sfa', - database = 'sfa') - #self.connection = psycopg2.connect(user = self.config.SFA_PLC_DB_USER, - #password = self.config.SFA_PLC_DB_PASSWORD, - #database = self.config.SFA_PLC_DB_NAME) - except psycopg2.OperationalError: - # Fall back on TCP - self.connection = psycopg2.connect(user = self.config.SFA_PLC_DB_USER, - password = self.config.SFA_PLC_DB_PASSWORD, - database = self.config.SFA_PLC_DB_NAME, - host = self.config.SFA_PLC_DB_HOST, - port = self.config.SFA_PLC_DB_PORT) - self.connection.set_client_encoding("UNICODE") - else: - self.connection = pgdb.connect(user = self.config.SFA_PLC_DB_USER, - password = self.config.SFA_PLC_DB_PASSWORD, - host = "%s:%d" % (self.config.SFA_PLC_DB_HOST, self.config.SFA_PLC_DB_PORT), - database = self.config.SFA_PLC_DB_NAME) - - return self.connection.cursor() + #Close connection to database def close(self): @@ -147,32 +111,7 @@ class SlabDB: self.connection.close() self.connection = None - def selectall(self, query, hashref = True, key_field = None): - """ - Return each row as a dictionary keyed on field name (like DBI - selectrow_hashref()). If key_field is specified, return rows - as a dictionary keyed on the specified field (like DBI - selectall_hashref()). - - """ - cursor = self.cursor() - cursor.execute(query) - rows = cursor.fetchall() - cursor.close() - self.connection.commit() - - if hashref or key_field is not None: - # Return each row as a dictionary keyed on field name - # (like DBI selectrow_hashref()). - labels = [column[0] for column in cursor.description] - rows = [dict(zip(labels, row)) for row in rows] - - if key_field is not None and key_field in labels: - # Return rows as a dictionary keyed on the specified field - # (like DBI selectall_hashref()). - return dict([(row[key_field], row) for row in rows]) - else: - return rows + def exists(self, tablename): @@ -180,7 +119,6 @@ class SlabDB: Checks if the table specified as tablename exists. """ - try: metadata = MetaData (bind=self.slab_engine) @@ -190,19 +128,7 @@ class SlabDB: except NoSuchTableError: print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES EXISTS NOPE! tablename %s " %(tablename) return False - ##mark = self.cursor() - #sql = "SELECT * from pg_tables" - ##mark.execute(sql) - ##rows = mark.fetchall() - ##mark.close() - ##labels = [column[0] for column in mark.description] - ##rows = [dict(zip(labels, row)) for row in rows] - #rows = self.selectall(sql) - #rows = filter(lambda row: row['tablename'].startswith(tablename), rows) - #if rows: - #return True - #print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES EXISTS NOPE ! tablename %s " %(tablename) - #return False + def createtable(self, tablename ): """ @@ -210,147 +136,21 @@ class SlabDB: the table schema. """ - #metadata = MetaData (bind=engine) - #table=Table (tablename, metadata, autoload=True) - #records = Table ( 'records', SlabBase.metadata,autoload=True ) - #records = Table ( 'records', SlabBase.metadata,Column ('record_id', Integer, primary_key=True), ) + print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES createtable SlabBase.metadata.sorted_tables %s \r\n engine %s" %(SlabBase.metadata.sorted_tables , slab_engine) - #table.create(bind =engine) SlabBase.metadata.create_all(slab_engine) return - - - def insert(self, table, columns,values): - """ - Inserts data (values) into the columns of the specified table. - - """ - mark = self.cursor() - statement = "INSERT INTO " + table + \ - "(" + ",".join(columns) + ") " + \ - "VALUES(" + ", ".join(values) + ");" - - mark.execute(statement) - self.connection.commit() - mark.close() - self.close() - return - - #def insert_slab_slice(self, person_rec): - #""" - #Inserts information about a user and his slice into the slice table. - - #""" - #sfatable = SfaTable() - #keys = slice_table.keys() - - ##returns a list of records from the sfa table (dicts) - ##the filters specified will return only one matching record, into a list of dicts - ##Finds the slice associated with the user (Senslabs slices hrns contains the user hrn) - - #userrecord = sfatable.find({'hrn': person_rec['hrn'], 'type':'user'}) - #slicerec = sfatable.find({'hrn': person_rec['hrn']+'_slice', 'type':'slice'}) - #if slicerec : - #if (isinstance (userrecord, list)): - #userrecord = userrecord[0] - #if (isinstance (slicerec, list)): - #slicerec = slicerec[0] - - #oar_dflt_jobid = -1 - #values = [ str(oar_dflt_jobid), ' \''+ str(slicerec['hrn']) + '\'', str(userrecord['record_id']), str( slicerec['record_id'])] - - #self.insert('slice_senslab', keys, values) - #else : - #logger.error("Trying to import a not senslab slice") - #return - - - def update(self, table, column_names, values, whereclause, valueclause): - """ - Updates a record in a given table. - - """ - #Creates the values string for the update SQL command - vclause = valueclause - if len(column_names) is not len(values): - return - else: - valueslist = [] - valuesdict = dict(zip(column_names,values)) - for k in valuesdict.keys(): - valuesdict[k] = str(valuesdict[k]) - #v = ' \''+ str(k) + '\''+ '='+' \''+ valuesdict[k]+'\'' - v = str(k) + '=' + valuesdict[k] - valueslist.append(v) - if isinstance(vclause,str): - vclause = '\''+ vclause + '\'' - statement = "UPDATE %s SET %s WHERE %s = %s" % \ - (table, ", ".join(valueslist), whereclause, vclause) - print>>sys.stderr,"\r\n \r\n SLABPOSTGRES.PY update statement %s valuesdict %s valueslist %s" %(statement,valuesdict,valueslist) - mark = self.cursor() - mark.execute(statement) - self.connection.commit() - mark.close() - self.close() - - return - - #def update_senslab_slice(self, slice_rec): - #sfatable = SfaTable() - #hrn = str(slice_rec['hrn']) - #userhrn = hrn.rstrip('_slice') - #userrecord = sfatable.find({'hrn': userhrn, 'type':'user'}) - #print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES.PY update_senslab_slice : userrecord %s slice_rec %s userhrn %s" %( userrecord, slice_rec, userhrn) - #if (isinstance (userrecord, list)): - #userrecord = userrecord[0] - #columns = [ 'record_id_user', 'oar_job_id'] - #values = [slice_rec['record_id_user'],slice_rec['oar_job_id']] - #self.update('slice',columns, values,'record_id_slice', slice_rec['record_id_slice']) - #return - - - def find(self, tablename,record_filter = None, columns=None): - print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES.PY find : record_filter %s %s columns %s %s" %( record_filter , type(record_filter),columns , type(columns)) - if not columns: - columns = "*" - else: - columns = ",".join(columns) - sql = "SELECT %s FROM %s WHERE True " % (columns, tablename) + def find (self, name = None, filter_dict = None): + if filter_dict: + filter_statement = "and_(SlabSliceDB." + for k in filter_dict: + filter_statement += str(k)+ "==" + str(filter_dict[l]) + filter_statement +=')' + print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES find filter_statement %s"%(filter_statement) + slab_dbsession.query(SlabSliceDB).filter(filter_statement) - #if isinstance(record_filter, (list, tuple, set)): - #ints = filter(lambda x: isinstance(x, (int, long)), record_filter) - #strs = filter(lambda x: isinstance(x, StringTypes), record_filter) - #record_filter = Filter(SfaRecord.all_fields, {'record_id': ints, 'hrn': strs}) - #sql += "AND (%s) %s " % record_filter.sql("OR") - #elif isinstance(record_filter, dict): - #record_filter = Filter(SfaRecord.all_fields, record_filter) - #sql += " AND (%s) %s" % record_filter.sql("AND") - #elif isinstance(record_filter, StringTypes): - #record_filter = Filter(SfaRecord.all_fields, {'hrn':[record_filter]}) - #sql += " AND (%s) %s" % record_filter.sql("AND") - #elif isinstance(record_filter, int): - #record_filter = Filter(SfaRecord.all_fields, {'record_id':[record_filter]}) - #sql += " AND (%s) %s" % record_filter.sql("AND") - - if isinstance(record_filter, dict): - for k in record_filter.keys(): - #sql += "AND "+' \''+ str(k) + '\''+ '='+' \''+ str(record_filter[k])+'\'' - #sql += "AND "+ str(k) + '=' + str(record_filter[k]) - sql += "AND "+ str(k) +'='+' \''+ str(record_filter[k])+'\'' - elif isinstance(record_filter, str): - sql += "AND slice_hrn ="+ ' \''+record_filter+'\'' - - #elif isinstance(record_filter, int): - #record_filter = Filter(SfaRecord.all_fields, {'record_id':[record_filter]}) - #sql += " AND (%s) %s" % record_filter.sql("AND") - sql += ";" - print>>sys.stderr, " \r\n \r\n \t SLABPOSTGRES.PY find : sql %s record_filter %s %s" %(sql, record_filter , type(record_filter)) - results = self.selectall(sql) - if isinstance(results, dict): - results = [results] - return results -- 2.45.2