From: Sandrine Avakian Date: Thu, 11 Jul 2013 12:25:04 +0000 (+0200) Subject: Changing iotlabpostgres to remove global variable iotlab_dbsession X-Git-Tag: sfa-2.1-27~51 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=00ddf8e86e3722427a806c7863f9206af0d66146;p=sfa.git Changing iotlabpostgres to remove global variable iotlab_dbsession and to put a singleton instead. --- diff --git a/sfa/iotlab/OARrestapi.py b/sfa/iotlab/OARrestapi.py index 0223f472..ba785a5c 100644 --- a/sfa/iotlab/OARrestapi.py +++ b/sfa/iotlab/OARrestapi.py @@ -13,20 +13,15 @@ from sfa.util.config import Config from sfa.util.sfalogging import logger -OAR_REQUEST_POST_URI_DICT = {'POST_job':{'uri': '/oarapi/jobs.json'}, - 'DELETE_jobs_id':{'uri':'/oarapi/jobs/id.json'}, - } - -POST_FORMAT = {'json' : {'content':"application/json", 'object':json},} - -#OARpostdatareqfields = {'resource' :"/nodes=", 'command':"sleep", \ - #'workdir':"/home/", 'walltime':""} - +class JsonPage: + """Class used to manipulate json pages given by OAR. + """ -class JsonPage: - """Class used to manipulate jsopn pages given by OAR.""" def __init__(self): + """Defines attributes to manipulate and parse the json pages. + + """ #All are boolean variables self.concatenate = False #Indicates end of data, no more pages to be loaded. @@ -38,11 +33,13 @@ class JsonPage: self.raw_json = None def FindNextPage(self): - """ Gets next data page from OAR when the query's results - are too big to be transmitted in a single page. - Uses the "links' item in the json returned to check if - an additionnal page has to be loaded. - Returns : next page , next offset query + """ + Gets next data page from OAR when the query's results + are too big to be transmitted in a single page. + Uses the "links' item in the json returned to check if + an additionnal page has to be loaded. + + :returns: next page , next offset query """ if "links" in self.raw_json: for page in self.raw_json['links']: @@ -53,7 +50,7 @@ class JsonPage: print>>sys.stderr, "\r\n \t FindNextPage NEXT LINK" return - if self.concatenate : + if self.concatenate: self.end = True self.next_page = False self.next_offset = None @@ -79,7 +76,6 @@ class JsonPage: tmp['items'].extend(page['items']) return tmp - def ResetNextPage(self): self.next_page = True self.next_offset = None @@ -88,10 +84,21 @@ class JsonPage: class OARrestapi: - def __init__(self, config_file = '/etc/sfa/oar_config.py'): - self.oarserver = {} + # classes attributes + + OAR_REQUEST_POST_URI_DICT = {'POST_job': {'uri': '/oarapi/jobs.json'}, + 'DELETE_jobs_id': \ + {'uri': '/oarapi/jobs/id.json'}, + } + POST_FORMAT = {'json': {'content': "application/json", 'object': json}} + + #OARpostdatareqfields = {'resource' :"/nodes=", 'command':"sleep", \ + #'workdir':"/home/", 'walltime':""} + + def __init__(self, config_file='/etc/sfa/oar_config.py'): + self.oarserver = {} self.oarserver['uri'] = None self.oarserver['postformat'] = 'json' @@ -108,23 +115,25 @@ class OARrestapi: #logger.setLevelDebug() self.oarserver['ip'] = self.OAR_IP self.oarserver['port'] = self.OAR_PORT - self.jobstates = ['Terminated', 'Hold', 'Waiting', 'toLaunch', \ - 'toError', 'toAckReservation', 'Launching', \ - 'Finishing', 'Running', 'Suspended', 'Resuming',\ - 'Error'] + self.jobstates = ['Terminated', 'Hold', 'Waiting', 'toLaunch', + 'toError', 'toAckReservation', 'Launching', + 'Finishing', 'Running', 'Suspended', 'Resuming', + 'Error'] self.parser = OARGETParser(self) - def GETRequestToOARRestAPI(self, request, strval=None, next_page=None, username = None ): + def GETRequestToOARRestAPI(self, request, strval=None, + next_page=None, username=None): self.oarserver['uri'] = \ - OARGETParser.OARrequests_uri_dict[request]['uri'] + OARGETParser.OARrequests_uri_dict[request]['uri'] #Get job details with username if 'owner' in OARGETParser.OARrequests_uri_dict[request] and username: - self.oarserver['uri'] += OARGETParser.OARrequests_uri_dict[request]['owner'] + username + self.oarserver['uri'] += \ + OARGETParser.OARrequests_uri_dict[request]['owner'] + username headers = {} data = json.dumps({}) - logger.debug("OARrestapi \tGETRequestToOARRestAPI %s" %(request)) + logger.debug("OARrestapi \tGETRequestToOARRestAPI %s" % (request)) if strval: self.oarserver['uri'] = self.oarserver['uri'].\ replace("id",str(strval)) @@ -171,7 +180,7 @@ class OARrestapi: #first check that all params for are OK try: - self.oarserver['uri'] = OAR_REQUEST_POST_URI_DICT[request]['uri'] + self.oarserver['uri'] = self.OAR_REQUEST_POST_URI_DICT[request]['uri'] except KeyError: logger.log_exc("OARrestapi \tPOSTRequestToOARRestAPI request not \ @@ -184,7 +193,7 @@ class OARrestapi: data = json.dumps(datadict) headers = {'X-REMOTE_IDENT':username, \ - 'content-type': POST_FORMAT['json']['content'], \ + 'content-type': self.POST_FORMAT['json']['content'], \ 'content-length':str(len(data))} try : @@ -236,7 +245,7 @@ def AddNodeRadio(tuplelist, value): def AddMobility(tuplelist, value): if value is 0: tuplelist.append(('mobile', 'False')) - else : + else: tuplelist.append(('mobile', 'True')) def AddPosX(tuplelist, value): @@ -315,7 +324,7 @@ class OARGETParser: api_timezone = self.json_page.raw_json['api_timezone'], api_timestamp = self.json_page.raw_json['api_timestamp'], oar_version = self.json_page.raw_json['oar_version'] ) - else : + else: self.version_json_dict.update(api_version = \ self.json_page.raw_json['api'] , apilib_version = self.json_page.raw_json['apilib'], diff --git a/sfa/iotlab/iotlabapi.py b/sfa/iotlab/iotlabapi.py index d748ef37..dd7300e6 100644 --- a/sfa/iotlab/iotlabapi.py +++ b/sfa/iotlab/iotlabapi.py @@ -5,8 +5,8 @@ from sfa.util.sfalogging import logger from sfa.storage.alchemy import dbsession from sqlalchemy.orm import joinedload from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey -from sfa.iotlab.iotlabpostgres import iotlab_dbsession, IotlabXP - +# from sfa.iotlab.iotlabpostgres import iotlab_dbsession, IotlabXP +from sfa.iotlab.iotlabpostgres import IotlabDB, IotlabXP from sfa.iotlab.OARrestapi import OARrestapi from sfa.iotlab.LDAPapi import LDAPapi @@ -30,6 +30,7 @@ class IotlabTestbedAPI(): :param config: configuration object from sfa.util.config :type config: Config object """ + self.iotlab_db = IotlabDB(config) self.oar = OARrestapi() self.ldap = LDAPapi() self.time_format = "%Y-%m-%d %H:%M:%S" @@ -792,8 +793,8 @@ class IotlabTestbedAPI(): logger.debug("IOTLABDRIVER \r\n \r\n \t AddLeases iotlab_ex_row %s" \ %(iotlab_ex_row)) - iotlab_dbsession.add(iotlab_ex_row) - iotlab_dbsession.commit() + self.iotlab_db.iotlab_dbsession.add(iotlab_ex_row) + self.iotlab_db.iotlab_dbsession.commit() logger.debug("IOTLABDRIVER \t AddLeases hostname_list start_time %s " \ %(start_time)) @@ -857,8 +858,8 @@ class IotlabTestbedAPI(): deleted_jobs = set_jobs_psql.difference(kept_jobs) deleted_jobs = list(deleted_jobs) if len(deleted_jobs) > 0: - iotlab_dbsession.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') - iotlab_dbsession.commit() + self.iotlab_db.iotlab_dbsession.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') + self.iotlab_db.iotlab_dbsession.commit() return @@ -889,7 +890,7 @@ class IotlabTestbedAPI(): #the same user in LDAP SA 27/07/12 job_oar_list = [] - jobs_psql_query = iotlab_dbsession.query(IotlabXP).all() + jobs_psql_query = self.iotlab_db.iotlab_dbsession.query(IotlabXP).all() jobs_psql_dict = dict([(row.job_id, row.__dict__ ) for row in jobs_psql_query ]) #jobs_psql_dict = jobs_psql_dict) logger.debug("IOTLABDRIVER \tGetLeases jobs_psql_dict %s"\ @@ -1017,8 +1018,8 @@ class IotlabTestbedAPI(): #""" ##new_row = FederatedToIotlab(iotlab_hrn, federated_hrn) - ##iotlab_dbsession.add(new_row) - ##iotlab_dbsession.commit() + ##self.iotlab_db.iotlab_dbsession.add(new_row) + ##self.iotlab_db.iotlab_dbsession.commit() #logger.debug("IOTLABDRIVER UpdatePerson EMPTY - DO NOTHING \r\n ") #return diff --git a/sfa/iotlab/iotlabpostgres.py b/sfa/iotlab/iotlabpostgres.py index 9097fdf1..684ed106 100644 --- a/sfa/iotlab/iotlabpostgres.py +++ b/sfa/iotlab/iotlabpostgres.py @@ -1,6 +1,5 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker - from sfa.util.config import Config from sfa.util.sfalogging import logger @@ -66,80 +65,138 @@ class IotlabXP (IotlabBase): -class IotlabDB: +class IotlabDB(object): """ SQL Alchemy connection class. From alchemy.py """ - def __init__(self, config, debug = False): - self.sl_base = IotlabBase - dbname = "iotlab_sfa" - if debug == True : - l_echo_pool = True - l_echo = True - else: - l_echo_pool = False - l_echo = False - - self.iotlab_session = None - # the former PostgreSQL.py used the psycopg2 directly and was doing - #self.connection.set_client_encoding("UNICODE") - # it's unclear how to achieve this in sqlalchemy, nor if it's needed - # at all - # http://www.sqlalchemy.org/docs/dialects/postgresql.html#unicode - # we indeed have /var/lib/pgsql/data/postgresql.conf where - # this setting is unset, it might be an angle to tweak that if need be - # try a unix socket first - omitting the hostname does the trick - unix_url = "postgresql+psycopg2://%s:%s@:%s/%s"% \ - (config.SFA_DB_USER, config.SFA_DB_PASSWORD, \ - config.SFA_DB_PORT, dbname) - - # the TCP fallback method - tcp_url = "postgresql+psycopg2://%s:%s@%s:%s/%s"% \ - (config.SFA_DB_USER, config.SFA_DB_PASSWORD, config.SFA_DB_HOST, \ - config.SFA_DB_PORT, dbname) - for url in [ unix_url, tcp_url ] : - try: - self.iotlab_engine = create_engine (url, echo_pool = \ - l_echo_pool, echo = l_echo) - self.check() - self.url = url + # Stores the unique Singleton instance- + _connection_singleton = None + _dbname = "iotlab_sfa" + + + class Singleton: + """ + Class used with this Python singleton design pattern + @todo Add all variables, and methods needed for the + Singleton class below + """ + + def __init__(self, config, debug = False): + self.iotlab_engine = None + self.iotlab_session = None + self.create_engine(config, debug) + self.session() + + def create_engine(self, config, debug = False): + + + if debug == True: + l_echo_pool = True + l_echo = True + else: + l_echo_pool = False + l_echo = False + # the former PostgreSQL.py used the psycopg2 directly and was doing + #self.connection.set_client_encoding("UNICODE") + # it's unclear how to achieve this in sqlalchemy, nor if it's needed + # at all + # http://www.sqlalchemy.org/docs/dialects/postgresql.html#unicode + # we indeed have /var/lib/pgsql/data/postgresql.conf where + # this setting is unset, it might be an angle to tweak that if need + # be try a unix socket first + # - omitting the hostname does the trick + unix_url = "postgresql+psycopg2://%s:%s@:%s/%s"% \ + (config.SFA_DB_USER, config.SFA_DB_PASSWORD, \ + config.SFA_DB_PORT, IotlabDB._dbname) + + # the TCP fallback method + tcp_url = "postgresql+psycopg2://%s:%s@%s:%s/%s"% \ + (config.SFA_DB_USER, config.SFA_DB_PASSWORD, config.SFA_DB_HOST, \ + config.SFA_DB_PORT, IotlabDB._dbname) + + print "URL", tcp_url , unix_url + for url in [ unix_url, tcp_url ] : + try: + self.iotlab_engine = create_engine (url, echo_pool = + l_echo_pool, echo = l_echo) + self.check() + self.url = url + return + except: + pass + self.iotlab_engine = None + + + raise Exception, "Could not connect to database" + + def check (self): + """ Check if a table exists by trying a selection + on the table. + + """ + self.iotlab_engine.execute ("select 1").scalar() + + + def session (self): + """ + Creates a SQLalchemy session. Once the session object is created + it should be used throughout the code for all the operations on + tables for this given database. + + """ + if self.iotlab_session is None: + Session = sessionmaker() + self.iotlab_session = Session(bind = self.iotlab_engine) + print "\r\n \r\n \r\n OOOOOOOOOHHHHHHHHH YEEEEEEEEEEEEEAH" + return self.iotlab_session + + def close_session(self): + """ + Closes connection to database. + + """ + if self.iotlab_session is None: return - except: - pass - self.iotlab_engine = None - raise Exception, "Could not connect to database" + self.iotlab_session.close() + self.iotlab_session = None - def check (self): - """ Cehck if a table exists by trying a selection - on the table. - """ - self.iotlab_engine.execute ("select 1").scalar() + def __init__(self, config, debug = False): + self.sl_base = IotlabBase + # Check whether we already have an instance + if IotlabDB._connection_singleton is None: + IotlabDB._connection_singleton = IotlabDB.Singleton(config, debug) + # Store instance reference as the only member in the handle + self._EventHandler_singleton = IotlabDB._connection_singleton - def session (self): - """ - Creates a SQLalchemy session. Once the session object is created - it should be used throughout the code for all the operations on - tables for this given database. - """ - if self.iotlab_session is None: - Session = sessionmaker() - self.iotlab_session = Session(bind = self.iotlab_engine) - return self.iotlab_session - def close_session(self): + def __getattr__(self, aAttr): """ - Closes connection to database. + Delegate access to implementation. + :param attr: Attribute wanted. + :return: Attribute """ - if self.iotlab_session is None: return - self.iotlab_session.close() - self.iotlab_session = None + return getattr(self._connection_singleton, aAttr) + + + + # def __setattr__(self, aAttr, aValue): + # """Delegate access to implementation. + + # :param attr: Attribute wanted. + # :param value: Vaule to be set. + # :return: Result of operation. + # """ + # return setattr(self._connection_singleton, aAttr, aValue) + + + def exists(self, tablename): @@ -167,12 +224,12 @@ class IotlabDB: """ logger.debug("SLABPOSTGRES createtable IotlabBase.metadata.sorted_tables \ - %s \r\n engine %s" %(IotlabBase.metadata.sorted_tables , iotlab_engine)) - IotlabBase.metadata.create_all(iotlab_engine) + %s \r\n engine %s" %(IotlabBase.metadata.sorted_tables , self.iotlab_engine)) + IotlabBase.metadata.create_all(self.iotlab_engine) return -iotlab_alchemy = IotlabDB(Config()) -iotlab_engine = iotlab_alchemy.iotlab_engine -iotlab_dbsession = iotlab_alchemy.session() +# iotlab_alchemy = IotlabDB(Config()) +# iotlab_engine = iotlab_alchemy.iotlab_engine +# iotlab_dbsession = iotlab_alchemy.session()