From 0a4f0ce95a763b227ed4b7c553804a1c072867d5 Mon Sep 17 00:00:00 2001 From: Sandrine Avakian Date: Thu, 17 Oct 2013 14:57:09 +0200 Subject: [PATCH] Adding more error check in OARrestapi. Changing iotlabpostgres classes name (Db, tables, and table fields) so that it can be reused in an other testbed. Propagating the changes in iotlabaggregate and iotlabapi. Conflicts: sfa/iotlab/iotlabpostgres.py --- sfa/importer/iotlabimporter.py | 10 +-- sfa/iotlab/OARrestapi.py | 41 ++++++++--- sfa/iotlab/iotlabaggregate.py | 2 +- sfa/iotlab/iotlabapi.py | 42 +++++++----- sfa/iotlab/iotlabpostgres.py | 121 ++++++++++++++++++--------------- 5 files changed, 126 insertions(+), 90 deletions(-) diff --git a/sfa/importer/iotlabimporter.py b/sfa/importer/iotlabimporter.py index 5bd03444..86874375 100644 --- a/sfa/importer/iotlabimporter.py +++ b/sfa/importer/iotlabimporter.py @@ -7,7 +7,7 @@ from sfa.util.config import Config from sfa.util.xrn import Xrn, get_authority, hrn_to_urn from sfa.iotlab.iotlabdriver import IotlabDriver -from sfa.iotlab.iotlabpostgres import IotlabDB +from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB from sfa.trust.certificate import Keypair, convert_public_key from sfa.trust.gid import create_uuid @@ -505,7 +505,7 @@ class IotlabImporter: def run(self, options): """ - Create the special iotlab table, iotlab_xp, in the iotlab database. + Create the special iotlab table, testbed_xp, in the iotlab database. Import everything (users, slices, nodes and sites from OAR and LDAP) into the SFA database. Delete stale records that are no longer in OAR or LDAP. @@ -515,12 +515,12 @@ class IotlabImporter: config = Config() iotlabdriver = IotlabDriver(config) - iotlab_db = IotlabDB(config) + iotlab_db = TestbedAdditionalSfaDB(config) #Create special slice table for iotlab - if not iotlab_db.exists('iotlab_xp'): + if not iotlab_db.exists('testbed_xp'): iotlab_db.createtable() - self.logger.info("IotlabImporter.run: iotlab_xp table created ") + self.logger.info("IotlabImporter.run: testbed_xp table created ") # import site and node records in site into the SFA db. self.import_sites_and_nodes(iotlabdriver) diff --git a/sfa/iotlab/OARrestapi.py b/sfa/iotlab/OARrestapi.py index 515b7926..a4ce7a8b 100644 --- a/sfa/iotlab/OARrestapi.py +++ b/sfa/iotlab/OARrestapi.py @@ -213,21 +213,31 @@ class OARrestapi: conn = HTTPConnection(self.oarserver['ip'], self.oarserver['port']) conn.request("GET", self.oarserver['uri'], data, headers) - resp = (conn.getresponse()).read() + resp = conn.getresponse() + body = resp.read() + except Exception as error: + logger.log_exc("GET_OAR_SRVR : Connection error: %s " + % (error)) + raise Exception ("GET_OAR_SRVR : Connection error %s " %(error)) + + finally: conn.close() - except HTTPException, error: - logger.log_exc("GET_OAR_SRVR : Problem with OAR server : %s " - % (error)) + # except HTTPException, error: + # logger.log_exc("GET_OAR_SRVR : Problem with OAR server : %s " + # % (error)) #raise ServerError("GET_OAR_SRVR : Could not reach OARserver") + if resp.status >= 400: + raise ValueError ("Response Error %s, %s" %(resp.status, + resp.reason)) try: - js_dict = json.loads(resp) + js_dict = json.loads(body) #print "\r\n \t\t\t js_dict keys" , js_dict.keys(), " \r\n", js_dict return js_dict except ValueError, error: logger.log_exc("Failed to parse Server Response: %s ERROR %s" - % (js_dict, error)) + % (body, error)) #raise ServerError("Failed to parse Server Response:" + js) @@ -260,17 +270,28 @@ class OARrestapi: conn = HTTPConnection(self.oarserver['ip'], \ self.oarserver['port']) conn.request("POST", self.oarserver['uri'], data, headers) - resp = (conn.getresponse()).read() - conn.close() + resp = conn.getresponse() + body = resp.read() + except NotConnected: logger.log_exc("POSTRequestToOARRestAPI NotConnected ERROR: \ data %s \r\n \t\n \t\t headers %s uri %s" \ %(data,headers,self.oarserver['uri'])) + except Exception as error: + logger.log_exc("POST_OAR_SERVER : Connection error: %s " + % (error)) + raise Exception ("POST_OAR_SERVER : Connection error %s " %(error)) + + finally: + conn.close() + + if resp.status >= 400: + raise ValueError ("Response Error %s, %s" %(resp.status, + resp.reason)) - #raise ServerError("POST_OAR_SRVR : error") try: - answer = json.loads(resp) + answer = json.loads(body) logger.debug("POSTRequestToOARRestAPI : answer %s" % (answer)) return answer diff --git a/sfa/iotlab/iotlabaggregate.py b/sfa/iotlab/iotlabaggregate.py index 9cc600fc..fc5b2dbc 100644 --- a/sfa/iotlab/iotlabaggregate.py +++ b/sfa/iotlab/iotlabaggregate.py @@ -329,7 +329,7 @@ class IotlabAggregate: try: rspec_lease['slice_id'] = lease['slice_id'] except KeyError: - #No info on the slice used in iotlab_xp table + #No info on the slice used in testbed_xp table pass rspec_lease['start_time'] = lease['t_from'] rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) \ diff --git a/sfa/iotlab/iotlabapi.py b/sfa/iotlab/iotlabapi.py index ab271694..de5d00df 100644 --- a/sfa/iotlab/iotlabapi.py +++ b/sfa/iotlab/iotlabapi.py @@ -11,7 +11,7 @@ from sfa.util.sfalogging import logger from sfa.storage.alchemy import dbsession from sqlalchemy.orm import joinedload from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey -from sfa.iotlab.iotlabpostgres import IotlabDB, IotlabXP +from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB, TestbedXP from sfa.iotlab.OARrestapi import OARrestapi from sfa.iotlab.LDAPapi import LDAPapi @@ -37,7 +37,7 @@ class IotlabTestbedAPI(): :param config: configuration object from sfa.util.config :type config: Config object """ - self.iotlab_db = IotlabDB(config) + self.iotlab_db = TestbedAdditionalSfaDB(config) self.oar = OARrestapi() self.ldap = LDAPapi() self.time_format = "%Y-%m-%d %H:%M:%S" @@ -244,6 +244,7 @@ class IotlabTestbedAPI(): :returns: dicionary with nodes' hostnames belonging to the job. :rtype: dict + .. warning: Unused. SA 16/10/13 """ req = "GET_jobs_id_resources" @@ -434,8 +435,13 @@ class IotlabTestbedAPI(): def GetSites(self, site_filter_name_list=None, return_fields_list=None): """Returns the list of Iotlab's sites with the associated nodes and - their properties as dictionaries. + the sites' properties as dictionaries. + Site properties: + ['address_ids', 'slice_ids', 'name', 'node_ids', 'url', 'person_ids', + 'site_tag_ids', 'enabled', 'site', 'longitude', 'pcu_ids', + 'max_slivers', 'max_slices', 'ext_consortium_id', 'date_created', + 'latitude', 'is_public', 'peer_site_id', 'peer_id', 'abbreviated_name'] Uses the OAR request GET_sites to find the Iotlab's sites. :param site_filter_name_list: used to specify specific sites @@ -443,7 +449,7 @@ class IotlabTestbedAPI(): :type site_filter_name_list: list :type return_fields_list: list - .. warning:: unused + """ site_dict = self.oar.parser.SendRequest("GET_sites") #site_dict : dict where the key is the sit ename @@ -520,7 +526,7 @@ class IotlabTestbedAPI(): Add a federated user straight to db when the user issues a lease request with iotlab nodes and that he has not registered with iotlab yet (that is he does not have a LDAP entry yet). - Uses parts of the routines in SlabImport when importing user from LDAP. + Uses parts of the routines in IotlabImport when importing user from LDAP. Called by AddPerson, right after LdapAddUser. :param user_dict: Must contain email, hrn and pkey to get a GID and be added to the SFA db. @@ -830,13 +836,13 @@ class IotlabTestbedAPI(): logger.debug("IOTLAB_API \r\n \r\n \t AddLeases %s %s %s " \ %(type(slice_record['hrn']), type(job_id), type(end_time))) - iotlab_ex_row = IotlabXP(slice_hrn = slice_record['hrn'], job_id=job_id, + iotlab_ex_row = TestbedXP(slice_hrn = slice_record['hrn'], job_id=job_id, end_time= end_time) logger.debug("IOTLAB_API \r\n \r\n \t AddLeases iotlab_ex_row %s" \ %(iotlab_ex_row)) - self.iotlab_db.iotlab_session.add(iotlab_ex_row) - self.iotlab_db.iotlab_session.commit() + self.iotlab_db.testbed_session.add(iotlab_ex_row) + self.iotlab_db.testbed_session.commit() logger.debug("IOTLAB_API \t AddLeases hostname_list start_time %s " \ %(start_time)) @@ -886,7 +892,7 @@ class IotlabTestbedAPI(): # @staticmethod - # def update_jobs_in_iotlabdb( job_oar_list, jobs_psql): + # def update_experiments_in_additional_sfa_db( job_oar_list, jobs_psql): # """ Cleans the iotlab db by deleting expired and cancelled jobs. # Compares the list of job ids given by OAR with the job ids that # are already in the database, deletes the jobs that are no longer in @@ -900,13 +906,13 @@ class IotlabTestbedAPI(): # set_jobs_psql = set(jobs_psql) # kept_jobs = set(job_oar_list).intersection(set_jobs_psql) - # logger.debug ( "\r\n \t\ update_jobs_in_iotlabdb jobs_psql %s \r\n \t \ + # logger.debug ( "\r\n \t\ update_experiments_in_additional_sfa_db jobs_psql %s \r\n \t \ # job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs)) # deleted_jobs = set_jobs_psql.difference(kept_jobs) # deleted_jobs = list(deleted_jobs) # if len(deleted_jobs) > 0: - # self.iotlab_db.iotlab_session.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') - # self.iotlab_db.iotlab_session.commit() + # self.iotlab_db.testbed_session.query(TestbedXP).filter(TestbedXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') + # self.iotlab_db.testbed_session.commit() # return @@ -943,7 +949,7 @@ class IotlabTestbedAPI(): Two purposes: -Fetch all the jobs from OAR (running, waiting..) complete the reservation information with slice hrn - found in iotlab_xp table. If not available in the table, + found in testbed_xp table. If not available in the table, assume it is a iotlab slice. -Updates the iotlab table, deleting jobs when necessary. @@ -965,7 +971,7 @@ class IotlabTestbedAPI(): #the same user in LDAP SA 27/07/12 job_oar_list = [] - jobs_psql_query = self.iotlab_db.iotlab_session.query(IotlabXP).all() + jobs_psql_query = self.iotlab_db.testbed_session.query(TestbedXP).all() jobs_psql_dict = dict([(row.job_id, row.__dict__) for row in jobs_psql_query]) #jobs_psql_dict = jobs_psql_dict) @@ -1031,7 +1037,7 @@ class IotlabTestbedAPI(): if lease_filter_dict is None: reservation_list = unfiltered_reservation_list - self.iotlab_db.update_jobs_in_iotlabdb(job_oar_list, jobs_psql_id_list) + self.iotlab_db.update_experiments_in_additional_sfa_db(job_oar_list, jobs_psql_id_list) logger.debug(" IOTLAB_API.PY \tGetLeases reservation_list %s" % (reservation_list)) @@ -1112,8 +1118,8 @@ class IotlabTestbedAPI(): #""" ##new_row = FederatedToIotlab(iotlab_hrn, federated_hrn) - ##self.iotlab_db.iotlab_session.add(new_row) - ##self.iotlab_db.iotlab_session.commit() + ##self.iotlab_db.testbed_session.add(new_row) + ##self.iotlab_db.testbed_session.commit() #logger.debug("IOTLAB_API UpdatePerson EMPTY - DO NOTHING \r\n ") #return @@ -1193,7 +1199,7 @@ class IotlabTestbedAPI(): #of the user otherwise will mess up the RegRecord in #Resolve, don't know why - SA 08/08/2012 - #Only one entry for one user = one slice in iotlab_xp table + #Only one entry for one user = one slice in testbed_xp table #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first() #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() diff --git a/sfa/iotlab/iotlabpostgres.py b/sfa/iotlab/iotlabpostgres.py index 3c65278b..d454cbf7 100644 --- a/sfa/iotlab/iotlabpostgres.py +++ b/sfa/iotlab/iotlabpostgres.py @@ -23,32 +23,32 @@ slice_table = {'record_id_user': 'integer PRIMARY KEY references X ON DELETE \ 'record_id_slice': 'integer', 'slice_hrn': 'text NOT NULL'} #Dict with all the specific iotlab tables -tablenames_dict = {'iotlab_xp': slice_table} +# tablenames_dict = {'testbed_xp': slice_table} -IotlabBase = declarative_base() +TestbedBase = declarative_base() -class IotlabXP (IotlabBase): +class TestbedXP (TestbedBase): """ SQL alchemy class to manipulate the rows of the slice_iotlab table in - iotlab_sfa database. Handles the records representation and creates the + lease_table database. Handles the records representation and creates the table if it does not exist yet. """ - __tablename__ = 'iotlab_xp' + __tablename__ = 'testbed_xp' slice_hrn = Column(String) - job_id = Column(Integer, primary_key=True) + experiment_id = Column(Integer, primary_key=True) end_time = Column(Integer, nullable=False) - def __init__(self, slice_hrn=None, job_id=None, end_time=None): + def __init__(self, slice_hrn=None, experiment_id=None, end_time=None): """ Defines a row of the slice_iotlab table """ if slice_hrn: self.slice_hrn = slice_hrn - if job_id: - self.job_id = job_id + if experiment_id: + self.experiment_id = experiment_id if end_time: self.end_time = end_time @@ -56,20 +56,20 @@ class IotlabXP (IotlabBase): """Prints the SQLAlchemy record to the format defined by the function. """ - result = " 0: - self.iotlab_session.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') - self.iotlab_session.commit() + set_experiment_list_in_db = set(experiment_list_in_db) + + kept_experiments = set(experiment_list_from_testbed).intersection(set_experiment_list_in_db) + logger.debug("\r\n \t update_experiments_in_additional_sfa_db \ + experiment_list_in_db %s \r\n \ + experiment_list_from_testbed %s \ + kept_experiments %s " + % (set_experiment_list_in_db, + experiment_list_from_testbed, kept_experiments)) + deleted_experiments = set_experiment_list_in_db.difference( + kept_experiments) + deleted_experiments = list(deleted_experiments) + if len(deleted_experiments) > 0: + self.testbed_session.query(TestbedXP).filter(TestbedXP.job_id.in_(deleted_experiments)).delete(synchronize_session='fetch') + self.testbed_session.commit() return def __init__(self, config, debug=False): - self.sl_base = IotlabBase + self.sl_base = TestbedBase # Check whether we already have an instance - if IotlabDB._connection_singleton is None: - IotlabDB._connection_singleton = IotlabDB.Singleton(config, debug) + if TestbedAdditionalSfaDB._connection_singleton is None: + TestbedAdditionalSfaDB._connection_singleton = \ + TestbedAdditionalSfaDB.Singleton(config, debug) # Store instance reference as the only member in the handle - self._EventHandler_singleton = IotlabDB._connection_singleton + self._EventHandler_singleton = \ + TestbedAdditionalSfaDB._connection_singleton def __getattr__(self, aAttr): """ @@ -240,7 +249,7 @@ class IotlabDB(object): :rtype: bool """ - metadata = MetaData(bind=self.iotlab_engine) + metadata = MetaData(bind=self.testbed_engine) try: table = Table(tablename, metadata, autoload=True) return True @@ -257,8 +266,8 @@ class IotlabDB(object): """ - logger.debug("SLABPOSTGRES createtable \ - IotlabBase.metadata.sorted_tables %s \r\n engine %s" - % (IotlabBase.metadata.sorted_tables, self.iotlab_engine)) - IotlabBase.metadata.create_all(self.iotlab_engine) + logger.debug("IOTLABPOSTGRES createtable \ + TestbedBase.metadata.sorted_tables %s \r\n engine %s" + % (TestbedBase.metadata.sorted_tables, self.testbed_engine)) + TestbedBase.metadata.create_all(self.testbed_engine) return -- 2.43.0