from sfa.util.xrn import Xrn, get_authority, hrn_to_urn
from sfa.iotlab.iotlabdriver import IotlabDriver
-from sfa.iotlab.iotlabpostgres import IotlabDB
+from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB
from sfa.trust.certificate import Keypair, convert_public_key
from sfa.trust.gid import create_uuid
def run(self, options):
"""
- Create the special iotlab table, iotlab_xp, in the iotlab database.
+ Create the special iotlab table, testbed_xp, in the iotlab database.
Import everything (users, slices, nodes and sites from OAR
and LDAP) into the SFA database.
Delete stale records that are no longer in OAR or LDAP.
config = Config()
iotlabdriver = IotlabDriver(config)
- iotlab_db = IotlabDB(config)
+ iotlab_db = TestbedAdditionalSfaDB(config)
#Create special slice table for iotlab
- if not iotlab_db.exists('iotlab_xp'):
+ if not iotlab_db.exists('testbed_xp'):
iotlab_db.createtable()
- self.logger.info("IotlabImporter.run: iotlab_xp table created ")
+ self.logger.info("IotlabImporter.run: testbed_xp table created ")
# import site and node records in site into the SFA db.
self.import_sites_and_nodes(iotlabdriver)
conn = HTTPConnection(self.oarserver['ip'],
self.oarserver['port'])
conn.request("GET", self.oarserver['uri'], data, headers)
- resp = (conn.getresponse()).read()
+ resp = conn.getresponse()
+ body = resp.read()
+ except Exception as error:
+ logger.log_exc("GET_OAR_SRVR : Connection error: %s "
+ % (error))
+ raise Exception ("GET_OAR_SRVR : Connection error %s " %(error))
+
+ finally:
conn.close()
- except HTTPException, error:
- logger.log_exc("GET_OAR_SRVR : Problem with OAR server : %s "
- % (error))
+ # except HTTPException, error:
+ # logger.log_exc("GET_OAR_SRVR : Problem with OAR server : %s "
+ # % (error))
#raise ServerError("GET_OAR_SRVR : Could not reach OARserver")
+ if resp.status >= 400:
+ raise ValueError ("Response Error %s, %s" %(resp.status,
+ resp.reason))
try:
- js_dict = json.loads(resp)
+ js_dict = json.loads(body)
#print "\r\n \t\t\t js_dict keys" , js_dict.keys(), " \r\n", js_dict
return js_dict
except ValueError, error:
logger.log_exc("Failed to parse Server Response: %s ERROR %s"
- % (js_dict, error))
+ % (body, error))
#raise ServerError("Failed to parse Server Response:" + js)
conn = HTTPConnection(self.oarserver['ip'], \
self.oarserver['port'])
conn.request("POST", self.oarserver['uri'], data, headers)
- resp = (conn.getresponse()).read()
- conn.close()
+ resp = conn.getresponse()
+ body = resp.read()
+
except NotConnected:
logger.log_exc("POSTRequestToOARRestAPI NotConnected ERROR: \
data %s \r\n \t\n \t\t headers %s uri %s" \
%(data,headers,self.oarserver['uri']))
+ except Exception as error:
+ logger.log_exc("POST_OAR_SERVER : Connection error: %s "
+ % (error))
+ raise Exception ("POST_OAR_SERVER : Connection error %s " %(error))
+
+ finally:
+ conn.close()
+
+ if resp.status >= 400:
+ raise ValueError ("Response Error %s, %s" %(resp.status,
+ resp.reason))
- #raise ServerError("POST_OAR_SRVR : error")
try:
- answer = json.loads(resp)
+ answer = json.loads(body)
logger.debug("POSTRequestToOARRestAPI : answer %s" % (answer))
return answer
try:
rspec_lease['slice_id'] = lease['slice_id']
except KeyError:
- #No info on the slice used in iotlab_xp table
+ #No info on the slice used in testbed_xp table
pass
rspec_lease['start_time'] = lease['t_from']
rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) \
from sfa.storage.alchemy import dbsession
from sqlalchemy.orm import joinedload
from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey
-from sfa.iotlab.iotlabpostgres import IotlabDB, IotlabXP
+from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB, TestbedXP
from sfa.iotlab.OARrestapi import OARrestapi
from sfa.iotlab.LDAPapi import LDAPapi
:param config: configuration object from sfa.util.config
:type config: Config object
"""
- self.iotlab_db = IotlabDB(config)
+ self.iotlab_db = TestbedAdditionalSfaDB(config)
self.oar = OARrestapi()
self.ldap = LDAPapi()
self.time_format = "%Y-%m-%d %H:%M:%S"
:returns: dicionary with nodes' hostnames belonging to the job.
:rtype: dict
+ .. warning: Unused. SA 16/10/13
"""
req = "GET_jobs_id_resources"
def GetSites(self, site_filter_name_list=None, return_fields_list=None):
"""Returns the list of Iotlab's sites with the associated nodes and
- their properties as dictionaries.
+ the sites' properties as dictionaries.
+ Site properties:
+ ['address_ids', 'slice_ids', 'name', 'node_ids', 'url', 'person_ids',
+ 'site_tag_ids', 'enabled', 'site', 'longitude', 'pcu_ids',
+ 'max_slivers', 'max_slices', 'ext_consortium_id', 'date_created',
+ 'latitude', 'is_public', 'peer_site_id', 'peer_id', 'abbreviated_name']
Uses the OAR request GET_sites to find the Iotlab's sites.
:param site_filter_name_list: used to specify specific sites
:type site_filter_name_list: list
:type return_fields_list: list
- .. warning:: unused
+
"""
site_dict = self.oar.parser.SendRequest("GET_sites")
#site_dict : dict where the key is the sit ename
Add a federated user straight to db when the user issues a lease
request with iotlab nodes and that he has not registered with iotlab
yet (that is he does not have a LDAP entry yet).
- Uses parts of the routines in SlabImport when importing user from LDAP.
+ Uses parts of the routines in IotlabImport when importing user from LDAP.
Called by AddPerson, right after LdapAddUser.
:param user_dict: Must contain email, hrn and pkey to get a GID
and be added to the SFA db.
logger.debug("IOTLAB_API \r\n \r\n \t AddLeases %s %s %s " \
%(type(slice_record['hrn']), type(job_id), type(end_time)))
- iotlab_ex_row = IotlabXP(slice_hrn = slice_record['hrn'], job_id=job_id,
+ iotlab_ex_row = TestbedXP(slice_hrn = slice_record['hrn'], job_id=job_id,
end_time= end_time)
logger.debug("IOTLAB_API \r\n \r\n \t AddLeases iotlab_ex_row %s" \
%(iotlab_ex_row))
- self.iotlab_db.iotlab_session.add(iotlab_ex_row)
- self.iotlab_db.iotlab_session.commit()
+ self.iotlab_db.testbed_session.add(iotlab_ex_row)
+ self.iotlab_db.testbed_session.commit()
logger.debug("IOTLAB_API \t AddLeases hostname_list start_time %s " \
%(start_time))
# @staticmethod
- # def update_jobs_in_iotlabdb( job_oar_list, jobs_psql):
+ # def update_experiments_in_additional_sfa_db( job_oar_list, jobs_psql):
# """ Cleans the iotlab db by deleting expired and cancelled jobs.
# Compares the list of job ids given by OAR with the job ids that
# are already in the database, deletes the jobs that are no longer in
# set_jobs_psql = set(jobs_psql)
# kept_jobs = set(job_oar_list).intersection(set_jobs_psql)
- # logger.debug ( "\r\n \t\ update_jobs_in_iotlabdb jobs_psql %s \r\n \t \
+ # logger.debug ( "\r\n \t\ update_experiments_in_additional_sfa_db jobs_psql %s \r\n \t \
# job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs))
# deleted_jobs = set_jobs_psql.difference(kept_jobs)
# deleted_jobs = list(deleted_jobs)
# if len(deleted_jobs) > 0:
- # self.iotlab_db.iotlab_session.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch')
- # self.iotlab_db.iotlab_session.commit()
+ # self.iotlab_db.testbed_session.query(TestbedXP).filter(TestbedXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch')
+ # self.iotlab_db.testbed_session.commit()
# return
Two purposes:
-Fetch all the jobs from OAR (running, waiting..)
complete the reservation information with slice hrn
- found in iotlab_xp table. If not available in the table,
+ found in testbed_xp table. If not available in the table,
assume it is a iotlab slice.
-Updates the iotlab table, deleting jobs when necessary.
#the same user in LDAP SA 27/07/12
job_oar_list = []
- jobs_psql_query = self.iotlab_db.iotlab_session.query(IotlabXP).all()
+ jobs_psql_query = self.iotlab_db.testbed_session.query(TestbedXP).all()
jobs_psql_dict = dict([(row.job_id, row.__dict__)
for row in jobs_psql_query])
#jobs_psql_dict = jobs_psql_dict)
if lease_filter_dict is None:
reservation_list = unfiltered_reservation_list
- self.iotlab_db.update_jobs_in_iotlabdb(job_oar_list, jobs_psql_id_list)
+ self.iotlab_db.update_experiments_in_additional_sfa_db(job_oar_list, jobs_psql_id_list)
logger.debug(" IOTLAB_API.PY \tGetLeases reservation_list %s"
% (reservation_list))
#"""
##new_row = FederatedToIotlab(iotlab_hrn, federated_hrn)
- ##self.iotlab_db.iotlab_session.add(new_row)
- ##self.iotlab_db.iotlab_session.commit()
+ ##self.iotlab_db.testbed_session.add(new_row)
+ ##self.iotlab_db.testbed_session.commit()
#logger.debug("IOTLAB_API UpdatePerson EMPTY - DO NOTHING \r\n ")
#return
#of the user otherwise will mess up the RegRecord in
#Resolve, don't know why - SA 08/08/2012
- #Only one entry for one user = one slice in iotlab_xp table
+ #Only one entry for one user = one slice in testbed_xp table
#slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first()
#raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
'record_id_slice': 'integer', 'slice_hrn': 'text NOT NULL'}
#Dict with all the specific iotlab tables
-tablenames_dict = {'iotlab_xp': slice_table}
+# tablenames_dict = {'testbed_xp': slice_table}
-IotlabBase = declarative_base()
+TestbedBase = declarative_base()
-class IotlabXP (IotlabBase):
+class TestbedXP (TestbedBase):
""" SQL alchemy class to manipulate the rows of the slice_iotlab table in
- iotlab_sfa database. Handles the records representation and creates the
+ lease_table database. Handles the records representation and creates the
table if it does not exist yet.
"""
- __tablename__ = 'iotlab_xp'
+ __tablename__ = 'testbed_xp'
slice_hrn = Column(String)
- job_id = Column(Integer, primary_key=True)
+ experiment_id = Column(Integer, primary_key=True)
end_time = Column(Integer, nullable=False)
- def __init__(self, slice_hrn=None, job_id=None, end_time=None):
+ def __init__(self, slice_hrn=None, experiment_id=None, end_time=None):
"""
Defines a row of the slice_iotlab table
"""
if slice_hrn:
self.slice_hrn = slice_hrn
- if job_id:
- self.job_id = job_id
+ if experiment_id:
+ self.experiment_id = experiment_id
if end_time:
self.end_time = end_time
"""Prints the SQLAlchemy record to the format defined
by the function.
"""
- result = "<iotlab_xp : slice_hrn = %s , job_id %s end_time = %s" \
- % (self.slice_hrn, self.job_id, self.end_time)
+ result = "<testbed_xp : slice_hrn = %s , experiment_id %s end_time = %s" \
+ % (self.slice_hrn, self.experiment_id, self.end_time)
result += ">"
return result
-class IotlabDB(object):
+class TestbedAdditionalSfaDB(object):
""" SQL Alchemy connection class.
From alchemy.py
"""
# Stores the unique Singleton instance-
_connection_singleton = None
# defines the database name
- dbname = "iotlab_sfa"
+ dbname = "lease_table"
class Singleton:
"""
"""
def __init__(self, config, debug=False):
- self.iotlab_engine = None
- self.iotlab_session = None
+ self.testbed_engine = None
+ self.testbed_session = None
self.url = None
- self.create_iotlab_engine(config, debug)
+ self.create_testbed_engine(config, debug)
self.session()
- def create_iotlab_engine(self, config, debug=False):
+ def create_testbed_engine(self, config, debug=False):
"""Creates the SQLAlchemy engine, which is the starting point for
any SQLAlchemy application.
:param config: configuration object created by SFA based on the
# - omitting the hostname does the trick
unix_url = "postgresql+psycopg2://%s:%s@:%s/%s" \
% (config.SFA_DB_USER, config.SFA_DB_PASSWORD,
- config.SFA_DB_PORT, IotlabDB.dbname)
+ config.SFA_DB_PORT, TestbedAdditionalSfaDB.dbname)
# the TCP fallback method
tcp_url = "postgresql+psycopg2://%s:%s@%s:%s/%s" \
% (config.SFA_DB_USER, config.SFA_DB_PASSWORD,
- config.SFA_DB_HOST, config.SFA_DB_PORT, IotlabDB.dbname)
+ config.SFA_DB_HOST, config.SFA_DB_PORT, TestbedAdditionalSfaDB.dbname)
for url in [unix_url, tcp_url]:
try:
- self.iotlab_engine = create_engine(
+ self.testbed_engine = create_engine(
url, echo_pool=l_echo_pool, echo=l_echo)
self.check()
self.url = url
return
except:
pass
- self.iotlab_engine = None
+ self.testbed_engine = None
raise Exception("Could not connect to database")
on the table.
"""
- self.iotlab_engine.execute("select 1").scalar()
+ self.testbed_engine.execute("select 1").scalar()
def session(self):
tables for this given database.
"""
- if self.iotlab_session is None:
+ if self.testbed_session is None:
Session = sessionmaker()
- self.iotlab_session = Session(bind=self.iotlab_engine)
- return self.iotlab_session
+ self.testbed_session = Session(bind=self.testbed_engine)
+ return self.testbed_session
def close_session(self):
"""
Closes connection to database.
"""
- if self.iotlab_session is None:
+ if self.testbed_session is None:
return
- self.iotlab_session.close()
- self.iotlab_session = None
+ self.testbed_session.close()
+ self.testbed_session = None
- def update_jobs_in_iotlabdb(self, job_oar_list, jobs_psql):
+ def update_experiments_in_additional_sfa_db(self,
+ experiment_list_from_testbed, experiment_list_in_db):
""" Cleans the iotlab db by deleting expired and cancelled jobs.
- Compares the list of job ids given by OAR with the job ids that
- are already in the database, deletes the jobs that are no longer in
- the OAR job id list.
+ Compares the list of experiment ids given by the testbed with the
+ experiment ids that are already in the database, deletes the
+ experiments that are no longer in the testbed experiment id list.
- :param job_oar_list: list of job ids coming from OAR
- :type job_oar_list: list
- :param job_psql: list of job ids from the database.
- :type job_psql: list
+ :param experiment_list_from_testbed: list of experiment ids coming
+ from testbed
+ :type experiment_list_from_testbed: list
+ :param experiment_list_in_db: list of experiment ids from the sfa
+ additionnal database.
+ :type experiment_list_in_db: list
:returns: None
"""
#Turn the list into a set
- set_jobs_psql = set(jobs_psql)
-
- kept_jobs = set(job_oar_list).intersection(set_jobs_psql)
- logger.debug("\r\n \t update_jobs_in_iotlabdb jobs_psql %s \r\n \
- job_oar_list %s kept_jobs %s "
- % (set_jobs_psql, job_oar_list, kept_jobs))
- deleted_jobs = set_jobs_psql.difference(kept_jobs)
- deleted_jobs = list(deleted_jobs)
- if len(deleted_jobs) > 0:
- self.iotlab_session.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch')
- self.iotlab_session.commit()
+ set_experiment_list_in_db = set(experiment_list_in_db)
+
+ kept_experiments = set(experiment_list_from_testbed).intersection(set_experiment_list_in_db)
+ logger.debug("\r\n \t update_experiments_in_additional_sfa_db \
+ experiment_list_in_db %s \r\n \
+ experiment_list_from_testbed %s \
+ kept_experiments %s "
+ % (set_experiment_list_in_db,
+ experiment_list_from_testbed, kept_experiments))
+ deleted_experiments = set_experiment_list_in_db.difference(
+ kept_experiments)
+ deleted_experiments = list(deleted_experiments)
+ if len(deleted_experiments) > 0:
+ self.testbed_session.query(TestbedXP).filter(TestbedXP.job_id.in_(deleted_experiments)).delete(synchronize_session='fetch')
+ self.testbed_session.commit()
return
def __init__(self, config, debug=False):
- self.sl_base = IotlabBase
+ self.sl_base = TestbedBase
# Check whether we already have an instance
- if IotlabDB._connection_singleton is None:
- IotlabDB._connection_singleton = IotlabDB.Singleton(config, debug)
+ if TestbedAdditionalSfaDB._connection_singleton is None:
+ TestbedAdditionalSfaDB._connection_singleton = \
+ TestbedAdditionalSfaDB.Singleton(config, debug)
# Store instance reference as the only member in the handle
- self._EventHandler_singleton = IotlabDB._connection_singleton
+ self._EventHandler_singleton = \
+ TestbedAdditionalSfaDB._connection_singleton
def __getattr__(self, aAttr):
"""
:rtype: bool
"""
- metadata = MetaData(bind=self.iotlab_engine)
+ metadata = MetaData(bind=self.testbed_engine)
try:
table = Table(tablename, metadata, autoload=True)
return True
"""
- logger.debug("SLABPOSTGRES createtable \
- IotlabBase.metadata.sorted_tables %s \r\n engine %s"
- % (IotlabBase.metadata.sorted_tables, self.iotlab_engine))
- IotlabBase.metadata.create_all(self.iotlab_engine)
+ logger.debug("IOTLABPOSTGRES createtable \
+ TestbedBase.metadata.sorted_tables %s \r\n engine %s"
+ % (TestbedBase.metadata.sorted_tables, self.testbed_engine))
+ TestbedBase.metadata.create_all(self.testbed_engine)
return
git://git.onelab.eu / sfa.git / commitdiff