X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;ds=sidebyside;f=sfa%2Fiotlab%2Fiotlabapi.py;h=4f8bcfe9d562bc93da9644140b3111b932e0e4cf;hb=3ea0bfc6a27ddb29dc0d76ca2757d9fededfbc6d;hp=bbae92b218644f91b98e6cef50d152ca27a9d00e;hpb=66be8bb9e8f140b123ca5ddb81b965757b01d547;p=sfa.git diff --git a/sfa/iotlab/iotlabapi.py b/sfa/iotlab/iotlabapi.py index bbae92b2..4f8bcfe9 100644 --- a/sfa/iotlab/iotlabapi.py +++ b/sfa/iotlab/iotlabapi.py @@ -11,7 +11,7 @@ from sfa.util.sfalogging import logger from sfa.storage.alchemy import dbsession from sqlalchemy.orm import joinedload from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey -from sfa.iotlab.iotlabpostgres import IotlabDB, IotlabXP +from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB, LeaseTableXP from sfa.iotlab.OARrestapi import OARrestapi from sfa.iotlab.LDAPapi import LDAPapi @@ -26,7 +26,7 @@ from sfa.iotlab.iotlabaggregate import iotlab_xrn_object class IotlabTestbedAPI(): """ Class enabled to use LDAP and OAR api calls. """ - _MINIMUM_DURATION = 600 + _MINIMUM_DURATION = 10 # 10 units of granularity 60 s, 10 mins def __init__(self, config): """Creates an instance of OARrestapi and LDAPapi which will be used to @@ -37,12 +37,12 @@ class IotlabTestbedAPI(): :param config: configuration object from sfa.util.config :type config: Config object """ - self.iotlab_db = IotlabDB(config) + self.iotlab_db = TestbedAdditionalSfaDB(config) self.oar = OARrestapi() self.ldap = LDAPapi() self.time_format = "%Y-%m-%d %H:%M:%S" self.root_auth = config.SFA_REGISTRY_ROOT_AUTH - self.grain = 1 # 10 mins lease minimum, 1 sec granularity + self.grain = 60 # 10 mins lease minimum, 60 sec granularity #import logging, logging.handlers #from sfa.util.sfalogging import _SfaLogger #sql_logger = _SfaLogger(loggername = 'sqlalchemy.engine', \ @@ -50,7 +50,7 @@ class IotlabTestbedAPI(): return @staticmethod - def GetMinExperimentDurationInSec(): + def GetMinExperimentDurationInGranularity(): """ Returns the minimum allowed duration for an experiment on the testbed. In seconds. @@ -69,8 +69,7 @@ class IotlabTestbedAPI(): existing_records = {} existing_hrns_by_types = {} - logger.debug("IOTLAB_API \tGetPeers peer_filter %s, \ - " %(peer_filter)) + logger.debug("IOTLAB_API \tGetPeers peer_filter %s " % (peer_filter)) all_records = dbsession.query(RegRecord).filter(RegRecord.type.like('%authority%')).all() for record in all_records: @@ -80,20 +79,20 @@ class IotlabTestbedAPI(): else: existing_hrns_by_types[record.type].append(record.hrn) - - logger.debug("IOTLAB_API \tGetPeer\texisting_hrns_by_types %s "\ - %( existing_hrns_by_types)) + logger.debug("IOTLAB_API \tGetPeer\texisting_hrns_by_types %s " + % (existing_hrns_by_types)) records_list = [] try: if peer_filter: - records_list.append(existing_records[(peer_filter,'authority')]) + records_list.append(existing_records[(peer_filter, + 'authority')]) else: for hrn in existing_hrns_by_types['authority']: - records_list.append(existing_records[(hrn,'authority')]) + records_list.append(existing_records[(hrn, 'authority')]) - logger.debug("IOTLAB_API \tGetPeer \trecords_list %s " \ - %(records_list)) + logger.debug("IOTLAB_API \tGetPeer \trecords_list %s " + % (records_list)) except KeyError: pass @@ -112,10 +111,11 @@ class IotlabTestbedAPI(): If a filter is specified, looks for the user whose properties match the filter, otherwise returns the whole enabled users'list. - :param person_filter: Must be a list of dictionnaries - with users properties when not set to None. - :param person_filter: list of dict - :returns:Returns a list of users whose accounts are enabled + :param person_filter: Must be a list of dictionnaries with users + properties when not set to None. + :type person_filter: list of dict + + :returns: Returns a list of users whose accounts are enabled found in ldap. :rtype: list of dicts @@ -244,6 +244,7 @@ class IotlabTestbedAPI(): :returns: dicionary with nodes' hostnames belonging to the job. :rtype: dict + .. warning: Unused. SA 16/10/13 """ req = "GET_jobs_id_resources" @@ -434,8 +435,13 @@ class IotlabTestbedAPI(): def GetSites(self, site_filter_name_list=None, return_fields_list=None): """Returns the list of Iotlab's sites with the associated nodes and - their properties as dictionaries. + the sites' properties as dictionaries. + Site properties: + ['address_ids', 'slice_ids', 'name', 'node_ids', 'url', 'person_ids', + 'site_tag_ids', 'enabled', 'site', 'longitude', 'pcu_ids', + 'max_slivers', 'max_slices', 'ext_consortium_id', 'date_created', + 'latitude', 'is_public', 'peer_site_id', 'peer_id', 'abbreviated_name'] Uses the OAR request GET_sites to find the Iotlab's sites. :param site_filter_name_list: used to specify specific sites @@ -443,7 +449,7 @@ class IotlabTestbedAPI(): :type site_filter_name_list: list :type return_fields_list: list - .. warning:: unused + """ site_dict = self.oar.parser.SendRequest("GET_sites") #site_dict : dict where the key is the sit ename @@ -520,7 +526,7 @@ class IotlabTestbedAPI(): Add a federated user straight to db when the user issues a lease request with iotlab nodes and that he has not registered with iotlab yet (that is he does not have a LDAP entry yet). - Uses parts of the routines in SlabImport when importing user from LDAP. + Uses parts of the routines in IotlabImport when importing user from LDAP. Called by AddPerson, right after LdapAddUser. :param user_dict: Must contain email, hrn and pkey to get a GID and be added to the SFA db. @@ -830,13 +836,13 @@ class IotlabTestbedAPI(): logger.debug("IOTLAB_API \r\n \r\n \t AddLeases %s %s %s " \ %(type(slice_record['hrn']), type(job_id), type(end_time))) - iotlab_ex_row = IotlabXP(slice_hrn = slice_record['hrn'], job_id=job_id, + iotlab_ex_row = LeaseTableXP(slice_hrn = slice_record['hrn'], job_id=job_id, end_time= end_time) logger.debug("IOTLAB_API \r\n \r\n \t AddLeases iotlab_ex_row %s" \ %(iotlab_ex_row)) - self.iotlab_db.iotlab_session.add(iotlab_ex_row) - self.iotlab_db.iotlab_session.commit() + self.iotlab_db.testbed_session.add(iotlab_ex_row) + self.iotlab_db.testbed_session.commit() logger.debug("IOTLAB_API \t AddLeases hostname_list start_time %s " \ %(start_time)) @@ -886,7 +892,7 @@ class IotlabTestbedAPI(): # @staticmethod - # def update_jobs_in_iotlabdb( job_oar_list, jobs_psql): + # def update_experiments_in_additional_sfa_db( job_oar_list, jobs_psql): # """ Cleans the iotlab db by deleting expired and cancelled jobs. # Compares the list of job ids given by OAR with the job ids that # are already in the database, deletes the jobs that are no longer in @@ -900,16 +906,40 @@ class IotlabTestbedAPI(): # set_jobs_psql = set(jobs_psql) # kept_jobs = set(job_oar_list).intersection(set_jobs_psql) - # logger.debug ( "\r\n \t\ update_jobs_in_iotlabdb jobs_psql %s \r\n \t \ + # logger.debug ( "\r\n \t\ update_experiments_in_additional_sfa_db jobs_psql %s \r\n \t \ # job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs)) # deleted_jobs = set_jobs_psql.difference(kept_jobs) # deleted_jobs = list(deleted_jobs) # if len(deleted_jobs) > 0: - # self.iotlab_db.iotlab_session.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') - # self.iotlab_db.iotlab_session.commit() + # self.iotlab_db.testbed_session.query(LeaseTableXP).filter(LeaseTableXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') + # self.iotlab_db.testbed_session.commit() # return + @staticmethod + def filter_lease_name(reservation_list, filter_value): + filtered_reservation_list = list(reservation_list) + logger.debug("IOTLAB_API \t filter_lease_name reservation_list %s" \ + % (reservation_list)) + for reservation in reservation_list: + if 'slice_hrn' in reservation and \ + reservation['slice_hrn'] != filter_value: + filtered_reservation_list.remove(reservation) + + logger.debug("IOTLAB_API \t filter_lease_name filtered_reservation_list %s" \ + % (filtered_reservation_list)) + return filtered_reservation_list + + @staticmethod + def filter_lease_start_time(reservation_list, filter_value): + filtered_reservation_list = list(reservation_list) + + for reservation in reservation_list: + if 't_from' in reservation and \ + reservation['t_from'] > filter_value: + filtered_reservation_list.remove(reservation) + + return filtered_reservation_list def GetLeases(self, lease_filter_dict=None, login=None): """ @@ -919,7 +949,7 @@ class IotlabTestbedAPI(): Two purposes: -Fetch all the jobs from OAR (running, waiting..) complete the reservation information with slice hrn - found in iotlab_xp table. If not available in the table, + found in testbed_xp table. If not available in the table, assume it is a iotlab slice. -Updates the iotlab table, deleting jobs when necessary. @@ -930,7 +960,7 @@ class IotlabTestbedAPI(): """ - unfiltered_reservation_list = self.GetReservedNodes() + unfiltered_reservation_list = self.GetReservedNodes(login) reservation_list = [] #Find the slice associated with this user iotlab ldap uid @@ -941,7 +971,7 @@ class IotlabTestbedAPI(): #the same user in LDAP SA 27/07/12 job_oar_list = [] - jobs_psql_query = self.iotlab_db.iotlab_session.query(IotlabXP).all() + jobs_psql_query = self.iotlab_db.testbed_session.query(LeaseTableXP).all() jobs_psql_dict = dict([(row.job_id, row.__dict__) for row in jobs_psql_query]) #jobs_psql_dict = jobs_psql_dict) @@ -959,7 +989,7 @@ class IotlabTestbedAPI(): if resa['lease_id'] in jobs_psql_dict: job_info = jobs_psql_dict[resa['lease_id']] logger.debug("IOTLAB_API \tGetLeases job_info %s" - % (job_info)) + % (job_info)) resa['slice_hrn'] = job_info['slice_hrn'] resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice') @@ -976,17 +1006,38 @@ class IotlabTestbedAPI(): iotlab_xrn = iotlab_xrn_object(self.root_auth, node) resa['component_id_list'].append(iotlab_xrn.urn) - if lease_filter_dict: - logger.debug("IOTLAB_API \tGetLeases resa_ %s \ - \r\n leasefilter %s" % (resa, lease_filter_dict)) + if lease_filter_dict: + logger.debug("IOTLAB_API \tGetLeases \ + \r\n leasefilter %s" % ( lease_filter_dict)) + + filter_dict_functions = { + 'slice_hrn' : IotlabTestbedAPI.filter_lease_name, + 't_from' : IotlabTestbedAPI.filter_lease_start_time + } + reservation_list = list(unfiltered_reservation_list) + for filter_type in lease_filter_dict: + logger.debug("IOTLAB_API \tGetLeases reservation_list %s" \ + % (reservation_list)) + reservation_list = filter_dict_functions[filter_type](\ + reservation_list,lease_filter_dict[filter_type] ) + + # Filter the reservation list with a maximum timespan so that the + # leases and jobs running after this timestamp do not appear + # in the result leases. + # if 'start_time' in : + # if resa['start_time'] < lease_filter_dict['start_time']: + # reservation_list.append(resa) + + + # if 'name' in lease_filter_dict and \ + # lease_filter_dict['name'] == resa['slice_hrn']: + # reservation_list.append(resa) - if lease_filter_dict['name'] == resa['slice_hrn']: - reservation_list.append(resa) if lease_filter_dict is None: reservation_list = unfiltered_reservation_list - self.iotlab_db.update_jobs_in_iotlabdb(job_oar_list, jobs_psql_id_list) + self.iotlab_db.update_experiments_in_additional_sfa_db(job_oar_list, jobs_psql_id_list) logger.debug(" IOTLAB_API.PY \tGetLeases reservation_list %s" % (reservation_list)) @@ -1067,8 +1118,8 @@ class IotlabTestbedAPI(): #""" ##new_row = FederatedToIotlab(iotlab_hrn, federated_hrn) - ##self.iotlab_db.iotlab_session.add(new_row) - ##self.iotlab_db.iotlab_session.commit() + ##self.iotlab_db.testbed_session.add(new_row) + ##self.iotlab_db.testbed_session.commit() #logger.debug("IOTLAB_API UpdatePerson EMPTY - DO NOTHING \r\n ") #return @@ -1148,7 +1199,7 @@ class IotlabTestbedAPI(): #of the user otherwise will mess up the RegRecord in #Resolve, don't know why - SA 08/08/2012 - #Only one entry for one user = one slice in iotlab_xp table + #Only one entry for one user = one slice in testbed_xp table #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first() #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()