File containing the IotlabShell, used to interact with nodes, users,
slices, leases and keys, as well as the dedicated iotlab database and table,
holding information about which slice is running which job.
-TODO: Remove interactons with the SFA DB and put it in the driver iotlabdriver
-instead.
"""
from datetime import datetime
from sfa.util.sfalogging import logger
+from sfa.util.sfatime import SFATIME_FORMAT
-
-from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB, LeaseTableXP
from sfa.iotlab.OARrestapi import OARrestapi
from sfa.iotlab.LDAPapi import LDAPapi
-from sfa.util.xrn import Xrn, hrn_to_urn, get_authority
-
-from sfa.iotlab.iotlabxrn import xrn_object
class IotlabShell():
""" Class enabled to use LDAP and OAR api calls. """
:type config: Config object
"""
- self.leases_db = TestbedAdditionalSfaDB(config)
+ # self.leases_db = TestbedAdditionalSfaDB(config)
self.oar = OARrestapi()
self.ldap = LDAPapi()
- self.time_format = "%Y-%m-%d %H:%M:%S"
+ self.time_format = SFATIME_FORMAT
self.root_auth = config.SFA_REGISTRY_ROOT_AUTH
self.grain = 60 # 10 mins lease minimum, 60 sec granularity
#import logging, logging.handlers
#Get job resources list from OAR
node_id_list = self.oar.parser.SendRequest(req, job_id, username)
logger.debug("IOTLAB_API \t GetJobsResources %s " %(node_id_list))
-
+ resources = self.GetNodes()
+ oar_id_node_dict = {}
+ for node in resources:
+ oar_id_node_dict[node['oar_id']] = node['hostname']
hostname_list = \
- self.__get_hostnames_from_oar_node_ids(node_id_list)
+ self.__get_hostnames_from_oar_node_ids(oar_id_node_dict,
+ node_id_list)
#Replaces the previous entry "assigned_network_address" /
"""
return self.oar.parser.SendRequest("GET_running_jobs")
- def __get_hostnames_from_oar_node_ids(self, oar_id_node_dict,
+ @staticmethod
+ def __get_hostnames_from_oar_node_ids(oar_id_node_dict,
resource_id_list ):
"""Get the hostnames of the nodes from their OAR identifiers.
Get the list of nodes dict using GetNodes and find the hostname
return jobid
- def AddLeases(self, hostname_list, slice_record,
- lease_start_time, lease_duration):
-
- """Creates a job in OAR corresponding to the information provided
- as parameters. Adds the job id and the slice hrn in the iotlab
- database so that we are able to know which slice has which nodes.
-
- :param hostname_list: list of nodes' OAR hostnames.
- :param slice_record: sfa slice record, must contain login and hrn.
- :param lease_start_time: starting time , unix timestamp format
- :param lease_duration: duration in minutes
-
- :type hostname_list: list
- :type slice_record: dict
- :type lease_start_time: integer
- :type lease_duration: integer
- :returns: job_id, can be None if the job request failed.
-
- """
- logger.debug("IOTLAB_API \r\n \r\n \t AddLeases hostname_list %s \
- slice_record %s lease_start_time %s lease_duration %s "\
- %( hostname_list, slice_record , lease_start_time, \
- lease_duration))
-
- #tmp = slice_record['reg-researchers'][0].split(".")
- username = slice_record['login']
- #username = tmp[(len(tmp)-1)]
- job_id = self.LaunchExperimentOnOAR(hostname_list, \
- slice_record['hrn'], \
- lease_start_time, lease_duration, \
- username)
- if job_id is not None:
- start_time = \
- datetime.fromtimestamp(int(lease_start_time)).\
- strftime(self.time_format)
- end_time = lease_start_time + lease_duration
-
-
- logger.debug("IOTLAB_API \r\n \r\n \t AddLeases TURN ON LOGGING SQL \
- %s %s %s "%(slice_record['hrn'], job_id, end_time))
-
- logger.debug("IOTLAB_API \r\n \r\n \t AddLeases %s %s %s " \
- %(type(slice_record['hrn']), type(job_id), type(end_time)))
-
- iotlab_ex_row = LeaseTableXP(slice_hrn = slice_record['hrn'],
- experiment_id=job_id,
- end_time= end_time)
-
- logger.debug("IOTLAB_API \r\n \r\n \t AddLeases iotlab_ex_row %s" \
- %(iotlab_ex_row))
- self.leases_db.testbed_session.add(iotlab_ex_row)
- self.leases_db.testbed_session.commit()
-
- logger.debug("IOTLAB_API \t AddLeases hostname_list start_time %s "
- %(start_time))
-
- return job_id
#Delete the jobs from job_iotlab table
@staticmethod
- def filter_lease_name(reservation_list, filter_value):
+ def filter_lease(reservation_list, filter_type, filter_value ):
+ """Filters the lease reservation list by removing each lease whose
+ filter_type is not equal to the filter_value provided. Returns the list
+ of leases in one slice, defined by the slice_hrn if filter_type
+ is 'slice_hrn'. Otherwise, returns all leases scheduled starting from
+ the filter_value if filter_type is 't_from'.
+
+ :param reservation_list: leases list
+ :type reservation_list: list of dictionary
+ :param filter_type: can be either 't_from' or 'slice hrn'
+ :type filter_type: string
+ :param filter_value: depending on the filter_type, can be the slice_hrn
+ or can be defining a timespan.
+ :type filter_value: if filter_type is 't_from', filter_value is int.
+ if filter_type is 'slice_hrn', filter_value is a string.
+
+
+ :returns: filtered_reservation_list, contains only leases running or
+ scheduled in the given slice (wanted_slice).Dict keys are
+ 'lease_id','reserved_nodes','slice_id', 'state', 'user',
+ 'component_id_list','slice_hrn', 'resource_ids', 't_from', 't_until'
+ :rtype: list of dict
+
+ """
filtered_reservation_list = list(reservation_list)
logger.debug("IOTLAB_API \t filter_lease_name reservation_list %s" \
% (reservation_list))
- for reservation in reservation_list:
- if 'slice_hrn' in reservation and \
- reservation['slice_hrn'] != filter_value:
- filtered_reservation_list.remove(reservation)
+ try:
+ for reservation in reservation_list:
+ if \
+ (filter_type is 'slice_hrn' and \
+ reservation['slice_hrn'] != filter_value) or \
+ (filter_type is 't_from' and \
+ reservation['t_from'] > filter_value):
+ filtered_reservation_list.remove(reservation)
+ except TypeError:
+ logger.log_exc("Iotlabshell filter_lease : filter_type %s \
+ filter_value %s not in lease" %(filter_type,
+ filter_value))
- logger.debug("IOTLAB_API \t filter_lease_name filtered_reservation_list\
- %s" % (filtered_reservation_list))
return filtered_reservation_list
- @staticmethod
- def filter_lease_start_time(reservation_list, filter_value):
- filtered_reservation_list = list(reservation_list)
-
- for reservation in reservation_list:
- if 't_from' in reservation and \
- reservation['t_from'] > filter_value:
- filtered_reservation_list.remove(reservation)
+ # @staticmethod
+ # def filter_lease_start_time(reservation_list, timespan):
+ # """Filters the lease reservation list by removing each lease whose
+ # slice_hrn is not the wanted_slice provided. Returns the list of leases
+ # in one slice (wanted_slice).
- return filtered_reservation_list
+ # """
+ # filtered_reservation_list = list(reservation_list)
+ # for reservation in reservation_list:
+ # if 't_from' in reservation and \
+ # reservation['t_from'] > timespan:
+ # filtered_reservation_list.remove(reservation)
- def GetLeases(self, lease_filter_dict=None, login=None):
- """
+ # return filtered_reservation_list
- Get the list of leases from OAR with complete information
- about which slice owns which jobs and nodes.
- Two purposes:
- -Fetch all the jobs from OAR (running, waiting..)
- complete the reservation information with slice hrn
- found in testbed_xp table. If not available in the table,
- assume it is a iotlab slice.
- -Updates the iotlab table, deleting jobs when necessary.
-
- :returns: reservation_list, list of dictionaries with 'lease_id',
- 'reserved_nodes','slice_id', 'state', 'user', 'component_id_list',
- 'slice_hrn', 'resource_ids', 't_from', 't_until'
- :rtype: list
-
- """
- unfiltered_reservation_list = self.GetReservedNodes(login)
-
- reservation_list = []
- #Find the slice associated with this user iotlab ldap uid
- logger.debug(" IOTLAB_API.PY \tGetLeases login %s\
- unfiltered_reservation_list %s "
- % (login, unfiltered_reservation_list))
- #Create user dict first to avoid looking several times for
- #the same user in LDAP SA 27/07/12
- job_oar_list = []
- jobs_psql_query = self.leases_db.testbed_session.query(LeaseTableXP).all()
- jobs_psql_dict = dict([(row.experiment_id, row.__dict__)
- for row in jobs_psql_query])
- #jobs_psql_dict = jobs_psql_dict)
- logger.debug("IOTLAB_API \tGetLeases jobs_psql_dict %s"
- % (jobs_psql_dict))
- jobs_psql_id_list = [row.experiment_id for row in jobs_psql_query]
-
- for resa in unfiltered_reservation_list:
- logger.debug("IOTLAB_API \tGetLeases USER %s"
- % (resa['user']))
- #Construct list of jobs (runing, waiting..) in oar
- job_oar_list.append(resa['lease_id'])
- #If there is information on the job in IOTLAB DB ]
- #(slice used and job id)
- if resa['lease_id'] in jobs_psql_dict:
- job_info = jobs_psql_dict[resa['lease_id']]
- logger.debug("IOTLAB_API \tGetLeases job_info %s"
- % (job_info))
- resa['slice_hrn'] = job_info['slice_hrn']
- resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice')
-
- #otherwise, assume it is a iotlab slice:
- else:
- resa['slice_id'] = hrn_to_urn(self.root_auth + '.' +
- resa['user'] + "_slice", 'slice')
- resa['slice_hrn'] = Xrn(resa['slice_id']).get_hrn()
-
- resa['component_id_list'] = []
- #Transform the hostnames into urns (component ids)
- for node in resa['reserved_nodes']:
-
- iotlab_xrn = xrn_object(self.root_auth, node)
- resa['component_id_list'].append(iotlab_xrn.urn)
-
- if lease_filter_dict:
- logger.debug("IOTLAB_API \tGetLeases \
- \r\n leasefilter %s" % ( lease_filter_dict))
-
- filter_dict_functions = {
- 'slice_hrn' : IotlabShell.filter_lease_name,
- 't_from' : IotlabShell.filter_lease_start_time
- }
- reservation_list = list(unfiltered_reservation_list)
- for filter_type in lease_filter_dict:
- logger.debug("IOTLAB_API \tGetLeases reservation_list %s" \
- % (reservation_list))
- reservation_list = filter_dict_functions[filter_type](\
- reservation_list,lease_filter_dict[filter_type] )
-
- # Filter the reservation list with a maximum timespan so that the
- # leases and jobs running after this timestamp do not appear
- # in the result leases.
- # if 'start_time' in :
- # if resa['start_time'] < lease_filter_dict['start_time']:
- # reservation_list.append(resa)
-
-
- # if 'name' in lease_filter_dict and \
- # lease_filter_dict['name'] == resa['slice_hrn']:
- # reservation_list.append(resa)
-
-
- if lease_filter_dict is None:
- reservation_list = unfiltered_reservation_list
-
- self.leases_db.update_experiments_in_additional_sfa_db(job_oar_list, jobs_psql_id_list)
-
- logger.debug(" IOTLAB_API.PY \tGetLeases reservation_list %s"
- % (reservation_list))
- return reservation_list
#TODO FUNCTIONS SECTION 04/07/2012 SA
- ##TODO : Is UnBindObjectFromPeer still necessary ? Currently does nothing
- ##04/07/2012 SA
- #@staticmethod
- #def UnBindObjectFromPeer( auth, object_type, object_id, shortname):
- #""" This method is a hopefully temporary hack to let the sfa correctly
- #detach the objects it creates from a remote peer object. This is
- #needed so that the sfa federation link can work in parallel with
- #RefreshPeer, as RefreshPeer depends on remote objects being correctly
- #marked.
- #Parameters:
- #auth : struct, API authentication structure
- #AuthMethod : string, Authentication method to use
- #object_type : string, Object type, among 'site','person','slice',
- #'node','key'
- #object_id : int, object_id
- #shortname : string, peer shortname
- #FROM PLC DOC
-
- #"""
- #logger.warning("IOTLAB_API \tUnBindObjectFromPeer EMPTY-\
- #DO NOTHING \r\n ")
- #return
-
- ##TODO Is BindObjectToPeer still necessary ? Currently does nothing
- ##04/07/2012 SA
- #|| Commented out 28/05/13 SA
- #def BindObjectToPeer(self, auth, object_type, object_id, shortname=None, \
- #remote_object_id=None):
- #"""This method is a hopefully temporary hack to let the sfa correctly
- #attach the objects it creates to a remote peer object. This is needed
- #so that the sfa federation link can work in parallel with RefreshPeer,
- #as RefreshPeer depends on remote objects being correctly marked.
- #Parameters:
- #shortname : string, peer shortname
- #remote_object_id : int, remote object_id, set to 0 if unknown
- #FROM PLC API DOC
-
- #"""
- #logger.warning("IOTLAB_API \tBindObjectToPeer EMPTY - DO NOTHING \r\n ")
- #return
##TODO UpdateSlice 04/07/2012 SA || Commented out 28/05/13 SA
##Funciton should delete and create another job since oin iotlab slice=job
git://git.onelab.eu / sfa.git / blobdiff