From a86c3f7d62fd48a5322aeab21d45b84965ff9401 Mon Sep 17 00:00:00 2001 From: Sandrine Avakian Date: Wed, 24 Jul 2013 17:13:04 +0200 Subject: [PATCH] More cleaning and documenting. --- sfa/iotlab/LDAPapi.py | 39 +++-- sfa/iotlab/OARrestapi.py | 298 ++++++++++++++++++-------------- sfa/iotlab/iotlabaggregate.py | 10 +- sfa/iotlab/iotlabapi.py | 185 +++++++++++--------- sfa/iotlab/iotlabdriver.py | 315 +++++++++++++++++----------------- sfa/iotlab/iotlabpostgres.py | 147 ++++++++-------- sfa/iotlab/iotlabslices.py | 38 ++-- 7 files changed, 567 insertions(+), 465 deletions(-) diff --git a/sfa/iotlab/LDAPapi.py b/sfa/iotlab/LDAPapi.py index 284ca44a..709a554a 100644 --- a/sfa/iotlab/LDAPapi.py +++ b/sfa/iotlab/LDAPapi.py @@ -1,15 +1,26 @@ +""" +This API is adapted for OpenLDAP. +The file contains all LDAP classes and methods needed to: + - Load the LDAP connection configuration file (login, address..) with + LdapConfig + - Connect to LDAP with ldap_co + - Create a unique LDAP login and password for a user based on his email or + last name and first name with LoginPassword. + - Manage entries in LDAP using SFA records with LDAPapi + (Search, Add, Delete, Modify) + +""" import random from passlib.hash import ldap_salted_sha1 as lssha + from sfa.util.xrn import get_authority -import ldap +from sfa.util.sfalogging import logger from sfa.util.config import Config - +import ldap import ldap.modlist as modlist -from sfa.util.sfalogging import logger -import os.path -#API for OpenLDAP +import os.path class LdapConfig(): @@ -34,7 +45,7 @@ class LdapConfig(): self.config_path = os.path.dirname(config_file) except IOError: raise IOError, "Could not find or load the configuration file: %s" \ - % config_file + % config_file class ldap_co: @@ -86,7 +97,7 @@ class ldap_co: """ Binding method. :returns: dictionary with the bind status. True if Successful, - False if not and in this case the error message( {'bool', 'message'} ) + False if not and in this case the error message({'bool','message'}) :rtype: dict """ @@ -96,8 +107,8 @@ class ldap_co: # Bind/authenticate with a user with apropriate #rights to add objects - self.ldapserv.simple_bind_s(self.ldapAdminDN, \ - self.ldapAdminPassword) + self.ldapserv.simple_bind_s(self.ldapAdminDN, + self.ldapAdminPassword) except ldap.LDAPError, error: return {'bool': False, 'message': error} @@ -108,6 +119,9 @@ class ldap_co: """ Close the LDAP connection. Can throw an exception if the unbinding fails. + :returns: dictionary with the bind status if fails. + False if not and in this case the error message({'bool','message'}) + :rtype: dict or None """ try: @@ -378,12 +392,15 @@ class LDAPapi: try: login = \ lower_first_name[0:index] + \ - lower_last_name[0:self.login_pwd.login_max_length-index] + lower_last_name[0: + self.login_pwd.login_max_length + - index] login_filter = '(uid=' + login + ')' except KeyError: print "lower_first_name - lower_last_name too short" - logger.debug("LDAP.API \t LdapGenerateUniqueLogin login %s" % (login)) + logger.debug("LDAP.API \t LdapGenerateUniqueLogin login %s" + % (login)) return login except ldap.LDAPError, error: diff --git a/sfa/iotlab/OARrestapi.py b/sfa/iotlab/OARrestapi.py index 414dd5a8..18679080 100644 --- a/sfa/iotlab/OARrestapi.py +++ b/sfa/iotlab/OARrestapi.py @@ -1,9 +1,18 @@ +""" +File used to handle issuing request to OAR and parse OAR's JSON responses. +Contains the following classes: +- JsonPage : handles multiple pages OAR answers. +- OARRestapi : handles issuing POST or GET requests to OAR. +- ParsingResourcesFull : dedicated to parsing OAR's answer to a get resources +full request. +- OARGETParser : handles parsing the Json answers to different GET requests. + +""" from httplib import HTTPConnection, HTTPException, NotConnected import json from sfa.util.config import Config from sfa.util.sfalogging import logger import os.path -import sys class JsonPage: @@ -136,7 +145,7 @@ class OARrestapi: except IOError: raise IOError, "Could not find or load the configuration file: %s" \ - % config_file + % config_file #logger.setLevelDebug() self.oarserver['ip'] = self.OAR_IP self.oarserver['port'] = self.OAR_PORT @@ -149,7 +158,7 @@ class OARrestapi: def GETRequestToOARRestAPI(self, request, strval=None, - next_page=None, username=None): + next_page=None, username=None): """Makes a GET request to OAR. @@ -186,7 +195,7 @@ class OARrestapi: logger.debug("OARrestapi \tGETRequestToOARRestAPI %s" % (request)) if strval: self.oarserver['uri'] = self.oarserver['uri'].\ - replace("id",str(strval)) + replace("id", str(strval)) if next_page: self.oarserver['uri'] += next_page @@ -195,21 +204,21 @@ class OARrestapi: headers['X-REMOTE_IDENT'] = username logger.debug("OARrestapi: \t GETRequestToOARRestAPI \ - self.oarserver['uri'] %s strval %s" \ - %(self.oarserver['uri'], strval)) - try : + self.oarserver['uri'] %s strval %s" + % (self.oarserver['uri'], strval)) + try: #seems that it does not work if we don't add this headers['content-length'] = '0' - conn = HTTPConnection(self.oarserver['ip'], \ - self.oarserver['port']) + conn = HTTPConnection(self.oarserver['ip'], + self.oarserver['port']) conn.request("GET", self.oarserver['uri'], data, headers) - resp = ( conn.getresponse()).read() + resp = (conn.getresponse()).read() conn.close() - except HTTPException, error : - logger.log_exc("GET_OAR_SRVR : Problem with OAR server : %s " \ - %(error)) + except HTTPException, error: + logger.log_exc("GET_OAR_SRVR : Problem with OAR server : %s " + % (error)) #raise ServerError("GET_OAR_SRVR : Could not reach OARserver") try: js_dict = json.loads(resp) @@ -217,8 +226,8 @@ class OARrestapi: return js_dict except ValueError, error: - logger.log_exc("Failed to parse Server Response: %s ERROR %s"\ - %(js_dict, error)) + logger.log_exc("Failed to parse Server Response: %s ERROR %s" + % (js_dict, error)) #raise ServerError("Failed to parse Server Response:" + js) @@ -486,24 +495,14 @@ class ParsingResourcesFull(): class OARGETParser: - # resources_fulljson_dict = { - # 'network_address': AddNodeNetworkAddr, - # 'site': AddNodeSite, - # # 'radio': AddNodeRadio, - # 'mobile': AddMobility, - # 'x': AddPosX, - # 'y': AddPosY, - # 'z':AddPosZ, - # 'archi':AddHardwareType, - # 'state':AddBootState, - # 'id' : AddOarNodeId, - # } - - - def __init__(self, srv) : + """Class providing parsing methods associated to specific GET requests. + + """ + + def __init__(self, srv): self.version_json_dict = { - 'api_version' : None , 'apilib_version' :None,\ - 'api_timezone': None, 'api_timestamp': None, 'oar_version': None ,} + 'api_version': None, 'apilib_version': None, + 'api_timezone': None, 'api_timestamp': None, 'oar_version': None} self.config = Config() self.interface_hrn = self.config.SFA_INTERFACE_HRN self.timezone_json_dict = { @@ -519,13 +518,11 @@ class OARGETParser: self.json_page = JsonPage() self.parsing_resourcesfull = ParsingResourcesFull() self.site_dict = {} + self.jobs_list = [] self.SendRequest("GET_version") - - - - def ParseVersion(self) : + def ParseVersion(self): """Parses the OAR answer to the GET_version ( /oarapi/version.json.) Finds the OAR apilib version currently used. Has an impact on the json @@ -535,20 +532,20 @@ class OARGETParser: """ - if 'oar_version' in self.json_page.raw_json : - self.version_json_dict.update(api_version = - self.json_page.raw_json['api_version'], - apilib_version=self.json_page.raw_json['apilib_version'], - api_timezone=self.json_page.raw_json['api_timezone'], - api_timestamp=self.json_page.raw_json['api_timestamp'], - oar_version=self.json_page.raw_json['oar_version']) + if 'oar_version' in self.json_page.raw_json: + self.version_json_dict.update( + api_version=self.json_page.raw_json['api_version'], + apilib_version=self.json_page.raw_json['apilib_version'], + api_timezone=self.json_page.raw_json['api_timezone'], + api_timestamp=self.json_page.raw_json['api_timestamp'], + oar_version=self.json_page.raw_json['oar_version']) else: - self.version_json_dict.update(api_version = - self.json_page.raw_json['api'], - apilib_version=self.json_page.raw_json['apilib'], - api_timezone=self.json_page.raw_json['api_timezone'], - api_timestamp=self.json_page.raw_json['api_timestamp'], - oar_version=self.json_page.raw_json['oar']) + self.version_json_dict.update( + api_version=self.json_page.raw_json['api'], + apilib_version=self.json_page.raw_json['apilib'], + api_timezone=self.json_page.raw_json['api_timezone'], + api_timestamp=self.json_page.raw_json['api_timestamp'], + oar_version=self.json_page.raw_json['oar']) print self.version_json_dict['apilib_version'] @@ -616,17 +613,19 @@ class OARGETParser: """ job_resources = ['wanted_resources', 'name', 'id', 'start_time', - 'state', 'owner', 'walltime', 'message'] - + 'state', 'owner', 'walltime', 'message'] - job_resources_full = ['launching_directory', 'links', + # Unused variable providing the contents of the json dict returned from + # get job resources full request + job_resources_full = [ + 'launching_directory', 'links', 'resubmit_job_id', 'owner', 'events', 'message', - 'scheduled_start', 'id', 'array_id', 'exit_code', - 'properties', 'state','array_index', 'walltime', + 'scheduled_start', 'id', 'array_id', 'exit_code', + 'properties', 'state', 'array_index', 'walltime', 'type', 'initial_request', 'stop_time', 'project', - 'start_time', 'dependencies','api_timestamp','submission_time', + 'start_time', 'dependencies', 'api_timestamp', 'submission_time', 'reservation', 'stdout_file', 'types', 'cpuset_name', - 'name', 'wanted_resources','queue','stderr_file','command'] + 'name', 'wanted_resources', 'queue', 'stderr_file', 'command'] job_info = self.json_page.raw_json @@ -654,7 +653,7 @@ class OARGETParser: return job_resources - def ParseResources(self) : + def ParseResources(self): """ Parses the json produced by a get_resources request on oar.""" #logger.debug("OARESTAPI \tParseResources " ) @@ -663,31 +662,38 @@ class OARGETParser: self.ParseNodes() def ParseReservedNodes(self): - """ Returns an array containing the list of the reserved nodes """ + """ Returns an array containing the list of the jobs scheduled + with the reserved nodes if available. + + :returns: list of job dicts, each dict containing the following keys: + t_from, t_until, resources_ids (of the reserved nodes for this job). + If the information is not available, default values will be set for + these keys. The other keys are : state, lease_id and user. + :rtype: list + + """ #resources are listed inside the 'items' list from the json reservation_list = [] job = {} #Parse resources info - for json_element in self.json_page.raw_json['items']: + for json_element in self.json_page.raw_json['items']: #In case it is a real reservation (not asap case) if json_element['scheduled_start']: job['t_from'] = json_element['scheduled_start'] job['t_until'] = int(json_element['scheduled_start']) + \ - int(json_element['walltime']) + int(json_element['walltime']) #Get resources id list for the job - job['resource_ids'] = [ node_dict['id'] for node_dict - in json_element['resources']] + job['resource_ids'] = [node_dict['id'] for node_dict + in json_element['resources']] else: job['t_from'] = "As soon as possible" job['t_until'] = "As soon as possible" job['resource_ids'] = ["Undefined"] - job['state'] = json_element['state'] job['lease_id'] = json_element['id'] - job['user'] = json_element['owner'] #logger.debug("OARRestapi \tParseReservedNodes job %s" %(job)) reservation_list.append(job) @@ -699,16 +705,31 @@ class OARGETParser: """ Gets the list of nodes currently in use from the attributes of the running jobs. + :returns: list of hostnames, the nodes that are currently involved in + running jobs. + :rtype: list + + """ - logger.debug("OARESTAPI \tParseRunningJobs__________________________ ") + logger.debug("OARESTAPI \tParseRunningJobs_________________ ") #resources are listed inside the 'items' list from the json nodes = [] - for job in self.json_page.raw_json['items']: + for job in self.json_page.raw_json['items']: for node in job['nodes']: nodes.append(node['network_address']) return nodes def ChangeRawJsonDependingOnApilibVersion(self): + """ + Check if the OAR apilib version is different from 0.2.10, in which case + the Json answer is also dict instead as a plain list. + + .. warning:: the whole code is assuming the json contains a 'items' key + .. seealso:: ConcatenateJsonPages, ParseJobs, ParseReservedNodes, + ParseJobsIdResources, ParseResources, ParseRunningJobs + .. todo:: Clean the whole code. Either suppose the apilib will always + provide the 'items' key, or handle different options. + """ if self.version_json_dict['apilib_version'] != "0.2.10": self.json_page.raw_json = self.json_page.raw_json['items'] @@ -720,15 +741,19 @@ class OARGETParser: """ return - def ParseResourcesFull(self) : + def ParseResourcesFull(self): """ This method is responsible for parsing all the attributes of all the nodes returned by OAR when issuing a get resources full. The information from the nodes and the sites are separated. Updates the node_dictlist so that the dictionnary of the platform's nodes is available afterwards. + :returns: node_dictlist, a list of dictionaries about the nodes and + their properties. + :rtype: list + """ - logger.debug("OARRESTAPI ParseResourcesFull________________________ ") + logger.debug("OARRESTAPI ParseResourcesFull___________ ") #print self.json_page.raw_json[1] #resources are listed inside the 'items' list from the json self.ChangeRawJsonDependingOnApilibVersion() @@ -736,9 +761,16 @@ class OARGETParser: self.ParseSites() return self.node_dictlist - def ParseResourcesFullSites(self) : - """ UNUSED. Originally used to get information from the sites. - ParseResourcesFull is used instead. + def ParseResourcesFullSites(self): + """ Called by GetSites which is unused. + Originally used to get information from the sites, with for each site + the list of nodes it has, along with their properties. + + :return: site_dict, dictionary of sites + :rtype: dict + + .. warning:: unused + .. seealso:: GetSites (IotlabTestbedAPI) """ self.ChangeRawJsonDependingOnApilibVersion() @@ -762,15 +794,15 @@ class OARGETParser: for dictline in self.json_page.raw_json: node_id = None # dictionary is empty and/or a new node has to be inserted - node_id = _resources_fulljson_dict['network_address'](\ - self.node_dictlist, dictline['network_address']) + node_id = _resources_fulljson_dict['network_address']( + self.node_dictlist, dictline['network_address']) for k in keys: if k in dictline: if k == 'network_address': continue - _resources_fulljson_dict[k](\ - self.node_dictlist[node_id], dictline[k]) + _resources_fulljson_dict[k]( + self.node_dictlist[node_id], dictline[k]) #The last property has been inserted in the property tuple list, #reset node_id @@ -794,8 +826,6 @@ class OARGETParser: """ return root_auth + '.' + hostname - - def ParseSites(self): """ Returns a list of dictionnaries containing the sites' attributes.""" @@ -805,7 +835,7 @@ class OARGETParser: #%(self.node_dictlist)) # Create a list of nodes per site_id for node_id in self.node_dictlist: - node = self.node_dictlist[node_id] + node = self.node_dictlist[node_id] if node['site'] not in nodes_per_site: nodes_per_site[node['site']] = [] @@ -818,94 +848,106 @@ class OARGETParser: # (name of the site) and value is a dictionary of properties, # including the list of the node_ids for node_id in self.node_dictlist: - node = self.node_dictlist[node_id] - node.update({'hrn':self.iotlab_hostname_to_hrn(self.interface_hrn, - node['hostname'])}) - self.node_dictlist.update({node_id:node}) + node = self.node_dictlist[node_id] + node.update({'hrn': self.iotlab_hostname_to_hrn(self.interface_hrn, + node['hostname'])}) + self.node_dictlist.update({node_id: node}) if node['site'] not in self.site_dict: self.site_dict[node['site']] = { - 'site':node['site'], - 'node_ids':nodes_per_site[node['site']], - 'latitude':"48.83726", - 'longitude':"- 2.10336", + 'site': node['site'], + 'node_ids': nodes_per_site[node['site']], + 'latitude': "48.83726", + 'longitude': "- 2.10336", 'name': config.SFA_REGISTRY_ROOT_AUTH, - 'pcu_ids':[], 'max_slices':None, 'ext_consortium_id':None, - 'max_slivers':None, 'is_public':True, 'peer_site_id': None, - 'abbreviated_name':"iotlab", 'address_ids': [], - 'url':"http,//www.senslab.info", 'person_ids':[], - 'site_tag_ids':[], 'enabled': True, 'slice_ids':[], - 'date_created': None, 'peer_id': None } - + 'pcu_ids': [], 'max_slices': None, + 'ext_consortium_id': None, + 'max_slivers': None, 'is_public': True, + 'peer_site_id': None, + 'abbreviated_name': "iotlab", 'address_ids': [], + 'url': "https://portal.senslab.info", 'person_ids': [], + 'site_tag_ids': [], 'enabled': True, 'slice_ids': [], + 'date_created': None, 'peer_id': None + } OARrequests_uri_dict = { 'GET_version': - {'uri':'/oarapi/version.json', 'parse_func': ParseVersion}, + {'uri': '/oarapi/version.json', 'parse_func': ParseVersion}, + 'GET_timezone': - {'uri':'/oarapi/timezone.json' ,'parse_func': ParseTimezone }, + {'uri': '/oarapi/timezone.json', 'parse_func': ParseTimezone}, + 'GET_jobs': - {'uri':'/oarapi/jobs.json','parse_func': ParseJobs}, + {'uri': '/oarapi/jobs.json', 'parse_func': ParseJobs}, + 'GET_jobs_id': - {'uri':'/oarapi/jobs/id.json','parse_func': ParseJobsIds}, + {'uri': '/oarapi/jobs/id.json', 'parse_func': ParseJobsIds}, + 'GET_jobs_id_resources': - {'uri':'/oarapi/jobs/id/resources.json',\ - 'parse_func': ParseJobsIdResources}, + {'uri': '/oarapi/jobs/id/resources.json', + 'parse_func': ParseJobsIdResources}, + 'GET_jobs_table': - {'uri':'/oarapi/jobs/table.json','parse_func': ParseJobsTable}, + {'uri': '/oarapi/jobs/table.json', 'parse_func': ParseJobsTable}, + 'GET_jobs_details': - {'uri':'/oarapi/jobs/details.json',\ - 'parse_func': ParseJobsDetails}, - 'GET_reserved_nodes': - {'uri': - '/oarapi/jobs/details.json?state=Running,Waiting,Launching',\ - 'owner':'&user=', - 'parse_func':ParseReservedNodes}, + {'uri': '/oarapi/jobs/details.json', 'parse_func': ParseJobsDetails}, + 'GET_reserved_nodes': + {'uri': + '/oarapi/jobs/details.json?state=Running,Waiting,Launching', + 'owner': '&user=', 'parse_func': ParseReservedNodes}, 'GET_running_jobs': - {'uri':'/oarapi/jobs/details.json?state=Running',\ - 'parse_func':ParseRunningJobs}, + {'uri': '/oarapi/jobs/details.json?state=Running', + 'parse_func': ParseRunningJobs}, + 'GET_resources_full': - {'uri':'/oarapi/resources/full.json',\ - 'parse_func': ParseResourcesFull}, + {'uri': '/oarapi/resources/full.json', + 'parse_func': ParseResourcesFull}, + 'GET_sites': - {'uri':'/oarapi/resources/full.json',\ - 'parse_func': ParseResourcesFullSites}, - 'GET_resources': - {'uri':'/oarapi/resources.json' ,'parse_func': ParseResources}, - 'DELETE_jobs_id': - {'uri':'/oarapi/jobs/id.json' ,'parse_func': ParseDeleteJobs} - } + {'uri': '/oarapi/resources/full.json', + 'parse_func': ParseResourcesFullSites}, + 'GET_resources': + {'uri': '/oarapi/resources.json', 'parse_func': ParseResources}, + 'DELETE_jobs_id': + {'uri': '/oarapi/jobs/id.json', 'parse_func': ParseDeleteJobs}} - def SendRequest(self, request, strval = None , username = None): + def SendRequest(self, request, strval=None, username=None): """ Connects to OAR , sends the valid GET requests and uses the appropriate json parsing functions. + :returns: calls to the appropriate parsing function, associated with the + GET request + :rtype: depends on the parsing function called. + + .. seealso:: OARrequests_uri_dict """ save_json = None self.json_page.ResetNextPage() save_json = [] - if request in self.OARrequests_uri_dict : + if request in self.OARrequests_uri_dict: while self.json_page.next_page: - self.json_page.raw_json = self.server.GETRequestToOARRestAPI(\ - request, \ - strval, \ - self.json_page.next_offset, \ - username) + self.json_page.raw_json = self.server.GETRequestToOARRestAPI( + request, + strval, + self.json_page.next_offset, + username) self.json_page.FindNextPage() if self.json_page.concatenate: save_json.append(self.json_page.raw_json) - if self.json_page.concatenate and self.json_page.end : + if self.json_page.concatenate and self.json_page.end: self.json_page.raw_json = \ self.json_page.ConcatenateJsonPages(save_json) return self.OARrequests_uri_dict[request]['parse_func'](self) else: - logger.error("OARRESTAPI OARGetParse __init__ : ERROR_REQUEST " \ - %(request)) + logger.error("OARRESTAPI OARGetParse __init__ : ERROR_REQUEST " + % (request)) diff --git a/sfa/iotlab/iotlabaggregate.py b/sfa/iotlab/iotlabaggregate.py index 86b14056..77859e59 100644 --- a/sfa/iotlab/iotlabaggregate.py +++ b/sfa/iotlab/iotlabaggregate.py @@ -1,3 +1,8 @@ +""" +File providing methods to generate valid RSpecs for the Iotlab testbed. +Contains methods to get information on slice, slivers, nodes and leases, +formatting them and turn it into a RSpec. +""" from sfa.util.xrn import hrn_to_urn, urn_to_hrn, get_authority from sfa.rspecs.rspec import RSpec @@ -140,7 +145,6 @@ class IotlabAggregate: return (slices, slivers) - def get_nodes(self, slices=None, slivers=[], options=None): """Returns the nodes in the slice using the rspec format, with all the nodes' properties. @@ -286,8 +290,8 @@ class IotlabAggregate: :rtype: dict .. note::There is no filtering of leases within a given time frame. - All the running or scheduled leases are returned. options - removed SA 15/05/2013 + All the running or scheduled leases are returned. options + removed SA 15/05/2013 """ diff --git a/sfa/iotlab/iotlabapi.py b/sfa/iotlab/iotlabapi.py index fb8541d4..cbf7b680 100644 --- a/sfa/iotlab/iotlabapi.py +++ b/sfa/iotlab/iotlabapi.py @@ -1,3 +1,9 @@ +""" +File containing the IotlabTestbedAPI, used to interact with nodes, users, +slices, leases and keys, as well as the dedicated iotlab database and table, +holding information about which slice is running which job. + +""" from datetime import datetime from sfa.util.sfalogging import logger @@ -5,9 +11,8 @@ from sfa.util.sfalogging import logger from sfa.storage.alchemy import dbsession from sqlalchemy.orm import joinedload from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey -# from sfa.iotlab.iotlabpostgres import iotlab_dbsession, IotlabXP from sfa.iotlab.iotlabpostgres import IotlabDB, IotlabXP -from sfa.iotlab.OARrestapi import OARrestapi +from sfa.iotlab.OARrestapi import OARrestapi from sfa.iotlab.LDAPapi import LDAPapi from sfa.util.xrn import Xrn, hrn_to_urn, get_authority @@ -21,6 +26,8 @@ from sfa.iotlab.iotlabaggregate import iotlab_xrn_object class IotlabTestbedAPI(): """ Class enabled to use LDAP and OAR api calls. """ + _MINIMUM_DURATION = 600 + def __init__(self, config): """Creates an instance of OARrestapi and LDAPapi which will be used to issue calls to OAR or LDAP methods. @@ -35,17 +42,20 @@ class IotlabTestbedAPI(): self.ldap = LDAPapi() self.time_format = "%Y-%m-%d %H:%M:%S" self.root_auth = config.SFA_REGISTRY_ROOT_AUTH - self.grain = 1 # 10 mins lease minimum, 1 sec granularity + self.grain = 1 # 10 mins lease minimum, 1 sec granularity #import logging, logging.handlers #from sfa.util.sfalogging import _SfaLogger #sql_logger = _SfaLogger(loggername = 'sqlalchemy.engine', \ #level=logging.DEBUG) return - @staticmethod def GetMinExperimentDurationInSec(): - return 600 + """ Returns the minimum allowed duration for an experiment on the + testbed. In seconds. + + """ + return IotlabTestbedAPI._MINIMUM_DURATION @staticmethod def GetPeers (peer_filter=None ): @@ -162,8 +172,8 @@ class IotlabTestbedAPI(): :rtype: dict """ - logger.debug("IOTLABDRIVER \tDeleteJobs jobid %s username %s "\ - %(job_id, username)) + logger.debug("IOTLABDRIVER \tDeleteJobs jobid %s username %s " + % (job_id, username)) if not job_id or job_id is -1: return @@ -171,15 +181,14 @@ class IotlabTestbedAPI(): reqdict['method'] = "delete" reqdict['strval'] = str(job_id) - - answer = self.oar.POSTRequestToOARRestAPI('DELETE_jobs_id', \ - reqdict,username) + answer = self.oar.POSTRequestToOARRestAPI('DELETE_jobs_id', + reqdict, username) if answer['status'] == 'Delete request registered': - ret = {job_id : True } + ret = {job_id: True} else: - ret = {job_id :False } + ret = {job_id: False} logger.debug("IOTLABDRIVER \tDeleteJobs jobid %s \r\n answer %s \ - username %s" %(job_id, answer, username)) + username %s" % (job_id, answer, username)) return ret @@ -416,17 +425,29 @@ class IotlabTestbedAPI(): dbsession.add(sfa_record) dbsession.commit() #Update the reg-researcher dependance table - sfa_record.reg_researchers = [user_record] + sfa_record.reg_researchers = [user_record] dbsession.commit() return - def GetSites(self, site_filter_name_list = None, return_fields_list = None): + def GetSites(self, site_filter_name_list=None, return_fields_list=None): + """Returns the list of Iotlab's sites with the associated nodes and + their properties as dictionaries. + + Uses the OAR request GET_sites to find the Iotlab's sites. + + :param site_filter_name_list: used to specify specific sites + :param return_fields_list: field that has to be returned + :type site_filter_name_list: list + :type return_fields_list: list + + .. warning:: unused + """ site_dict = self.oar.parser.SendRequest("GET_sites") #site_dict : dict where the key is the sit ename return_site_list = [] - if not ( site_filter_name_list or return_fields_list): + if not (site_filter_name_list or return_fields_list): return_site_list = site_dict.values() return return_site_list @@ -438,19 +459,15 @@ class IotlabTestbedAPI(): try: tmp[field] = site_dict[site_filter_name][field] except KeyError: - logger.error("GetSites KeyError %s "%(field)) + logger.error("GetSites KeyError %s " % (field)) return None return_site_list.append(tmp) else: - return_site_list.append( site_dict[site_filter_name]) - + return_site_list.append(site_dict[site_filter_name]) return return_site_list - - - #TODO : Check rights to delete person def DeletePerson(self, person_record): """Disable an existing account in iotlab LDAP. @@ -839,7 +856,8 @@ class IotlabTestbedAPI(): :rtype: dict """ - logger.debug("IOTLABDRIVER \t DeleteSliceFromNodese %s " %(slice_record)) + logger.debug("IOTLABDRIVER \t DeleteSliceFromNodes %s " + % (slice_record)) if isinstance(slice_record['oar_job_id'], list): oar_bool_answer = {} @@ -849,8 +867,8 @@ class IotlabTestbedAPI(): oar_bool_answer.update(ret) else: - oar_bool_answer = [self.DeleteJobs(slice_record['oar_job_id'], \ - slice_record['user'])] + oar_bool_answer = [self.DeleteJobs(slice_record['oar_job_id'], + slice_record['user'])] return oar_bool_answer @@ -890,7 +908,6 @@ class IotlabTestbedAPI(): # return - def GetLeases(self, lease_filter_dict=None, login=None): """ @@ -915,38 +932,38 @@ class IotlabTestbedAPI(): reservation_list = [] #Find the slice associated with this user iotlab ldap uid logger.debug(" IOTLABDRIVER.PY \tGetLeases login %s\ - unfiltered_reservation_list %s " %(login, unfiltered_reservation_list)) + unfiltered_reservation_list %s " + % (login, unfiltered_reservation_list)) #Create user dict first to avoid looking several times for #the same user in LDAP SA 27/07/12 job_oar_list = [] jobs_psql_query = self.iotlab_db.iotlab_session.query(IotlabXP).all() - jobs_psql_dict = dict([(row.job_id, row.__dict__ ) for row in jobs_psql_query ]) + jobs_psql_dict = dict([(row.job_id, row.__dict__) + for row in jobs_psql_query]) #jobs_psql_dict = jobs_psql_dict) - logger.debug("IOTLABDRIVER \tGetLeases jobs_psql_dict %s"\ - %(jobs_psql_dict)) - jobs_psql_id_list = [ row.job_id for row in jobs_psql_query ] - - + logger.debug("IOTLABDRIVER \tGetLeases jobs_psql_dict %s" + % (jobs_psql_dict)) + jobs_psql_id_list = [row.job_id for row in jobs_psql_query] for resa in unfiltered_reservation_list: - logger.debug("IOTLABDRIVER \tGetLeases USER %s"\ - %(resa['user'])) + logger.debug("IOTLABDRIVER \tGetLeases USER %s" + % (resa['user'])) #Construct list of jobs (runing, waiting..) in oar job_oar_list.append(resa['lease_id']) #If there is information on the job in IOTLAB DB ] #(slice used and job id) if resa['lease_id'] in jobs_psql_dict: job_info = jobs_psql_dict[resa['lease_id']] - logger.debug("IOTLABDRIVER \tGetLeases job_info %s"\ - %(job_info)) + logger.debug("IOTLABDRIVER \tGetLeases job_info %s" + % (job_info)) resa['slice_hrn'] = job_info['slice_hrn'] resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice') #otherwise, assume it is a iotlab slice: else: - resa['slice_id'] = hrn_to_urn(self.root_auth+'.'+ \ - resa['user'] +"_slice" , 'slice') + resa['slice_id'] = hrn_to_urn(self.root_auth + '.' + + resa['user'] + "_slice", 'slice') resa['slice_hrn'] = Xrn(resa['slice_id']).get_hrn() resa['component_id_list'] = [] @@ -958,7 +975,7 @@ class IotlabTestbedAPI(): if lease_filter_dict: logger.debug("IOTLABDRIVER \tGetLeases resa_ %s \ - \r\n leasefilter %s" %(resa, lease_filter_dict)) + \r\n leasefilter %s" % (resa, lease_filter_dict)) if lease_filter_dict['name'] == resa['slice_hrn']: reservation_list.append(resa) @@ -966,11 +983,10 @@ class IotlabTestbedAPI(): if lease_filter_dict is None: reservation_list = unfiltered_reservation_list - self.iotlab_db.update_jobs_in_iotlabdb(job_oar_list, jobs_psql_id_list) - logger.debug(" IOTLABDRIVER.PY \tGetLeases reservation_list %s"\ - %(reservation_list)) + logger.debug(" IOTLABDRIVER.PY \tGetLeases reservation_list %s" + % (reservation_list)) return reservation_list @@ -1075,14 +1091,15 @@ class IotlabTestbedAPI(): key_dict = {} for key in keys: - key_dict[key.key] = {'key_id': key.key_id, 'key': key.key, \ - 'email': key.reg_user.email, 'hrn':key.reg_user.hrn} + key_dict[key.key] = {'key_id': key.key_id, 'key': key.key, + 'email': key.reg_user.email, + 'hrn': key.reg_user.hrn} #ldap_rslt = self.ldap.LdapSearch({'enabled']=True}) #user_by_email = dict((user[1]['mail'][0], user[1]['sshPublicKey']) \ #for user in ldap_rslt) - logger.debug("IOTLABDRIVER GetKeys -key_dict %s \r\n " %(key_dict)) + logger.debug("IOTLABDRIVER GetKeys -key_dict %s \r\n " % (key_dict)) return key_dict #TODO : test @@ -1102,9 +1119,9 @@ class IotlabTestbedAPI(): """ all_user_keys = user_record['keys'] all_user_keys.remove(key_string) - new_attributes = {'sshPublicKey':all_user_keys} + new_attributes = {'sshPublicKey':all_user_keys} ret = self.ldap.LdapModifyUser(user_record, new_attributes) - logger.debug("IOTLABDRIVER DeleteKey %s- "%(ret)) + logger.debug("IOTLABDRIVER DeleteKey %s- " % (ret)) return ret['bool'] @@ -1130,14 +1147,14 @@ class IotlabTestbedAPI(): #Only one entry for one user = one slice in iotlab_xp table #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() - raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn = slice_filter).first() + raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first() #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() if raw_slicerec: #load_reg_researcher #raw_slicerec.reg_researchers raw_slicerec = raw_slicerec.__dict__ logger.debug(" IOTLABDRIVER \t get_slice_info slice_filter %s \ - raw_slicerec %s"%(slice_filter, raw_slicerec)) + raw_slicerec %s" % (slice_filter, raw_slicerec)) slicerec = raw_slicerec #only one researcher per slice so take the first one #slicerec['reg_researchers'] = raw_slicerec['reg_researchers'] @@ -1148,7 +1165,7 @@ class IotlabTestbedAPI(): return None @staticmethod - def _sql_get_slice_info_from_user(slice_filter ): + def _sql_get_slice_info_from_user(slice_filter): """ Get the slice record based on the user recordid by using a joinedload on the relationship reg_slices_as_researcher. Format the sql record @@ -1158,20 +1175,25 @@ class IotlabTestbedAPI(): :rtype:dict or None.. """ #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first() - raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id = slice_filter).first() + raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id=slice_filter).first() #raw_slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first() #Put it in correct order - user_needed_fields = ['peer_authority', 'hrn', 'last_updated', 'classtype', 'authority', 'gid', 'record_id', 'date_created', 'type', 'email', 'pointer'] - slice_needed_fields = ['peer_authority', 'hrn', 'last_updated', 'classtype', 'authority', 'gid', 'record_id', 'date_created', 'type', 'pointer'] + user_needed_fields = ['peer_authority', 'hrn', 'last_updated', + 'classtype', 'authority', 'gid', 'record_id', + 'date_created', 'type', 'email', 'pointer'] + slice_needed_fields = ['peer_authority', 'hrn', 'last_updated', + 'classtype', 'authority', 'gid', 'record_id', + 'date_created', 'type', 'pointer'] if raw_slicerec: #raw_slicerec.reg_slices_as_researcher raw_slicerec = raw_slicerec.__dict__ slicerec = {} slicerec = \ - dict([(k, raw_slicerec['reg_slices_as_researcher'][0].__dict__[k]) \ - for k in slice_needed_fields]) - slicerec['reg_researchers'] = dict([(k, raw_slicerec[k]) \ - for k in user_needed_fields]) + dict([(k, raw_slicerec[ + 'reg_slices_as_researcher'][0].__dict__[k]) + for k in slice_needed_fields]) + slicerec['reg_researchers'] = dict([(k, raw_slicerec[k]) + for k in user_needed_fields]) #TODO Handle multiple slices for one user SA 10/12/12 #for now only take the first slice record associated to the rec user ##slicerec = raw_slicerec['reg_slices_as_researcher'][0].__dict__ @@ -1184,8 +1206,8 @@ class IotlabTestbedAPI(): else: return None - def _get_slice_records(self, slice_filter = None, \ - slice_filter_type = None): + def _get_slice_records(self, slice_filter=None, + slice_filter_type=None): """ Get the slice record depending on the slice filter and its type. :param slice_filter: Can be either the slice hrn or the user's record @@ -1209,7 +1231,7 @@ class IotlabTestbedAPI(): slicerec = self._sql_get_slice_info(slice_filter) if slicerec is None: - return None + return None #return login, None #Get slice based on user id @@ -1278,11 +1300,11 @@ class IotlabTestbedAPI(): #If several jobs for one slice , put the slice record into # each lease information dict - for lease in leases_list : + for lease in leases_list: slicerec_dict = {} logger.debug("IOTLABDRIVER.PY \tGetSlices slice_filter %s \ - \ lease['slice_hrn'] %s" - % (slice_filter, lease['slice_hrn'])) + \t lease['slice_hrn'] %s" + % (slice_filter, lease['slice_hrn'])) if lease['slice_hrn'] == slice_hrn: slicerec_dict['slice_hrn'] = lease['slice_hrn'] slicerec_dict['hrn'] = lease['slice_hrn'] @@ -1290,32 +1312,31 @@ class IotlabTestbedAPI(): slicerec_dict['oar_job_id'] = lease['lease_id'] slicerec_dict.update( {'list_node_ids': - {'hostname': lease['reserved_nodes']} - }) + {'hostname': lease['reserved_nodes']}}) slicerec_dict.update({'node_ids': lease['reserved_nodes']}) #Update lease dict with the slice record if fixed_slicerec_dict: fixed_slicerec_dict['oar_job_id'] = [] fixed_slicerec_dict['oar_job_id'].append( - slicerec_dict['oar_job_id']) + slicerec_dict['oar_job_id']) slicerec_dict.update(fixed_slicerec_dict) #slicerec_dict.update({'hrn':\ #str(fixed_slicerec_dict['slice_hrn'])}) return_slicerec_dictlist.append(slicerec_dict) logger.debug("IOTLABDRIVER.PY \tGetSlices \ - OHOHOHOH %s" %(return_slicerec_dictlist )) + OHOHOHOH %s" %(return_slicerec_dictlist)) logger.debug("IOTLABDRIVER.PY \tGetSlices \ slicerec_dict %s return_slicerec_dictlist %s \ lease['reserved_nodes'] \ - %s" %(slicerec_dict, return_slicerec_dictlist, \ - lease['reserved_nodes'] )) + %s" % (slicerec_dict, return_slicerec_dictlist, + lease['reserved_nodes'])) logger.debug("IOTLABDRIVER.PY \tGetSlices RETURN \ - return_slicerec_dictlist %s" \ - %(return_slicerec_dictlist)) + return_slicerec_dictlist %s" + % (return_slicerec_dictlist)) return return_slicerec_dictlist @@ -1324,7 +1345,8 @@ class IotlabTestbedAPI(): #Get all slices from the iotlab sfa database , #put them in dict format #query_slice_list = dbsession.query(RegRecord).all() - query_slice_list = dbsession.query(RegSlice).options(joinedload('reg_researchers')).all() + query_slice_list = \ + dbsession.query(RegSlice).options(joinedload('reg_researchers')).all() for record in query_slice_list: tmp = record.__dict__ @@ -1336,12 +1358,12 @@ class IotlabTestbedAPI(): #Get all the jobs reserved nodes leases_list = self.GetReservedNodes() - for fixed_slicerec_dict in return_slicerec_dictlist: slicerec_dict = {} #Check if the slice belongs to a iotlab user if fixed_slicerec_dict['peer_authority'] is None: - owner = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0] + owner = fixed_slicerec_dict['hrn'].split( + ".")[1].split("_")[0] else: owner = None for lease in leases_list: @@ -1349,11 +1371,14 @@ class IotlabTestbedAPI(): slicerec_dict['oar_job_id'] = lease['lease_id'] #for reserved_node in lease['reserved_nodes']: - logger.debug("IOTLABDRIVER.PY \tGetSlices lease %s "\ - %(lease )) - - slicerec_dict.update({'node_ids':lease['reserved_nodes']}) - slicerec_dict.update({'list_node_ids':{'hostname':lease['reserved_nodes']}}) + logger.debug("IOTLABDRIVER.PY \tGetSlices lease %s " + % (lease)) + + slicerec_dict.update({'node_ids': + lease['reserved_nodes']}) + slicerec_dict.update({'list_node_ids': + {'hostname': + lease['reserved_nodes']}}) slicerec_dict.update(fixed_slicerec_dict) #slicerec_dict.update({'hrn':\ #str(fixed_slicerec_dict['slice_hrn'])}) diff --git a/sfa/iotlab/iotlabdriver.py b/sfa/iotlab/iotlabdriver.py index 2d768edd..04efeef6 100644 --- a/sfa/iotlab/iotlabdriver.py +++ b/sfa/iotlab/iotlabdriver.py @@ -1,3 +1,6 @@ +""" +Implements what a driver should provide for SFA to work. +""" from sfa.util.faults import SliverDoesNotExist, UnknownSfaType from sfa.util.sfalogging import logger from sfa.storage.alchemy import dbsession @@ -9,9 +12,6 @@ from sfa.rspecs.rspec import RSpec from sfa.util.xrn import Xrn, hrn_to_urn, get_authority - -from sfa.iotlab.iotlabpostgres import IotlabDB - from sfa.iotlab.iotlabaggregate import IotlabAggregate, iotlab_xrn_to_hostname from sfa.iotlab.iotlabslices import IotlabSlices @@ -38,14 +38,12 @@ class IotlabDriver(Driver): :type config: Config object """ - Driver.__init__ (self, config) + Driver.__init__(self, config) self.config = config - - # self.db = IotlabDB(config, debug = False) self.iotlab_api = IotlabTestbedAPI(config) self.cache = None - def augment_records_with_testbed_info (self, record_list ): + def augment_records_with_testbed_info(self, record_list): """ Adds specific testbed info to the records. @@ -56,7 +54,7 @@ class IotlabDriver(Driver): :rtype: list """ - return self.fill_record_info (record_list) + return self.fill_record_info(record_list) def fill_record_info(self, record_list): """ @@ -87,22 +85,23 @@ class IotlabDriver(Driver): #information is in the Iotlab's DB. if str(record['type']) == 'slice': if 'reg_researchers' in record and \ - isinstance(record['reg_researchers'], list) : + isinstance(record['reg_researchers'], list): record['reg_researchers'] = \ record['reg_researchers'][0].__dict__ - record.update({'PI':[record['reg_researchers']['hrn']], - 'researcher': [record['reg_researchers']['hrn']], - 'name':record['hrn'], - 'oar_job_id':[], - 'node_ids': [], - 'person_ids': [record['reg_researchers'] - ['record_id']], - # For client_helper.py compatibility - 'geni_urn': '', - # For client_helper.py compatibility - 'keys': '', - # For client_helper.py compatibility - 'key_ids': ''}) + record.update( + {'PI': [record['reg_researchers']['hrn']], + 'researcher': [record['reg_researchers']['hrn']], + 'name': record['hrn'], + 'oar_job_id': [], + 'node_ids': [], + 'person_ids': [record['reg_researchers'] + ['record_id']], + # For client_helper.py compatibility + 'geni_urn': '', + # For client_helper.py compatibility + 'keys': '', + # For client_helper.py compatibility + 'key_ids': ''}) #Get iotlab slice record and oar job id if any. recslice_list = self.iotlab_api.GetSlices( @@ -120,8 +119,8 @@ class IotlabDriver(Driver): % (rec['oar_job_id'])) record['node_ids'] = [self.iotlab_api.root_auth + - hostname for hostname in - rec['node_ids']] + hostname for hostname in + rec['node_ids']] except KeyError: pass @@ -145,52 +144,49 @@ class IotlabDriver(Driver): #Will update PIs and researcher for the slice recuser = recslice_list[0]['reg_researchers'] - logger.debug( "IOTLABDRIVER.PY \t fill_record_info USER \ + logger.debug("IOTLABDRIVER.PY \t fill_record_info USER \ recuser %s \r\n \r\n" % (recuser)) recslice = {} recslice = recslice_list[0] - recslice.update({'PI':[recuser['hrn']], - 'researcher': [recuser['hrn']], - 'name':record['hrn'], - 'node_ids': [], - 'oar_job_id': [], - 'person_ids':[recuser['record_id']]}) + recslice.update( + {'PI': [recuser['hrn']], + 'researcher': [recuser['hrn']], + 'name': record['hrn'], + 'node_ids': [], + 'oar_job_id': [], + 'person_ids': [recuser['record_id']]}) try: for rec in recslice_list: recslice['oar_job_id'].append(rec['oar_job_id']) except KeyError: pass - recslice.update({'type':'slice', \ - 'hrn':recslice_list[0]['hrn']}) - + recslice.update({'type': 'slice', + 'hrn': recslice_list[0]['hrn']}) #GetPersons takes [] as filters user_iotlab = self.iotlab_api.GetPersons([record]) - record.update(user_iotlab[0]) #For client_helper.py compatibility - record.update( { 'geni_urn':'', - 'keys':'', - 'key_ids':'' }) + record.update( + {'geni_urn': '', + 'keys': '', + 'key_ids': ''}) record_list.append(recslice) logger.debug("IOTLABDRIVER.PY \t \ fill_record_info ADDING SLICE\ INFO TO USER records %s" % (record_list)) - except TypeError, error: - logger.log_exc("IOTLABDRIVER \t fill_record_info EXCEPTION %s"\ - % (error)) + logger.log_exc("IOTLABDRIVER \t fill_record_info EXCEPTION %s" + % (error)) return record_list - def sliver_status(self, slice_urn, slice_hrn): """ - Receive a status request for slice named urn/hrn urn:publicid:IDN+iotlab+nturro_slice hrn iotlab.nturro_slice shall return a structure as described in @@ -204,7 +200,6 @@ class IotlabDriver(Driver): """ - #First get the slice with the slice hrn slice_list = self.iotlab_api.GetSlices(slice_filter=slice_hrn, slice_filter_type='slice_hrn') @@ -215,39 +210,31 @@ class IotlabDriver(Driver): #Used for fetching the user info witch comes along the slice info one_slice = slice_list[0] - #Make a list of all the nodes hostnames in use for this slice slice_nodes_list = [] - #for single_slice in slice_list: - #for node in single_slice['node_ids']: - #slice_nodes_list.append(node['hostname']) - #for node in one_slice: - #slice_nodes_list.append(node['hostname']) slice_nodes_list = one_slice['node_ids'] #Get all the corresponding nodes details - nodes_all = self.iotlab_api.GetNodes({'hostname':slice_nodes_list}, - ['node_id', 'hostname','site','boot_state']) - nodeall_byhostname = dict([(one_node['hostname'], one_node) \ - for one_node in nodes_all]) - - + nodes_all = self.iotlab_api.GetNodes( + {'hostname': slice_nodes_list}, + ['node_id', 'hostname', 'site', 'boot_state']) + nodeall_byhostname = dict([(one_node['hostname'], one_node) + for one_node in nodes_all]) for single_slice in slice_list: - #For compatibility top_level_status = 'empty' result = {} result.fromkeys( - ['geni_urn','geni_error', 'iotlab_login','geni_status', - 'geni_resources'], None) + ['geni_urn', 'geni_error', 'iotlab_login', 'geni_status', + 'geni_resources'], None) # result.fromkeys(\ # ['geni_urn','geni_error', 'pl_login','geni_status', # 'geni_resources'], None) # result['pl_login'] = one_slice['reg_researchers'][0].hrn result['iotlab_login'] = one_slice['user'] logger.debug("Slabdriver - sliver_status Sliver status \ - urn %s hrn %s single_slice %s \r\n " \ - %(slice_urn, slice_hrn, single_slice)) + urn %s hrn %s single_slice %s \r\n " + % (slice_urn, slice_hrn, single_slice)) if 'node_ids' not in single_slice: #No job in the slice @@ -267,15 +254,16 @@ class IotlabDriver(Driver): res = {} res['iotlab_hostname'] = node_hostname res['iotlab_boot_state'] = \ - nodeall_byhostname[node_hostname]['boot_state'] + nodeall_byhostname[node_hostname]['boot_state'] #res['pl_hostname'] = node['hostname'] #res['pl_boot_state'] = \ #nodeall_byhostname[node['hostname']]['boot_state'] #res['pl_last_contact'] = strftime(self.time_format, \ #gmtime(float(timestamp))) - sliver_id = Xrn(slice_urn, type='slice', \ - id=nodeall_byhostname[node_hostname]['node_id']).urn + sliver_id = Xrn( + slice_urn, type='slice', + id=nodeall_byhostname[node_hostname]['node_id']).urn res['geni_urn'] = sliver_id #node_name = node['hostname'] @@ -292,8 +280,8 @@ class IotlabDriver(Driver): result['geni_status'] = top_level_status result['geni_resources'] = resources - logger.debug("IOTLABDRIVER \tsliver_statusresources %s res %s "\ - %(resources,res)) + logger.debug("IOTLABDRIVER \tsliver_statusresources %s res %s " + % (resources, res)) return result @staticmethod @@ -308,26 +296,25 @@ class IotlabDriver(Driver): :rtype: RegUser """ - return dbsession.query(RegRecord).filter_by(hrn = hrn).first() - + return dbsession.query(RegRecord).filter_by(hrn=hrn).first() - def testbed_name (self): + def testbed_name(self): """ Returns testbed's name. - + :returns: testbed authority name. :rtype: string """ return self.hrn # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory - def aggregate_version (self): + def aggregate_version(self): """ - Returns the testbed's supported rspec advertisement and - request versions. - + Returns the testbed's supported rspec advertisement and request + versions. + :returns: rspec versions supported ad a dictionary. :rtype: dict """ @@ -340,12 +327,9 @@ class IotlabDriver(Driver): if rspec_version.content_type in ['*', 'request']: request_rspec_versions.append(rspec_version.to_dict()) return { - 'testbed':self.testbed_name(), + 'testbed': self.testbed_name(), 'geni_request_rspec_versions': request_rspec_versions, - 'geni_ad_rspec_versions': ad_rspec_versions, - } - - + 'geni_ad_rspec_versions': ad_rspec_versions} def _get_requested_leases_list(self, rspec): """ @@ -368,18 +352,18 @@ class IotlabDriver(Driver): if not lease.get('lease_id'): if get_authority(lease['component_id']) == \ - self.iotlab_api.root_auth: + self.iotlab_api.root_auth: single_requested_lease['hostname'] = \ - iotlab_xrn_to_hostname(\ - lease.get('component_id').strip()) + iotlab_xrn_to_hostname(\ + lease.get('component_id').strip()) single_requested_lease['start_time'] = \ - lease.get('start_time') + lease.get('start_time') single_requested_lease['duration'] = lease.get('duration') #Check the experiment's duration is valid before adding #the lease to the requested leases list duration_in_seconds = \ - int(single_requested_lease['duration']) - if duration_in_seconds >= self.iotlab_api.GetMinExperimentDurationInSec() : + int(single_requested_lease['duration']) + if duration_in_seconds >= self.iotlab_api.GetMinExperimentDurationInSec(): requested_lease_list.append(single_requested_lease) return requested_lease_list @@ -410,12 +394,11 @@ class IotlabDriver(Driver): if isinstance(lease['hostname'], str): lease['hostname'] = [lease['hostname']] - requested_job_dict[lease['start_time']] = lease else: job_lease = requested_job_dict[lease['start_time']] - if lease['duration'] == job_lease['duration'] : + if lease['duration'] == job_lease['duration']: job_lease['hostname'].append(lease['hostname']) return requested_job_dict @@ -431,16 +414,16 @@ class IotlabDriver(Driver): """ requested_lease_list = self._get_requested_leases_list(rspec) - logger.debug("IOTLABDRIVER _process_requested_jobs requested_lease_list \ - %s"%(requested_lease_list)) - job_dict = self._group_leases_by_start_time(requested_lease_list) + logger.debug("IOTLABDRIVER _process_requested_jobs \ + requested_lease_list %s" % (requested_lease_list)) + job_dict = self._group_leases_by_start_time(requested_lease_list) logger.debug("IOTLABDRIVER _process_requested_jobs job_dict\ - %s"%(job_dict)) + %s" % (job_dict)) return job_dict - def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, \ - users, options): + def create_sliver(self, slice_urn, slice_hrn, creds, rspec_string, + users, options): """Answer to CreateSliver. Creates the leases and slivers for the users from the information @@ -475,40 +458,39 @@ class IotlabDriver(Driver): if users: slice_record = users[0].get('slice_record', {}) logger.debug("IOTLABDRIVER.PY \t ===============create_sliver \t\ - creds %s \r\n \r\n users %s" \ - %(creds, users)) - slice_record['user'] = {'keys':users[0]['keys'], \ - 'email':users[0]['email'], \ - 'hrn':slice_record['reg-researchers'][0]} + creds %s \r\n \r\n users %s" + % (creds, users)) + slice_record['user'] = {'keys': users[0]['keys'], + 'email': users[0]['email'], + 'hrn': slice_record['reg-researchers'][0]} # parse rspec rspec = RSpec(rspec_string) logger.debug("IOTLABDRIVER.PY \t create_sliver \trspec.version \ - %s slice_record %s users %s" \ - %(rspec.version,slice_record, users)) - + %s slice_record %s users %s" + % (rspec.version, slice_record, users)) # ensure site record exists? # ensure slice record exists - #Removed options to verify_slice SA 14/08/12 - sfa_slice = slices.verify_slice(slice_hrn, slice_record, peer, \ - sfa_peer) + #Removed options in verify_slice SA 14/08/12 + #Removed peer record in verify_slice SA 18/07/13 + sfa_slice = slices.verify_slice(slice_hrn, slice_record, sfa_peer) # ensure person records exists - #verify_persons returns added persons but since the return value + #verify_persons returns added persons but the return value #is not used - slices.verify_persons(slice_hrn, sfa_slice, users, peer, \ - sfa_peer, options=options) + #Removed peer record and sfa_peer in verify_persons SA 18/07/13 + slices.verify_persons(slice_hrn, sfa_slice, users, options=options) #requested_attributes returned by rspec.version.get_slice_attributes() #unused, removed SA 13/08/12 #rspec.version.get_slice_attributes() - logger.debug("IOTLABDRIVER.PY create_sliver slice %s " %(sfa_slice)) + logger.debug("IOTLABDRIVER.PY create_sliver slice %s " % (sfa_slice)) # add/remove slice from nodes #requested_slivers = [node.get('component_id') \ - #for node in rspec.version.get_nodes_with_slivers()\ - #if node.get('authority_id') is self.iotlab_api.root_auth] + #for node in rspec.version.get_nodes_with_slivers()\ + #if node.get('authority_id') is self.iotlab_api.root_auth] #l = [ node for node in rspec.version.get_nodes_with_slivers() ] #logger.debug("SLADRIVER \tcreate_sliver requested_slivers \ #requested_slivers %s listnodes %s" \ @@ -516,60 +498,64 @@ class IotlabDriver(Driver): #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12. #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer) - requested_job_dict = self._process_requested_jobs(rspec) - - logger.debug("IOTLABDRIVER.PY \tcreate_sliver requested_job_dict %s "\ - %(requested_job_dict)) + logger.debug("IOTLABDRIVER.PY \tcreate_sliver requested_job_dict %s " + % (requested_job_dict)) #verify_slice_leases returns the leases , but the return value is unused #here. Removed SA 13/08/12 - slices.verify_slice_leases(sfa_slice, \ - requested_job_dict, peer) + slices.verify_slice_leases(sfa_slice, + requested_job_dict, peer) - return aggregate.get_rspec(slice_xrn=slice_urn, \ - login=sfa_slice['login'], version=rspec.version) + return aggregate.get_rspec(slice_xrn=slice_urn, + login=sfa_slice['login'], + version=rspec.version) - - def delete_sliver (self, slice_urn, slice_hrn, creds, options): + def delete_sliver(self, slice_urn, slice_hrn, creds, options): """ Deletes the lease associated with the slice hrn and the credentials if the slice belongs to iotlab. Answer to DeleteSliver. + :param slice_urn: urn of the slice + :param slice_hrn: name of the slice + :param creds: slice credenials + :type slice_urn: string + :type slice_hrn: string + :type creds: ? unused + :returns: 1 if the slice to delete was not found on iotlab, True if the deletion was successful, False otherwise otherwise. .. note:: Should really be named delete_leases because iotlab does not have any slivers, but only deals with leases. However, - SFA api only have delete_sliver define so far. SA 13.05/2013 + SFA api only have delete_sliver define so far. SA 13/05/2013 + .. note:: creds are unused, and are not used either in the dummy driver + delete_sliver . """ - sfa_slice_list = self.iotlab_api.GetSlices(slice_filter = slice_hrn, \ - slice_filter_type = 'slice_hrn') + sfa_slice_list = self.iotlab_api.GetSlices( + slice_filter=slice_hrn, + slice_filter_type='slice_hrn') if not sfa_slice_list: return 1 #Delete all leases in the slice for sfa_slice in sfa_slice_list: - - - logger.debug("IOTLABDRIVER.PY delete_sliver slice %s" %(sfa_slice)) + logger.debug("IOTLABDRIVER.PY delete_sliver slice %s" % (sfa_slice)) slices = IotlabSlices(self) # determine if this is a peer slice peer = slices.get_peer(slice_hrn) logger.debug("IOTLABDRIVER.PY delete_sliver peer %s \ - \r\n \t sfa_slice %s " %(peer, sfa_slice)) + \r\n \t sfa_slice %s " % (peer, sfa_slice)) try: - self.iotlab_api.DeleteSliceFromNodes(sfa_slice) return True - except : + except: return False - def list_resources (self, slice_urn, slice_hrn, creds, options): """ @@ -580,11 +566,18 @@ class IotlabDriver(Driver): nodes in a rspec format. Answer to ListResources. Caching unused. + :param slice_urn: urn of the slice + :param slice_hrn: name of the slice + :param creds: slice credenials + :type slice_urn: string + :type slice_hrn: string + :type creds: ? unused :param options: options used when listing resources (list_leases, info, geni_available) :returns: rspec string in xml :rtype: string + .. note:: creds are unused """ #cached_requested = options.get('cached', True) @@ -592,13 +585,13 @@ class IotlabDriver(Driver): version_manager = VersionManager() # get the rspec's return format from options rspec_version = \ - version_manager.get_version(options.get('geni_rspec_version')) + version_manager.get_version(options.get('geni_rspec_version')) version_string = "rspec_%s" % (rspec_version) #panos adding the info option to the caching key (can be improved) if options.get('info'): version_string = version_string + "_" + \ - options.get('info', 'default') + options.get('info', 'default') # Adding the list_leases option to the caching key if options.get('list_leases'): @@ -621,8 +614,8 @@ class IotlabDriver(Driver): #panos: passing user-defined options aggregate = IotlabAggregate(self) - rspec = aggregate.get_rspec(slice_xrn=slice_urn, \ - version=rspec_version, options=options) + rspec = aggregate.get_rspec(slice_xrn=slice_urn, + version=rspec_version, options=options) # cache the result #if self.cache and not slice_hrn: @@ -632,7 +625,7 @@ class IotlabDriver(Driver): return rspec - def list_slices (self, creds, options): + def list_slices(self, creds, options): """Answer to ListSlices. List slices belonging to iotlab, returns slice urns list. @@ -642,6 +635,7 @@ class IotlabDriver(Driver): :returns: slice urns list :rtype: list + .. note:: creds are unused """ # look in cache first #if self.cache: @@ -653,11 +647,12 @@ class IotlabDriver(Driver): # get data from db slices = self.iotlab_api.GetSlices() - logger.debug("IOTLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" %(slices)) + logger.debug("IOTLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" + % (slices)) slice_hrns = [iotlab_slice['hrn'] for iotlab_slice in slices] - slice_urns = [hrn_to_urn(slice_hrn, 'slice') \ - for slice_hrn in slice_hrns] + slice_urns = [hrn_to_urn(slice_hrn, 'slice') + for slice_hrn in slice_hrns] # cache the result #if self.cache: @@ -667,7 +662,7 @@ class IotlabDriver(Driver): return slice_urns - def register (self, sfa_record, hrn, pub_key): + def register(self, sfa_record, hrn, pub_key): """ Adding new user, slice, node or site should not be handled by SFA. @@ -681,12 +676,16 @@ class IotlabDriver(Driver): :param sfa_record: record provided by the client of the Register API call. :type sfa_record: dict + :param pub_key: public key of the user + :type pub_key: string + + .. note:: DOES NOTHING. Returns -1. """ return -1 - def update (self, old_sfa_record, new_sfa_record, hrn, new_key): + def update(self, old_sfa_record, new_sfa_record, hrn, new_key): """ No site or node record update allowed in Iotlab. The only modifications authorized here are key deletion/addition @@ -700,19 +699,24 @@ class IotlabDriver(Driver): :param old_sfa_record: what is in the db for this hrn :param new_sfa_record: what was passed to the Update call + :param new_key: the new user's public key + :param hrn: the user's sfa hrn + :type old_sfa_record: dictionary + :type new_sfa_record: dictionary + :type pub_key: string + :type hrn: string + TODO: needs review .. seealso::: update in driver.py. """ - pointer = old_sfa_record['pointer'] old_sfa_record_type = old_sfa_record['type'] # new_key implemented for users only - if new_key and old_sfa_record_type not in [ 'user' ]: + if new_key and old_sfa_record_type not in ['user']: raise UnknownSfaType(old_sfa_record_type) - if old_sfa_record_type == "user": update_fields = {} all_fields = new_sfa_record @@ -720,7 +724,6 @@ class IotlabDriver(Driver): if key in ['key', 'password']: update_fields[key] = all_fields[key] - if new_key: # must check this key against the previous one if it exists persons = self.iotlab_api.GetPersons([old_sfa_record]) @@ -741,28 +744,25 @@ class IotlabDriver(Driver): #remove all the other keys for key in keys_dict: self.iotlab_api.DeleteKey(person, key) - self.iotlab_api.AddPersonKey(person, \ - {'sshPublicKey': person['pkey']},{'sshPublicKey': new_key} ) - #self.iotlab_api.AddPersonKey(person, {'key_type': 'ssh', \ - #'key': new_key}) + self.iotlab_api.AddPersonKey( + person, {'sshPublicKey': person['pkey']}, + {'sshPublicKey': new_key}) return True - - def remove (self, sfa_record): + def remove(self, sfa_record): """ Removes users only. Mark the user as disabled in LDAP. The user and his slice are then deleted from the db by running an import on the registry. - - :param sfa_record: record is the existing sfa record in the db :type sfa_record: dict ..warning::As fas as the slice is concerned, here only the leases are removed from the slice. The slice is record itself is not removed from the db. + TODO: needs review TODO : REMOVE SLICE FROM THE DB AS WELL? SA 14/05/2013, @@ -777,15 +777,12 @@ class IotlabDriver(Driver): #No registering at a given site in Iotlab. #Once registered to the LDAP, all iotlab sites are #accesible. - if person : + if person: #Mark account as disabled in ldap return self.iotlab_api.DeletePerson(sfa_record) elif sfa_record_type == 'slice': - if self.iotlab_api.GetSlices(slice_filter = hrn, \ - slice_filter_type = 'slice_hrn'): + if self.iotlab_api.GetSlices(slice_filter=hrn, + slice_filter_type='slice_hrn'): ret = self.iotlab_api.DeleteSlice(sfa_record) - - - - return True \ No newline at end of file + return True diff --git a/sfa/iotlab/iotlabpostgres.py b/sfa/iotlab/iotlabpostgres.py index a35b2c56..70bc1510 100644 --- a/sfa/iotlab/iotlabpostgres.py +++ b/sfa/iotlab/iotlabpostgres.py @@ -1,13 +1,17 @@ +""" +File defining classes to handle the table in the iotlab dedicated database. +""" + from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from sfa.util.config import Config +# from sfa.util.config import Config from sfa.util.sfalogging import logger from sqlalchemy import Column, Integer, String from sqlalchemy import Table, MetaData from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.dialects import postgresql +# from sqlalchemy.dialects import postgresql from sqlalchemy.exc import NoSuchTableError @@ -15,8 +19,8 @@ from sqlalchemy.exc import NoSuchTableError #Dict holding the columns names of the table as keys #and their type, used for creation of the table slice_table = {'record_id_user': 'integer PRIMARY KEY references X ON DELETE \ -CASCADE ON UPDATE CASCADE','oar_job_id':'integer DEFAULT -1', \ -'record_id_slice':'integer', 'slice_hrn':'text NOT NULL'} + CASCADE ON UPDATE CASCADE', 'oar_job_id': 'integer DEFAULT -1', + 'record_id_slice': 'integer', 'slice_hrn': 'text NOT NULL'} #Dict with all the specific iotlab tables tablenames_dict = {'iotlab_xp': slice_table} @@ -25,70 +29,82 @@ tablenames_dict = {'iotlab_xp': slice_table} IotlabBase = declarative_base() - class IotlabXP (IotlabBase): - """ SQL alchemy class to manipulate slice_iotlab table in - iotlab_sfa database. + """ SQL alchemy class to manipulate the rows of the slice_iotlab table in + iotlab_sfa database. Handles the records representation and creates the + table if it does not exist yet. """ __tablename__ = 'iotlab_xp' - slice_hrn = Column(String) - job_id = Column(Integer, primary_key = True) - end_time = Column(Integer, nullable = False) - + job_id = Column(Integer, primary_key=True) + end_time = Column(Integer, nullable=False) - #oar_job_id = Column( Integer,default = -1) - #node_list = Column(postgresql.ARRAY(String), nullable =True) - - def __init__ (self, slice_hrn =None, job_id=None, end_time=None): + def __init__(self, slice_hrn=None, job_id=None, end_time=None): """ Defines a row of the slice_iotlab table """ if slice_hrn: self.slice_hrn = slice_hrn - if job_id : + if job_id: self.job_id = job_id if end_time: self.end_time = end_time - def __repr__(self): """Prints the SQLAlchemy record to the format defined by the function. """ result = " 0: self.iotlab_session.query(IotlabXP).filter(IotlabXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') self.iotlab_session.commit() - return - - - def __init__(self, config, debug = False): + def __init__(self, config, debug=False): self.sl_base = IotlabBase # Check whether we already have an instance @@ -193,14 +209,12 @@ class IotlabDB(object): # Store instance reference as the only member in the handle self._EventHandler_singleton = IotlabDB._connection_singleton - - def __getattr__(self, aAttr): """ Delegate access to implementation. - :param attr: Attribute wanted. - :return: Attribute + :param aAttr: Attribute wanted. + :returns: Attribute """ return getattr(self._connection_singleton, aAttr) @@ -215,27 +229,25 @@ class IotlabDB(object): # """ # return setattr(self._connection_singleton, aAttr, aValue) - - - - def exists(self, tablename): """ Checks if the table specified as tablename exists. + :param tablename: name of the table in the db that has to be checked. + :type tablename: string + :returns: True if the table exists, False otherwise. + :rtype: bool """ - + metadata = MetaData(bind=self.iotlab_engine) try: - metadata = MetaData (bind=self.iotlab_engine) - table = Table (tablename, metadata, autoload=True) + table = Table(tablename, metadata, autoload=True) return True except NoSuchTableError: - logger.log_exc("SLABPOSTGRES tablename %s does not exists" \ - %(tablename)) + logger.log_exc("SLABPOSTGRES tablename %s does not exist" + % (tablename)) return False - def createtable(self): """ Creates all the table sof the engine. @@ -243,13 +255,8 @@ class IotlabDB(object): """ - logger.debug("SLABPOSTGRES createtable IotlabBase.metadata.sorted_tables \ - %s \r\n engine %s" %(IotlabBase.metadata.sorted_tables , self.iotlab_engine)) + logger.debug("SLABPOSTGRES createtable \ + IotlabBase.metadata.sorted_tables %s \r\n engine %s" + % (IotlabBase.metadata.sorted_tables, self.iotlab_engine)) IotlabBase.metadata.create_all(self.iotlab_engine) return - - - -# iotlab_alchemy = IotlabDB(Config()) -# iotlab_engine = iotlab_alchemy.iotlab_engine -# iotlab_dbsession = iotlab_alchemy.session() diff --git a/sfa/iotlab/iotlabslices.py b/sfa/iotlab/iotlabslices.py index cc036b4d..f44575c9 100644 --- a/sfa/iotlab/iotlabslices.py +++ b/sfa/iotlab/iotlabslices.py @@ -1,13 +1,23 @@ +""" +This file defines the IotlabSlices class by which all the slice checkings +upon lease creation are done. +""" from sfa.util.xrn import get_authority, urn_to_hrn from sfa.util.sfalogging import logger MAXINT = 2L**31-1 -class IotlabSlices: +class IotlabSlices: + """ + This class is responsible for checking the slice when creating a + lease or a sliver. Those checks include verifying that the user is valid, + that the slice is known from the testbed or from our peers, that the list + of nodes involved has not changed (in this case the lease is modified + accordingly). + """ rspec_to_slice_tag = {'max_rate': 'net_max_rate'} - def __init__(self, driver): """ Get the reference to the driver here. @@ -42,18 +52,17 @@ class IotlabSlices: site_authority = get_authority(slice_authority).lower() # get this site's authority (sfa root authority or sub authority) - logger.debug("IOTLABSLICES \ get_peer slice_authority %s \ + logger.debug("IOTLABSLICES \t get_peer slice_authority %s \ site_authority %s hrn %s" - % (slice_authority, site_authority, hrn)) + % (slice_authority, site_authority, hrn)) # check if we are already peered with this site_authority #if so find the peer record peers = self.driver.iotlab_api.GetPeers(peer_filter=site_authority) for peer_record in peers: - if site_authority == peer_record.hrn: peer = peer_record - logger.debug(" IOTLABSLICES \tget_peer peer %s " %(peer)) + logger.debug(" IOTLABSLICES \tget_peer peer %s " % (peer)) return peer def get_sfa_peer(self, xrn): @@ -78,7 +87,6 @@ class IotlabSlices: return sfa_peer - def verify_slice_leases(self, sfa_slice, requested_jobs_dict, peer): """ Compare requested leases with the leases already scheduled/ @@ -87,11 +95,11 @@ class IotlabSlices: :param sfa_slice: sfa slice record :param requested_jobs_dict: dictionary of requested leases - :param peer: sfa peer + :param peer: sfa peer record :type sfa_slice: dict :type requested_jobs_dict: dict - :type peer: + :type peer: dict :returns: leases list of dictionary :rtype: list @@ -237,6 +245,7 @@ class IotlabSlices: :returns: list requested nodes hostnames :rtype: list + .. warning:: UNUSED SQA 24/07/13 .. seealso:: DeleteSliceFromNodes .. todo:: check what to do with the peer? Can not remove peer nodes from slice here. Anyway, in this case, the peer should have gotten the @@ -536,10 +545,11 @@ class IotlabSlices: #try: ##if peer: #person = persondict[user['email']] - #self.driver.iotlab_api.UnBindObjectFromPeer('person', - #person['person_id'], peer['shortname']) - ret = self.driver.iotlab_api.AddPersonKey(\ - user['email'], key) + #self.driver.iotlab_api.UnBindObjectFromPeer( + # 'person',person['person_id'], + # peer['shortname']) + ret = self.driver.iotlab_api.AddPersonKey( + user['email'], key) #if peer: #key_index = user_keys.index(key['key']) #remote_key_id = user['key_ids'][key_index] @@ -555,7 +565,7 @@ class IotlabSlices: # remove old keys (only if we are not appending) append = options.get('append', True) - if append == False: + if append is False: removed_keys = set(existing_keys).difference(requested_keys) for key in removed_keys: #if peer: -- 2.47.0