From bfbecb7f227b4ab005f2f2826091f90f2a997487 Mon Sep 17 00:00:00 2001 From: Sandrine Avakian Date: Thu, 18 Jul 2013 17:20:44 +0200 Subject: [PATCH] Fixing bug in DeleteLeases. Documenting, cleaning. --- sfa/iotlab/OARrestapi.py | 90 ++++----- sfa/iotlab/iotlabaggregate.py | 256 +++++++++++++----------- sfa/iotlab/iotlabapi.py | 199 +++++++++++-------- sfa/iotlab/iotlabslices.py | 363 ++++++++++++++++------------------ 4 files changed, 474 insertions(+), 434 deletions(-) diff --git a/sfa/iotlab/OARrestapi.py b/sfa/iotlab/OARrestapi.py index 35877ae8..414dd5a8 100644 --- a/sfa/iotlab/OARrestapi.py +++ b/sfa/iotlab/OARrestapi.py @@ -170,10 +170,8 @@ class OARrestapi: :type strval: integer :type next_page: boolean :type username: string - - :returns: a json dictionary if OAR successfully processed the GET - request. + request. .. seealso:: OARrequests_uri_dict """ @@ -232,7 +230,8 @@ class OARrestapi: #first check that all params for are OK try: - self.oarserver['uri'] = self.OAR_REQUEST_POST_URI_DICT[request]['uri'] + self.oarserver['uri'] = \ + self.OAR_REQUEST_POST_URI_DICT[request]['uri'] except KeyError: logger.log_exc("OARrestapi \tPOSTRequestToOARRestAPI request not \ @@ -279,6 +278,15 @@ class ParsingResourcesFull(): """ def __init__(self): + """ + Set the parsing dictionary. Works like a switch case, if the key is + found in the dictionary, then the associated function is called. + This is used in ParseNodes to create an usable dictionary from + the Json returned by OAR when issuing a GET resources full request. + + .. seealso:: ParseNodes + + """ self.resources_fulljson_dict = { 'network_address': self.AddNodeNetworkAddr, 'site': self.AddNodeSite, @@ -317,8 +325,8 @@ class ParsingResourcesFull(): The value associated with the key is a tuple list.It contains all the nodes attributes. The tuplelist will later be turned into a dict. - :param dictnode: should be set to the OARGETParser atribute node_dictlist. - It will store the information on the nodes. + :param dictnode: should be set to the OARGETParser atribute + node_dictlist. It will store the information on the nodes. :param value: the node_id is the network_address in the raw json. :type value: string :type dictnode: dictionary @@ -335,9 +343,10 @@ class ParsingResourcesFull(): """Add the site's node to the dictionary. - :param tuplelist: tuple list on which to add the node's site. Contains the - other node attributes as well. - :param value: value to add to the tuple list, in this case the node's site. + :param tuplelist: tuple list on which to add the node's site. + Contains the other node attributes as well. + :param value: value to add to the tuple list, in this case the node's + site. :type tuplelist: list :type value: string @@ -349,8 +358,9 @@ class ParsingResourcesFull(): # def AddNodeRadio(tuplelist, value): # """Add thenode's radio chipset type to the tuple list. - # :param tuplelist: tuple list on which to add the node's mobility status. The - # tuplelist is the value associated with the node's id in the OARGETParser + # :param tuplelist: tuple list on which to add the node's mobility + # status. The tuplelist is the value associated with the node's + # id in the OARGETParser # 's dictionary node_dictlist. # :param value: name of the radio chipset on the node. # :type tuplelist: list @@ -365,9 +375,9 @@ class ParsingResourcesFull(): def AddMobility(self, tuplelist, value): """Add if the node is a mobile node or not to the tuple list. - :param tuplelist: tuple list on which to add the node's mobility status. The - tuplelist is the value associated with the node's id in the OARGETParser - 's dictionary node_dictlist. + :param tuplelist: tuple list on which to add the node's mobility status. + The tuplelist is the value associated with the node's id in the + OARGETParser's dictionary node_dictlist. :param value: tells if a node is a mobile node or not. The value is found in the json. @@ -387,8 +397,8 @@ class ParsingResourcesFull(): """Add the node's position on the x axis. :param tuplelist: tuple list on which to add the node's position . The - tuplelist is the value associated with the node's id in the OARGETParser - 's dictionary node_dictlist. + tuplelist is the value associated with the node's id in the + OARGETParser's dictionary node_dictlist. :param value: the position x. :type tuplelist: list @@ -405,8 +415,8 @@ class ParsingResourcesFull(): """Add the node's position on the y axis. :param tuplelist: tuple list on which to add the node's position . The - tuplelist is the value associated with the node's id in the OARGETParser - 's dictionary node_dictlist. + tuplelist is the value associated with the node's id in the + OARGETParser's dictionary node_dictlist. :param value: the position y. :type tuplelist: list @@ -423,8 +433,8 @@ class ParsingResourcesFull(): """Add the node's position on the z axis. :param tuplelist: tuple list on which to add the node's position . The - tuplelist is the value associated with the node's id in the OARGETParser - 's dictionary node_dictlist. + tuplelist is the value associated with the node's id in the + OARGETParser's dictionary node_dictlist. :param value: the position z. :type tuplelist: list @@ -442,8 +452,8 @@ class ParsingResourcesFull(): """Add the node's state, Alive or Suspected. :param tuplelist: tuple list on which to add the node's state . The - tuplelist is the value associated with the node's id in the OARGETParser - 's dictionary node_dictlist. + tuplelist is the value associated with the node's id in the + OARGETParser 's dictionary node_dictlist. :param value: node's state. :type tuplelist: list @@ -523,7 +533,6 @@ class OARGETParser: to parse the jsons returned after a get request has been issued. Updates the attribute version_json_dict. - """ if 'oar_version' in self.json_page.raw_json : @@ -643,7 +652,6 @@ class OARGETParser: for resource in self.json_page.raw_json['items']: job_resources.append(resource['id']) - #logger.debug("OARESTAPI \tParseJobsIdResources %s" %(self.json_page.raw_json)) return job_resources def ParseResources(self) : @@ -668,8 +676,8 @@ class OARGETParser: job['t_until'] = int(json_element['scheduled_start']) + \ int(json_element['walltime']) #Get resources id list for the job - job['resource_ids'] = \ - [ node_dict['id'] for node_dict in json_element['resources']] + job['resource_ids'] = [ node_dict['id'] for node_dict + in json_element['resources']] else: job['t_from'] = "As soon as possible" job['t_until'] = "As soon as possible" @@ -702,7 +710,7 @@ class OARGETParser: def ChangeRawJsonDependingOnApilibVersion(self): - if self.version_json_dict['apilib_version'] != "0.2.10" : + if self.version_json_dict['apilib_version'] != "0.2.10": self.json_page.raw_json = self.json_page.raw_json['items'] def ParseDeleteJobs(self): @@ -806,14 +814,13 @@ class OARGETParser: if node['node_id'] not in nodes_per_site[node['site']]: nodes_per_site[node['site']].append(node['node_id']) - #Create a site dictionary whose key is site_login_base (name of the site) - # and value is a dictionary of properties, including the list - #of the node_ids + #Create a site dictionary whose key is site_login_base + # (name of the site) and value is a dictionary of properties, + # including the list of the node_ids for node_id in self.node_dictlist: node = self.node_dictlist[node_id] - #node.update({'hrn':self.iotlab_hostname_to_hrn(self.interface_hrn, \ - #node['site'],node['hostname'])}) - node.update({'hrn':self.iotlab_hostname_to_hrn(self.interface_hrn, node['hostname'])}) + node.update({'hrn':self.iotlab_hostname_to_hrn(self.interface_hrn, + node['hostname'])}) self.node_dictlist.update({node_id:node}) if node['site'] not in self.site_dict: @@ -821,26 +828,14 @@ class OARGETParser: 'site':node['site'], 'node_ids':nodes_per_site[node['site']], 'latitude':"48.83726", - 'longitude':"- 2.10336",'name':config.SFA_REGISTRY_ROOT_AUTH, + 'longitude':"- 2.10336", + 'name': config.SFA_REGISTRY_ROOT_AUTH, 'pcu_ids':[], 'max_slices':None, 'ext_consortium_id':None, 'max_slivers':None, 'is_public':True, 'peer_site_id': None, 'abbreviated_name':"iotlab", 'address_ids': [], 'url':"http,//www.senslab.info", 'person_ids':[], 'site_tag_ids':[], 'enabled': True, 'slice_ids':[], 'date_created': None, 'peer_id': None } - #if node['site_login_base'] not in self.site_dict.keys(): - #self.site_dict[node['site_login_base']] = {'login_base':node['site_login_base'], - #'node_ids':nodes_per_site[node['site_login_base']], - #'latitude':"48.83726", - #'longitude':"- 2.10336",'name':"senslab", - #'pcu_ids':[], 'max_slices':None, 'ext_consortium_id':None, - #'max_slivers':None, 'is_public':True, 'peer_site_id': None, - #'abbreviated_name':"senslab", 'address_ids': [], - #'url':"http,//www.senslab.info", 'person_ids':[], - #'site_tag_ids':[], 'enabled': True, 'slice_ids':[], - #'date_created': None, 'peer_id': None } - - OARrequests_uri_dict = { @@ -914,4 +909,3 @@ class OARGETParser: else: logger.error("OARRESTAPI OARGetParse __init__ : ERROR_REQUEST " \ %(request)) - diff --git a/sfa/iotlab/iotlabaggregate.py b/sfa/iotlab/iotlabaggregate.py index 8c3306d5..86b14056 100644 --- a/sfa/iotlab/iotlabaggregate.py +++ b/sfa/iotlab/iotlabaggregate.py @@ -1,4 +1,3 @@ -#import time from sfa.util.xrn import hrn_to_urn, urn_to_hrn, get_authority from sfa.rspecs.rspec import RSpec @@ -11,26 +10,43 @@ from sfa.rspecs.elements.lease import Lease from sfa.rspecs.elements.granularity import Granularity from sfa.rspecs.version_manager import VersionManager +from sfa.rspecs.elements.versions.iotlabv1Node import IotlabPosition, \ + IotlabNode, IotlabLocation -from sfa.rspecs.elements.versions.iotlabv1Node import IotlabPosition, IotlabNode, \ - IotlabLocation from sfa.util.sfalogging import logger - from sfa.util.xrn import Xrn + def iotlab_xrn_to_hostname(xrn): + """Returns a node's hostname from its xrn. + :param xrn: The nodes xrn identifier. + :type xrn: Xrn (from sfa.util.xrn) + + :returns: node's hostname. + :rtype: string + + """ return Xrn.unescape(Xrn(xrn=xrn, type='node').get_leaf()) + def iotlab_xrn_object(root_auth, hostname): - """Attributes are urn and hrn. - Get the hostname using iotlab_xrn_to_hostname on the urn. + """Creates a valid xrn object from the node's hostname and the authority + of the SFA server. + + :param hostname: the node's hostname. + :param root_auth: the SFA root authority. + :type hostname: string + :type root_auth: string :returns: the iotlab node's xrn :rtype: Xrn + """ - return Xrn('.'.join( [root_auth, Xrn.escape(hostname)]), type='node') + return Xrn('.'.join([root_auth, Xrn.escape(hostname)]), type='node') + class IotlabAggregate: + """Aggregate manager class for Iotlab. """ sites = {} nodes = {} @@ -49,9 +65,10 @@ class IotlabAggregate: def get_slice_and_slivers(self, slice_xrn, login=None): """ Get the slices and the associated leases if any from the iotlab - testbed. For each slice, get the nodes in the associated lease - and create a sliver with the necessary info and insertinto the - sliver bdictionary, keyed on the node hostnames. + testbed. One slice can have mutliple leases. + For each slice, get the nodes in the associated lease + and create a sliver with the necessary info and insert it into the + sliver dictionary, keyed on the node hostnames. Returns a dict of slivers based on the sliver's node_id. Called by get_rspec. @@ -61,10 +78,11 @@ class IotlabAggregate: :type slice_xrn: string :type login: string - :returns: a list of slices dict and a dictionary of Sliver object - :rtype: (list, dict) + :returns: a list of slices dict and a list of Sliver object + :rtype: (list, list) - ..note: There is no slivers in iotlab, only leases. + .. note: There is no real slivers in iotlab, only leases. The goal + is to be consistent with the SFA standard. """ slivers = {} @@ -75,24 +93,23 @@ class IotlabAggregate: slice_hrn, _ = urn_to_hrn(slice_xrn) slice_name = slice_hrn - slices = self.driver.iotlab_api.GetSlices(slice_filter= str(slice_name), \ - slice_filter_type = 'slice_hrn', \ - login=login) + slices = self.driver.iotlab_api.GetSlices(slice_filter=str(slice_name), + slice_filter_type='slice_hrn', + login=login) - logger.debug("Slabaggregate api \tget_slice_and_slivers \ - sfa_slice %s \r\n slices %s self.driver.hrn %s" \ - %(sfa_slice, slices, self.driver.hrn)) - if slices == []: + logger.debug("IotlabAggregate api \tget_slice_and_slivers \ + sfa_slice %s \r\n slices %s self.driver.hrn %s" + % (sfa_slice, slices, self.driver.hrn)) + if slices == []: return (sfa_slice, slivers) - # sort slivers by node id , if there is a job #and therefore, node allocated to this slice for sfa_slice in slices: try: - node_ids_list = sfa_slice['node_ids'] + node_ids_list = sfa_slice['node_ids'] except KeyError: - logger.log_exc("SLABAGGREGATE \t \ + logger.log_exc("IOTLABAGGREGATE \t \ get_slice_and_slivers No nodes in the slice \ - KeyError ") continue @@ -100,92 +117,87 @@ class IotlabAggregate: for node in node_ids_list: sliver_xrn = Xrn(slice_urn, type='sliver', id=node) sliver_xrn.set_authority(self.driver.hrn) - sliver = Sliver({'sliver_id':sliver_xrn.urn, + sliver = Sliver({'sliver_id': sliver_xrn.urn, 'name': sfa_slice['hrn'], 'type': 'iotlab-node', 'tags': []}) slivers[node] = sliver - #Add default sliver attribute : #connection information for iotlab - if get_authority (sfa_slice['hrn']) == self.driver.iotlab_api.root_auth: + if get_authority(sfa_slice['hrn']) == self.driver.iotlab_api.root_auth: tmp = sfa_slice['hrn'].split('.') ldap_username = tmp[1].split('_')[0] ssh_access = None - slivers['default_sliver'] = {'ssh': ssh_access , \ - 'login': ldap_username} + slivers['default_sliver'] = {'ssh': ssh_access, + 'login': ldap_username} #TODO get_slice_and_slivers Find the login of the external user - logger.debug("SLABAGGREGATE api get_slice_and_slivers slivers %s "\ - %(slivers)) + logger.debug("IOTLABAGGREGATE api get_slice_and_slivers slivers %s " + % (slivers)) return (slices, slivers) def get_nodes(self, slices=None, slivers=[], options=None): + """Returns the nodes in the slice using the rspec format, with all the + nodes' properties. + + Fetch the nodes ids in the slices dictionary and get all the nodes + properties from OAR. Makes a rspec dicitonary out of this and returns + it. If the slice does not have any job running or scheduled, that is + it has no reserved nodes, then returns an empty list. + + :param slices: list of slices (record dictionaries) + :param slivers: the list of slivers in all the slices + :type slices: list of dicts + :type slivers: list of Sliver object (dictionaries) + :returns: An empty list if the slice has no reserved nodes, a rspec + list with all the nodes and their properties (a dict per node) + otherwise. + :rtype: list + + .. seealso:: get_slice_and_slivers + + """ # NT: the semantic of this function is not clear to me : # if slice is not defined, then all the nodes should be returned # if slice is defined, we should return only the nodes that # are part of this slice # but what is the role of the slivers parameter ? # So i assume that slice['node_ids'] will be the same as slivers for us - #filter_dict = {} - #if slice_xrn: - #if not slices or not slices['node_ids']: - #return ([],[]) - #tags_filter = {} + if slices is not None: + for one_slice in slices: + try: + slice_nodes_list = one_slice['node_ids'] + # if we are dealing with a slice that has no node just + # return an empty list. In iotlab a slice can have multiple + # jobs scheduled, so it either has at least one lease or + # not at all. + except KeyError: + return [] # get the granularity in second for the reservation system grain = self.driver.iotlab_api.GetLeaseGranularity() - nodes = self.driver.iotlab_api.GetNodes() - #geni_available = options.get('geni_available') - #if geni_available: - #filter['boot_state'] = 'boot' - - #filter.update({'peer_id': None}) - #nodes = self.driver.iotlab_api.GetNodes(filter['hostname']) - #site_ids = [] - #interface_ids = [] - #tag_ids = [] nodes_dict = {} - #for node in nodes: - - #nodes_dict[node['node_id']] = node - #logger.debug("SLABAGGREGATE api get_nodes nodes %s "\ - #%(nodes )) - # get sites - #sites_dict = self.get_sites({'site_id': site_ids}) - # get interfaces - #interfaces = self.get_interfaces({'interface_id':interface_ids}) - # get tags - #node_tags = self.get_node_tags(tags_filter) - #if slices, this means we got to list all the nodes given to this slice # Make a list of all the nodes in the slice before getting their #attributes rspec_nodes = [] slice_nodes_list = [] - logger.debug("SLABAGGREGATE api get_nodes slice_nodes_list %s "\ - %(slices )) - if slices is not None: - for one_slice in slices: - try: - slice_nodes_list = one_slice['node_ids'] - except KeyError: - pass - #for node in one_slice['node_ids']: - #slice_nodes_list.append(node) + logger.debug("IOTLABAGGREGATE api get_nodes slice_nodes_list %s " + % (slices)) + reserved_nodes = self.driver.iotlab_api.GetNodesCurrentlyInUse() - logger.debug("SLABAGGREGATE api get_nodes slice_nodes_list %s "\ - %(slice_nodes_list)) + logger.debug("IOTLABAGGREGATE api get_nodes slice_nodes_list %s " + % (slice_nodes_list)) for node in nodes: nodes_dict[node['node_id']] = node if slice_nodes_list == [] or node['hostname'] in slice_nodes_list: @@ -199,12 +211,12 @@ class IotlabAggregate: rspec_node['archi'] = node['archi'] rspec_node['radio'] = node['radio'] - iotlab_xrn = iotlab_xrn_object(self.driver.iotlab_api.root_auth, \ - node['hostname']) + iotlab_xrn = iotlab_xrn_object(self.driver.iotlab_api.root_auth, + node['hostname']) rspec_node['component_id'] = iotlab_xrn.urn rspec_node['component_name'] = node['hostname'] rspec_node['component_manager_id'] = \ - hrn_to_urn(self.driver.iotlab_api.root_auth, \ + hrn_to_urn(self.driver.iotlab_api.root_auth, 'authority+sa') # Iotlab's nodes are federated : there is only one authority @@ -235,8 +247,8 @@ class IotlabAggregate: try: position[field] = node[field] except KeyError, error : - logger.log_exc("SLABAGGREGATE\t get_nodes \ - position %s "%(error)) + logger.log_exc("IOTLABAGGREGATE\t get_nodes \ + position %s "% (error)) rspec_node['position'] = position #rspec_node['interfaces'] = [] @@ -261,15 +273,19 @@ class IotlabAggregate: rspec_nodes.append(rspec_node) return (rspec_nodes) - #def get_all_leases(self, slice_record = None): + def get_all_leases(self): """ + Get list of lease dictionaries which all have the mandatory keys ('lease_id', 'hostname', 'site_id', 'name', 'start_time', 'duration'). All the leases running or scheduled are returned. + :returns: rspec lease dictionary with keys lease_id, component_id, + slice_id, start_time, duration. + :rtype: dict - ..note::There is no filtering of leases within a given time frame. + .. note::There is no filtering of leases within a given time frame. All the running or scheduled leases are returned. options removed SA 15/05/2013 @@ -285,7 +301,7 @@ class IotlabAggregate: #leases = self.driver.iotlab_api.GetLeases(lease_filter) leases = self.driver.iotlab_api.GetLeases() grain = self.driver.iotlab_api.GetLeaseGranularity() - site_ids = [] + # site_ids = [] rspec_leases = [] for lease in leases: #as many leases as there are nodes in the job @@ -293,7 +309,8 @@ class IotlabAggregate: rspec_lease = Lease() rspec_lease['lease_id'] = lease['lease_id'] #site = node['site_id'] - iotlab_xrn = iotlab_xrn_object(self.driver.iotlab_api.root_auth, node) + iotlab_xrn = iotlab_xrn_object(self.driver.iotlab_api.root_auth, + node) rspec_lease['component_id'] = iotlab_xrn.urn #rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn,\ #site, node['hostname']) @@ -304,45 +321,60 @@ class IotlabAggregate: pass rspec_lease['start_time'] = lease['t_from'] rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) \ - / grain + / grain rspec_leases.append(rspec_lease) return rspec_leases + def get_rspec(self, slice_xrn=None, login=None, version=None, + options=None): + """ + + Returns xml rspec: + - a full advertisement rspec with the testbed resources if slice_xrn + is not specified.If a lease option is given, also returns the + leases scheduled on the testbed. + - a manifest Rspec with the leases and nodes in slice's leases + if slice_xrn is not None. + + :param slice_xrn: srn of the slice + :param login: user'uid (ldap login) on iotlab + :param version: can be set to sfa or iotlab + :param options: used to specify if the leases should also be included in + the returned rspec. + :type slice_xrn: string + :type login: string + :type version: RSpecVersion + :type options: dict + + :returns: Xml Rspec. + :rtype: XML -#from plc/aggregate.py - def get_rspec(self, slice_xrn=None, login=None, version = None, \ - options=None): + """ rspec = None version_manager = VersionManager() version = version_manager.get_version(version) logger.debug("IotlabAggregate \t get_rspec ***version %s \ - version.type %s version.version %s options %s \r\n" \ - %(version,version.type,version.version,options)) + version.type %s version.version %s options %s \r\n" + % (version, version.type, version.version, options)) if slice_xrn is None: - rspec_version = version_manager._get_version(version.type, \ - version.version, 'ad') + rspec_version = version_manager._get_version(version.type, + version.version, 'ad') else: - rspec_version = version_manager._get_version(version.type, \ - version.version, 'manifest') + rspec_version = version_manager._get_version( + version.type, version.version, 'manifest') slices, slivers = self.get_slice_and_slivers(slice_xrn, login) #at this point sliver may be empty if no iotlab job #is running for this user/slice. rspec = RSpec(version=rspec_version, user_options=options) - - #if slice and 'expires' in slice: - #rspec.xml.set('expires',\ - #datetime_to_string(utcparse(slice['expires'])) - # add sliver defaults - #nodes, links = self.get_nodes(slice, slivers) logger.debug("\r\n \r\n IotlabAggregate \tget_rspec *** \ - slice_xrn %s slices %s\r\n \r\n"\ - %(slice_xrn, slices)) + slice_xrn %s slices %s\r\n \r\n" + % (slice_xrn, slices)) if options is not None: lease_option = options['list_leases'] @@ -352,49 +384,39 @@ class IotlabAggregate: #if slice_xrn : #lease_option = 'all' - if lease_option in ['all', 'resources']: #if not options.get('list_leases') or options.get('list_leases') #and options['list_leases'] != 'leases': nodes = self.get_nodes(slices, slivers) - logger.debug("\r\n \r\n IotlabAggregate \ lease_option %s \ - get rspec ******* nodes %s"\ - %(lease_option, nodes[0])) + logger.debug("\r\n \r\n IotlabAggregate \t lease_option %s \ + get rspec ******* nodes %s" + % (lease_option, nodes[0])) - sites_set = set([node['location']['site'] for node in nodes] ) + sites_set = set([node['location']['site'] for node in nodes]) #In case creating a job, slice_xrn is not set to None rspec.version.add_nodes(nodes) - if slice_xrn : + if slice_xrn: #Get user associated with this slice - #user = dbsession.query(RegRecord).filter_by(record_id = \ - #slices['record_id_user']).first() - - #ldap_username = (user.hrn).split('.')[1] - - #for one_slice in slices : ldap_username = slices[0]['hrn'] tmp = ldap_username.split('.') ldap_username = tmp[1].split('_')[0] if version.type == "Iotlab": - rspec.version.add_connection_information(ldap_username, \ - sites_set) + rspec.version.add_connection_information( + ldap_username, sites_set) default_sliver = slivers.get('default_sliver', []) if default_sliver: #default_sliver_attribs = default_sliver.get('tags', []) logger.debug("IotlabAggregate \tget_rspec **** \ - default_sliver%s \r\n" %(default_sliver)) + default_sliver%s \r\n" % (default_sliver)) for attrib in default_sliver: - rspec.version.add_default_sliver_attribute(attrib, \ - default_sliver[attrib]) + rspec.version.add_default_sliver_attribute( + attrib, default_sliver[attrib]) + if lease_option in ['all','leases']: - #leases = self.get_all_leases(slices) leases = self.get_all_leases() rspec.version.add_leases(leases) - - #logger.debug("IotlabAggregate \tget_rspec ******* rspec_toxml %s \r\n"\ - #%(rspec.toxml())) return rspec.toxml() diff --git a/sfa/iotlab/iotlabapi.py b/sfa/iotlab/iotlabapi.py index 32776170..fb8541d4 100644 --- a/sfa/iotlab/iotlabapi.py +++ b/sfa/iotlab/iotlabapi.py @@ -147,18 +147,20 @@ class IotlabTestbedAPI(): #return server_timestamp, server_tz def DeleteJobs(self, job_id, username): + """ - """ Deletes the job with the specified job_id and username on OAR by - posting a delete request to OAR. + Deletes the job with the specified job_id and username on OAR by + posting a delete request to OAR. :param job_id: job id in OAR. :param username: user's iotlab login in LDAP. - :type job_id:integer + :type job_id: integer :type username: string :returns: dictionary with the job id and if delete has been successful - (True) or no (False) + (True) or no (False) :rtype: dict + """ logger.debug("IOTLABDRIVER \tDeleteJobs jobid %s username %s "\ %(job_id, username)) @@ -340,15 +342,15 @@ class IotlabTestbedAPI(): """ Make a list of iotlab nodes and their properties from information - given by OAR. Search for specific nodes if some filters are specified. - Nodes properties returned if no return_fields_list given: - 'hrn','archi','mobile','hostname','site','boot_state','node_id', - 'radio','posx','posy','oar_id','posz'. + given by OAR. Search for specific nodes if some filters are + specified. Nodes properties returned if no return_fields_list given: + 'hrn','archi','mobile','hostname','site','boot_state','node_id', + 'radio','posx','posy','oar_id','posz'. :param node_filter_dict: dictionnary of lists with node properties :type node_filter_dict: dict :param return_fields_list: list of specific fields the user wants to be - returned. + returned. :type return_fields_list: list :returns: list of dictionaries with node properties :rtype: list @@ -389,15 +391,19 @@ class IotlabTestbedAPI(): @staticmethod def AddSlice(slice_record, user_record): - """Add slice to the local iotlab sfa tables if the slice comes - from a federated site and is not yet in the iotlab sfa DB, - although the user has already a LDAP login. - Called by verify_slice during lease/sliver creation. + """ + + Add slice to the local iotlab sfa tables if the slice comes + from a federated site and is not yet in the iotlab sfa DB, + although the user has already a LDAP login. + Called by verify_slice during lease/sliver creation. + :param slice_record: record of slice, must contain hrn, gid, slice_id - and authority of the slice. + and authority of the slice. :type slice_record: dictionary :param user_record: record of the user :type user_record: RegUser + """ sfa_record = RegSlice(hrn=slice_record['hrn'], @@ -447,29 +453,35 @@ class IotlabTestbedAPI(): #TODO : Check rights to delete person def DeletePerson(self, person_record): - """ Disable an existing account in iotlab LDAP. + """Disable an existing account in iotlab LDAP. + Users and techs can only delete themselves. PIs can only - delete themselves and other non-PIs at their sites. - ins can delete anyone. + delete themselves and other non-PIs at their sites. + ins can delete anyone. + :param person_record: user's record :type person_record: dict :returns: True if successful, False otherwise. :rtype: boolean + .. todo:: CHECK THAT ONLY THE USER OR ADMIN CAN DEL HIMSELF. """ #Disable user account in iotlab LDAP ret = self.ldap.LdapMarkUserAsDeleted(person_record) - logger.warning("IOTLABDRIVER DeletePerson %s " %(person_record)) + logger.warning("IOTLABDRIVER DeletePerson %s " % (person_record)) return ret['bool'] - def DeleteSlice(self, slice_record): - """ Deletes the specified slice and kills the jobs associated with - the slice if any, using DeleteSliceFromNodes. + """Deletes the specified slice and kills the jobs associated with + the slice if any, using DeleteSliceFromNodes. + + :param slice_record: record of the slice, must contain oar_job_id, user + :type slice_record: dict + :returns: True if all the jobs in the slice have been deleted, + or the list of jobs that could not be deleted otherwise. + :rtype: list or boolean - :returns: True if all the jobs in the slice have been deleted, - or the list of jobs that could not be deleted otherwise. - :rtype: list or boolean + .. seealso:: DeleteSliceFromNodes """ ret = self.DeleteSliceFromNodes(slice_record) @@ -536,13 +548,17 @@ class IotlabTestbedAPI(): def AddPerson(self, record): - """Adds a new account. Any fields specified in records are used, - otherwise defaults are used. Creates an appropriate login by calling - LdapAddUser. + """ + + Adds a new account. Any fields specified in records are used, + otherwise defaults are used. Creates an appropriate login by calling + LdapAddUser. + :param record: dictionary with the sfa user's properties. :returns: The uid of the added person if sucessful, otherwise returns - the error message from LDAP. + the error message from LDAP. :rtype: interger or string + """ ret = self.ldap.LdapAddUser(record) @@ -560,12 +576,13 @@ class IotlabTestbedAPI(): #TODO AddPersonKey 04/07/2012 SA def AddPersonKey(self, person_uid, old_attributes_dict, new_key_dict): """Adds a new key to the specified account. Adds the key to the - iotlab ldap, provided that the person_uid is valid. + iotlab ldap, provided that the person_uid is valid. + Non-admins can only modify their own keys. :param person_uid: user's iotlab login in LDAP :param old_attributes_dict: dict with the user's old sshPublicKey - :param new_key_dict:dict with the user's new sshPublicKey + :param new_key_dict: dict with the user's new sshPublicKey :type person_uid: string @@ -578,19 +595,24 @@ class IotlabTestbedAPI(): logger.warning("IOTLABDRIVER AddPersonKey EMPTY - DO NOTHING \r\n ") return ret['bool'] - def DeleteLeases(self, leases_id_list, slice_hrn ): + def DeleteLeases(self, leases_id_list, slice_hrn): """ + Deletes several leases, based on their job ids and the slice - they are associated with. Uses DeleteJobs to delete the jobs - on OAR. Note that one slice can contain multiple jobs, and in this case - all the jobs in the leases_id_list MUST belong to ONE slice, - since there is only one slice hrn provided here. + they are associated with. Uses DeleteJobs to delete the jobs + on OAR. Note that one slice can contain multiple jobs, and in this + case all the jobs in the leases_id_list MUST belong to ONE slice, + since there is only one slice hrn provided here. + :param leases_id_list: list of job ids that belong to the slice whose - slice hrn is provided. - :param slice_hrn: the slice hrn . - ..warning: Does not have a return value since there was no easy - way to handle failure when dealing with multiple job delete. Plus, - there was no easy way to report it to the user. + slice hrn is provided. + :param slice_hrn: the slice hrn. + :type slice_hrn: string + + .. warning:: Does not have a return value since there was no easy + way to handle failure when dealing with multiple job delete. Plus, + there was no easy way to report it to the user. + """ logger.debug("IOTLABDRIVER DeleteLeases leases_id_list %s slice_hrn %s \ \r\n " %(leases_id_list, slice_hrn)) @@ -804,14 +826,18 @@ class IotlabTestbedAPI(): #Delete the jobs from job_iotlab table def DeleteSliceFromNodes(self, slice_record): - """ Deletes all the running or scheduled jobs of a given slice - given its record. - :param slice_record: record of the slice + """ + + Deletes all the running or scheduled jobs of a given slice + given its record. + + :param slice_record: record of the slice, must contain oar_job_id, user :type slice_record: dict :returns: dict of the jobs'deletion status. Success= True, Failure= - False, for each job id. + False, for each job id. :rtype: dict + """ logger.debug("IOTLABDRIVER \t DeleteSliceFromNodese %s " %(slice_record)) @@ -866,18 +892,22 @@ class IotlabTestbedAPI(): def GetLeases(self, lease_filter_dict=None, login=None): - """ Get the list of leases from OAR with complete information - about which slice owns which jobs and nodes. - Two purposes: - -Fetch all the jobs from OAR (running, waiting..) - complete the reservation information with slice hrn - found in iotlab_xp table. If not available in the table, - assume it is a iotlab slice. - -Updates the iotlab table, deleting jobs when necessary. + """ + + Get the list of leases from OAR with complete information + about which slice owns which jobs and nodes. + Two purposes: + -Fetch all the jobs from OAR (running, waiting..) + complete the reservation information with slice hrn + found in iotlab_xp table. If not available in the table, + assume it is a iotlab slice. + -Updates the iotlab table, deleting jobs when necessary. + :returns: reservation_list, list of dictionaries with 'lease_id', - 'reserved_nodes','slice_id', 'state', 'user', 'component_id_list', - 'slice_hrn', 'resource_ids', 't_from', 't_until' + 'reserved_nodes','slice_id', 'state', 'user', 'component_id_list', + 'slice_hrn', 'resource_ids', 't_from', 't_until' :rtype: list + """ unfiltered_reservation_list = self.GetReservedNodes(login) @@ -1057,9 +1087,11 @@ class IotlabTestbedAPI(): #TODO : test def DeleteKey(self, user_record, key_string): - """ Deletes a key in the LDAP entry of the specified user. + """Deletes a key in the LDAP entry of the specified user. + Removes the key_string from the user's key list and updates the LDAP - user's entry with the new key attributes. + user's entry with the new key attributes. + :param key_string: The ssh key to remove :param user_record: User's record :type key_string: string @@ -1082,12 +1114,14 @@ class IotlabTestbedAPI(): def _sql_get_slice_info( slice_filter ): """ Get the slice record based on the slice hrn. Fetch the record of the - user associated with the slice by usingjoinedload based on t - he reg_researcher relationship. + user associated with the slice by using joinedload based on the + reg_researcher relationship. + :param slice_filter: the slice hrn we are looking for :type slice_filter: string :returns: the slice record enhanced with the user's information if the - slice was found, None it wasn't. + slice was found, None it wasn't. + :rtype: dict or None. """ #DO NOT USE RegSlice - reg_researchers to get the hrn @@ -1195,24 +1229,24 @@ class IotlabTestbedAPI(): return fixed_slicerec_dict - - def GetSlices(self, slice_filter = None, slice_filter_type = None, \ - login=None): - """ Get the slice records from the iotlab db and add lease information - if any. + def GetSlices(self, slice_filter=None, slice_filter_type=None, + login=None): + """Get the slice records from the iotlab db and add lease information + if any. :param slice_filter: can be the slice hrn or slice record id in the db - depending on the slice_filter_type. + depending on the slice_filter_type. :param slice_filter_type: defines the type of the filtering used, Can be - either 'slice_hrn' or "record_id'. + either 'slice_hrn' or "record_id'. :type slice_filter: string :type slice_filter_type: string :returns: a slice dict if slice_filter and slice_filter_type - are specified and a matching entry is found in the db. The result - is put into a list.Or a list of slice dictionnaries if no filters are - specified. + are specified and a matching entry is found in the db. The result + is put into a list.Or a list of slice dictionnaries if no filters + arespecified. :rtype: list + """ #login = None authorized_filter_types_list = ['slice_hrn', 'record_id_user'] @@ -1220,14 +1254,14 @@ class IotlabTestbedAPI(): #First try to get information on the slice based on the filter provided if slice_filter_type in authorized_filter_types_list: - fixed_slicerec_dict = \ - self._get_slice_records(slice_filter, slice_filter_type) + fixed_slicerec_dict = self._get_slice_records(slice_filter, + slice_filter_type) slice_hrn = fixed_slicerec_dict['hrn'] logger.debug(" IOTLABDRIVER \tGetSlices login %s \ slice record %s slice_filter %s \ - slice_filter_type %s " %(login, \ - fixed_slicerec_dict, slice_filter, \ + slice_filter_type %s " % (login, + fixed_slicerec_dict, slice_filter, slice_filter_type)) @@ -1235,7 +1269,7 @@ class IotlabTestbedAPI(): #jobs associated to this slice leases_list = [] - leases_list = self.GetLeases(login = login) + leases_list = self.GetLeases(login=login) #If no job is running or no job scheduled #return only the slice record if leases_list == [] and fixed_slicerec_dict: @@ -1244,24 +1278,27 @@ class IotlabTestbedAPI(): #If several jobs for one slice , put the slice record into # each lease information dict - for lease in leases_list : slicerec_dict = {} logger.debug("IOTLABDRIVER.PY \tGetSlices slice_filter %s \ - \ lease['slice_hrn'] %s" \ - %(slice_filter, lease['slice_hrn'])) - if lease['slice_hrn'] == slice_hrn: + \ lease['slice_hrn'] %s" + % (slice_filter, lease['slice_hrn'])) + if lease['slice_hrn'] == slice_hrn: slicerec_dict['slice_hrn'] = lease['slice_hrn'] slicerec_dict['hrn'] = lease['slice_hrn'] slicerec_dict['user'] = lease['user'] slicerec_dict['oar_job_id'] = lease['lease_id'] - slicerec_dict.update({'list_node_ids':{'hostname':lease['reserved_nodes']}}) - slicerec_dict.update({'node_ids':lease['reserved_nodes']}) + slicerec_dict.update( + {'list_node_ids': + {'hostname': lease['reserved_nodes']} + }) + slicerec_dict.update({'node_ids': lease['reserved_nodes']}) #Update lease dict with the slice record if fixed_slicerec_dict: fixed_slicerec_dict['oar_job_id'] = [] - fixed_slicerec_dict['oar_job_id'].append(slicerec_dict['oar_job_id']) + fixed_slicerec_dict['oar_job_id'].append( + slicerec_dict['oar_job_id']) slicerec_dict.update(fixed_slicerec_dict) #slicerec_dict.update({'hrn':\ #str(fixed_slicerec_dict['slice_hrn'])}) diff --git a/sfa/iotlab/iotlabslices.py b/sfa/iotlab/iotlabslices.py index ad12b1d8..cc036b4d 100644 --- a/sfa/iotlab/iotlabslices.py +++ b/sfa/iotlab/iotlabslices.py @@ -1,12 +1,11 @@ from sfa.util.xrn import get_authority, urn_to_hrn from sfa.util.sfalogging import logger - -MAXINT = 2L**31-1 +MAXINT = 2L**31-1 class IotlabSlices: - rspec_to_slice_tag = {'max_rate':'net_max_rate'} + rspec_to_slice_tag = {'max_rate': 'net_max_rate'} def __init__(self, driver): @@ -15,13 +14,18 @@ class IotlabSlices: """ self.driver = driver - def get_peer(self, xrn): """ - Find the authority of a resources based on its xrn. + Finds the authority of a resource based on its xrn. If the authority is Iotlab (local) return None, Otherwise, look up in the DB if Iotlab is federated with this site - authority and returns its DB record if it is the case, + authority and returns its DB record if it is the case. + + :param xrn: resource's xrn + :type xrn: string + :returns: peer record + :rtype: dict + """ hrn, hrn_type = urn_to_hrn(xrn) #Does this slice belong to a local site or a peer iotlab site? @@ -31,7 +35,7 @@ class IotlabSlices: slice_authority = get_authority(hrn) #Iotlab stuff #This slice belongs to the current site - if slice_authority == self.driver.iotlab_api.root_auth: + if slice_authority == self.driver.iotlab_api.root_auth: site_authority = slice_authority return None @@ -39,13 +43,12 @@ class IotlabSlices: # get this site's authority (sfa root authority or sub authority) logger.debug("IOTLABSLICES \ get_peer slice_authority %s \ - site_authority %s hrn %s" %(slice_authority, \ - site_authority, hrn)) - + site_authority %s hrn %s" + % (slice_authority, site_authority, hrn)) # check if we are already peered with this site_authority #if so find the peer record - peers = self.driver.iotlab_api.GetPeers(peer_filter = site_authority) + peers = self.driver.iotlab_api.GetPeers(peer_filter=site_authority) for peer_record in peers: if site_authority == peer_record.hrn: @@ -54,6 +57,15 @@ class IotlabSlices: return peer def get_sfa_peer(self, xrn): + """Returns the authority name for the xrn or None if the local site + is the authority. + + :param xrn: the xrn of the resource we are looking the authority for. + :type xrn: string + :returns: the resources's authority name. + :rtype: string + + """ hrn, hrn_type = urn_to_hrn(xrn) # return the authority for this hrn or None if we are the authority @@ -85,26 +97,25 @@ class IotlabSlices: """ - logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s \ - "%( sfa_slice)) + logger.debug("IOTLABSLICES verify_slice_leases sfa_slice %s " + % (sfa_slice)) #First get the list of current leases from OAR - leases = self.driver.iotlab_api.GetLeases({'name':sfa_slice['hrn']}) + leases = self.driver.iotlab_api.GetLeases({'name': sfa_slice['hrn']}) logger.debug("IOTLABSLICES verify_slice_leases requested_jobs_dict %s \ - leases %s "%(requested_jobs_dict, leases )) + leases %s " % (requested_jobs_dict, leases)) current_nodes_reserved_by_start_time = {} requested_nodes_by_start_time = {} leases_by_start_time = {} reschedule_jobs_dict = {} - #Create reduced dictionary with key start_time and value # the list of nodes #-for the leases already registered by OAR first # then for the new leases requested by the user #Leases already scheduled/running in OAR - for lease in leases : + for lease in leases: current_nodes_reserved_by_start_time[lease['t_from']] = \ lease['reserved_nodes'] leases_by_start_time[lease['t_from']] = lease @@ -116,8 +127,8 @@ class IotlabSlices: #Requested jobs for start_time in requested_jobs_dict: - requested_nodes_by_start_time[int(start_time)] = \ - requested_jobs_dict[start_time]['hostname'] + requested_nodes_by_start_time[int(start_time)] = \ + requested_jobs_dict[start_time]['hostname'] #Check if there is any difference between the leases already #registered in OAR and the requested jobs. #Difference could be: @@ -127,7 +138,7 @@ class IotlabSlices: logger.debug("IOTLABSLICES verify_slice_leases \ requested_nodes_by_start_time %s \ - "%(requested_nodes_by_start_time )) + "% (requested_nodes_by_start_time)) #Find all deleted leases start_time_list = \ list(set(leases_by_start_time.keys()).\ @@ -136,7 +147,6 @@ class IotlabSlices: for start_time in start_time_list] - #Find added or removed nodes in exisiting leases for start_time in requested_nodes_by_start_time: logger.debug("IOTLABSLICES verify_slice_leases start_time %s \ @@ -184,91 +194,111 @@ class IotlabSlices: job = requested_jobs_dict[str(start_time)] logger.debug("IOTLABSLICES \ - NEWLEASE slice %s job %s"\ - %(sfa_slice, job)) - self.driver.iotlab_api.AddLeases(job['hostname'], \ - sfa_slice, int(job['start_time']), \ - int(job['duration'])) + NEWLEASE slice %s job %s" + % (sfa_slice, job)) + self.driver.iotlab_api.AddLeases( + job['hostname'], + sfa_slice, int(job['start_time']), + int(job['duration'])) #Deleted leases are the ones with lease id not declared in the Rspec if deleted_leases: - self.driver.iotlab_api.DeleteLeases(deleted_leases, sfa_slice['hrn']) + self.driver.iotlab_api.DeleteLeases(deleted_leases, + sfa_slice['user']['uid']) logger.debug("IOTLABSLICES \ - verify_slice_leases slice %s deleted_leases %s"\ - %(sfa_slice, deleted_leases)) - + verify_slice_leases slice %s deleted_leases %s" + % (sfa_slice, deleted_leases)) - if reschedule_jobs_dict : - for start_time in reschedule_jobs_dict: + if reschedule_jobs_dict: + for start_time in reschedule_jobs_dict: job = reschedule_jobs_dict[start_time] - self.driver.iotlab_api.AddLeases(job['hostname'], \ - sfa_slice, int(job['start_time']), \ + self.driver.iotlab_api.AddLeases( + job['hostname'], + sfa_slice, int(job['start_time']), int(job['duration'])) return leases def verify_slice_nodes(self, sfa_slice, requested_slivers, peer): + """Check for wanted and unwanted nodes in the slice. + + Removes nodes and associated leases that the user does not want anymore + by deleteing the associated job in OAR (DeleteSliceFromNodes). + Returns the nodes' hostnames that are going to be in the slice. + + :param sfa_slice: slice record. Must contain node_ids and list_node_ids. + + :param requested_slivers: list of requested nodes' hostnames. + :param peer: unused so far. + + :type sfa_slice: dict + :type requested_slivers: list + :type peer: string + + :returns: list requested nodes hostnames + :rtype: list + + .. seealso:: DeleteSliceFromNodes + .. todo:: check what to do with the peer? Can not remove peer nodes from + slice here. Anyway, in this case, the peer should have gotten the + remove request too. + + """ current_slivers = [] deleted_nodes = [] if 'node_ids' in sfa_slice: - nodes = self.driver.iotlab_api.GetNodes(sfa_slice['list_node_ids'], \ + nodes = self.driver.iotlab_api.GetNodes( + sfa_slice['list_node_ids'], ['hostname']) current_slivers = [node['hostname'] for node in nodes] # remove nodes not in rspec - deleted_nodes = list(set(current_slivers).\ - difference(requested_slivers)) - # add nodes from rspec - #added_nodes = list(set(requested_slivers).\ - #difference(current_slivers)) - + deleted_nodes = list(set(current_slivers). + difference(requested_slivers)) logger.debug("IOTLABSLICES \tverify_slice_nodes slice %s\ - \r\n \r\n deleted_nodes %s"\ - %(sfa_slice, deleted_nodes)) + \r\n \r\n deleted_nodes %s" + % (sfa_slice, deleted_nodes)) if deleted_nodes: #Delete the entire experience self.driver.iotlab_api.DeleteSliceFromNodes(sfa_slice) - #self.driver.DeleteSliceFromNodes(sfa_slice['slice_hrn'], \ - #deleted_nodes) return nodes + def verify_slice(self, slice_hrn, slice_record, sfa_peer): + """Ensures slice record exists. + The slice record must exist either in Iotlab or in the other + federated testbed (sfa_peer). If the slice does not belong to Iotlab, + check if the user already exists in LDAP. In this case, adds the slice + to the sfa DB and associates its LDAP user. - def free_egre_key(self): - used = set() - for tag in self.driver.iotlab_api.GetSliceTags({'tagname': 'egre_key'}): - used.add(int(tag['value'])) - - for i in range(1, 256): - if i not in used: - key = i - break - else: - raise KeyError("No more EGRE keys available") - - return str(key) + :param slice_hrn: slice's name + :param slice_record: sfa record of the slice + :param sfa_peer: name of the peer authority if any.(not Iotlab). + :type slice_hrn: string + :type slice_record: dictionary + :type sfa_peer: string + .. seealso:: AddSlice + """ - - def verify_slice(self, slice_hrn, slice_record, peer, sfa_peer): - - #login_base = slice_hrn.split(".")[0] slicename = slice_hrn - slices_list = self.driver.iotlab_api.GetSlices(slice_filter = slicename, \ - slice_filter_type = 'slice_hrn') + # check if slice belongs to Iotlab + slices_list = self.driver.iotlab_api.GetSlices( + slice_filter=slicename, slice_filter_type='slice_hrn') + sfa_slice = None + if slices_list: for sl in slices_list: - logger.debug("SLABSLICE \tverify_slice slicename %s \ - slices_list %s sl %s \ slice_record %s"\ - %(slicename, slices_list,sl, \ - slice_record)) + logger.debug("SLABSLICE \t verify_slice slicename %s \ + slices_list %s sl %s \r slice_record %s" + % (slicename, slices_list, sl, slice_record)) sfa_slice = sl sfa_slice.update(slice_record) @@ -277,83 +307,67 @@ class IotlabSlices: ldap_user = self.driver.iotlab_api.ldap.LdapFindUser(\ slice_record['user']) logger.debug(" IOTLABSLICES \tverify_slice Oups \ - slice_record %s sfa_peer %s ldap_user %s"\ - %(slice_record, sfa_peer, ldap_user )) + slice_record %s sfa_peer %s ldap_user %s" + % (slice_record, sfa_peer, ldap_user)) #User already registered in ldap, meaning user should be in SFA db #and hrn = sfa_auth+ uid sfa_slice = {'hrn': slicename, - #'url': slice_record.get('url', slice_hrn), - #'description': slice_record.get('description', slice_hrn) - 'node_list' : [], - 'authority' : slice_record['authority'], - 'gid':slice_record['gid'], - #'record_id_user' : user.record_id, - 'slice_id' : slice_record['record_id'], - 'reg-researchers':slice_record['reg-researchers'], - #'record_id_slice': slice_record['record_id'], - 'peer_authority':str(sfa_peer) - - } - if ldap_user : - hrn = self.driver.iotlab_api.root_auth +'.'+ ldap_user['uid'] - + 'node_list': [], + 'authority': slice_record['authority'], + 'gid': slice_record['gid'], + 'slice_id': slice_record['record_id'], + 'reg-researchers': slice_record['reg-researchers'], + 'peer_authority': str(sfa_peer) + } + + if ldap_user: + hrn = self.driver.iotlab_api.root_auth + '.' + ldap_user['uid'] user = self.driver.get_user_record(hrn) - logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s" \ - %(hrn, user)) - #sfa_slice = {'slice_hrn': slicename, - ##'url': slice_record.get('url', slice_hrn), - ##'description': slice_record.get('description', slice_hrn) - #'node_list' : [], - #'authority' : slice_record['authority'], - #'gid':slice_record['gid'], - ##'record_id_user' : user.record_id, - #'slice_id' : slice_record['record_id'], - #'reg-researchers':slice_record['reg-researchers'], - ##'record_id_slice': slice_record['record_id'], - #'peer_authority':str(peer.hrn) - - #} - # add the slice - if sfa_slice : - self.driver.iotlab_api.AddSlice(sfa_slice, user) + logger.debug(" IOTLABSLICES \tverify_slice hrn %s USER %s" + % (hrn, user)) - if peer: - sfa_slice['slice_id'] = slice_record['record_id'] + # add the external slice to the local SFA iotlab DB + if sfa_slice: + self.driver.iotlab_api.AddSlice(sfa_slice, user) - #slice['slice_id'] = self.driver.iotlab_api.AddSlice(slice) logger.debug("IOTLABSLICES \tverify_slice ADDSLICE OK") - #slice['node_ids']=[] - #slice['person_ids'] = [] - #if peer: - #sfa_slice['peer_slice_id'] = slice_record.get('slice_id', None) - # mark this slice as an sfa peer record - #if sfa_peer: - #peer_dict = {'type': 'slice', 'hrn': slice_hrn, - #'peer_authority': sfa_peer, 'pointer': \ - #slice['slice_id']} - #self.registry.register_peer_object(self.credential, peer_dict) + return sfa_slice + def verify_persons(self, slice_hrn, slice_record, users, options={}): + """Ensures the users in users list exist and are enabled in LDAP. Adds + person if needed. - return sfa_slice + Checking that a user exist is based on the user's email. If the user is + still not found in the LDAP, it means that the user comes from another + federated. In this case an account has to be created in LDAP + so as to enable the user to use the testbed, since we trust the testbed + he comes from. This is done by calling AddPerson. + + :param slice_hrn: slice name + :param slice_record: record of the slice_hrn + :param users: users is a record list. Records can either be + local records or users records from known and trusted federated + sites.If the user is from another site that iotlab doesn't trust yet, + then Resolve will raise an error before getting to create_sliver. + + :type slice_hrn: string + :type slice_record: string + :type users: list + + .. seealso:: AddPerson + .. note:: Removed unused peer and sfa_peer parameters. SA 18/07/13. - def verify_persons(self, slice_hrn, slice_record, users, peer, sfa_peer, \ - options={}): - """ - users is a record list. Records can either be local records - or users records from known and trusted federated sites. - If the user is from another site that iotlab doesn't trust yet, - then Resolve will raise an error before getting to create_sliver. """ #TODO SA 21/08/12 verify_persons Needs review logger.debug("IOTLABSLICES \tverify_persons \tslice_hrn %s \ - \t slice_record %s\r\n users %s \t peer %s "\ - %( slice_hrn, slice_record, users, peer)) + \t slice_record %s\r\n users %s \t " + % (slice_hrn, slice_record, users)) users_by_id = {} - #users_by_hrn = {} + users_by_email = {} #users_dict : dict whose keys can either be the user's hrn or its id. #Values contains only id and hrn @@ -361,8 +375,7 @@ class IotlabSlices: #First create dicts by hrn and id for each user in the user record list: for info in users: - - if 'slice_record' in info : + if 'slice_record' in info: slice_rec = info['slice_record'] user = slice_rec['user'] @@ -370,44 +383,36 @@ class IotlabSlices: users_by_email[user['email']] = user users_dict[user['email']] = user - #if 'hrn' in user: - #users_by_hrn[user['hrn']] = user - #users_dict[user['hrn']] = user - - logger.debug( "SLABSLICE.PY \t verify_person \ + logger.debug("SLABSLICE.PY \t verify_person \ users_dict %s \r\n user_by_email %s \r\n \ - \tusers_by_id %s " \ - %(users_dict,users_by_email, users_by_id)) + \tusers_by_id %s " + % (users_dict, users_by_email, users_by_id)) existing_user_ids = [] - #existing_user_hrns = [] existing_user_emails = [] existing_users = [] # Check if user is in Iotlab LDAP using its hrn. # Assuming Iotlab is centralised : one LDAP for all sites, - # user'as record_id unknown from LDAP - # LDAP does not provide users id, therefore we rely on hrns containing - # the login of the user. - # If the hrn is not a iotlab hrn, the user may not be in LDAP. + # user's record_id unknown from LDAP + # LDAP does not provide users id, therefore we rely on email to find the + # user in LDAP - if users_by_email : + if users_by_email: #Construct the list of filters (list of dicts) for GetPersons - filter_user = [] - for email in users_by_email : - filter_user.append (users_by_email[email]) - #Check user's in LDAP with GetPersons + filter_user = [users_by_email[email] for email in users_by_email] + #Check user i in LDAP with GetPersons #Needed because what if the user has been deleted in LDAP but #is still in SFA? existing_users = self.driver.iotlab_api.GetPersons(filter_user) logger.debug(" \r\n SLABSLICE.PY \tverify_person filter_user \ - %s existing_users %s " \ - %(filter_user, existing_users)) - #User's in iotlab LDAP + %s existing_users %s " + % (filter_user, existing_users)) + #User is in iotlab LDAP if existing_users: - for user in existing_users : + for user in existing_users: users_dict[user['email']].update(user) - existing_user_emails.append(\ - users_dict[user['email']]['email']) + existing_user_emails.append( + users_dict[user['email']]['email']) # User from another known trusted federated site. Check @@ -415,52 +420,39 @@ class IotlabSlices: else: req = 'mail=' if isinstance(users, list): - req += users[0]['email'] else: req += users['email'] - ldap_reslt = self.driver.iotlab_api.ldap.LdapSearch(req) if ldap_reslt: logger.debug(" SLABSLICE.PY \tverify_person users \ USER already in Iotlab \t ldap_reslt %s \ - "%( ldap_reslt)) + " % (ldap_reslt)) existing_users.append(ldap_reslt[1]) else: #User not existing in LDAP - #TODO SA 21/08/12 raise smthg to add user or add it auto ? - #new_record = {} - #new_record['pkey'] = users[0]['keys'][0] - #new_record['mail'] = users[0]['email'] - - logger.debug(" SLABSLICE.PY \tverify_person users \ + logger.debug("SLABSLICE.PY \tverify_person users \ not in ldap ...NEW ACCOUNT NEEDED %s \r\n \t \ - ldap_reslt %s " %(users, ldap_reslt)) + ldap_reslt %s " % (users, ldap_reslt)) requested_user_emails = users_by_email.keys() requested_user_hrns = \ - [users_by_email[user]['hrn'] for user in users_by_email] + [users_by_email[user]['hrn'] for user in users_by_email] logger.debug("SLABSLICE.PY \tverify_person \ - users_by_email %s " %( users_by_email)) - #logger.debug("SLABSLICE.PY \tverify_person \ - #user_by_hrn %s " %( users_by_hrn)) - + users_by_email %s " % (users_by_email)) #Check that the user of the slice in the slice record #matches one of the existing users try: if slice_record['PI'][0] in requested_user_hrns: - #if slice_record['record_id_user'] in requested_user_ids and \ - #slice_record['PI'][0] in requested_user_hrns: logger.debug(" SLABSLICE \tverify_person ['PI']\ - slice_record %s" %(slice_record)) + slice_record %s" % (slice_record)) except KeyError: pass - # users to be added, removed or updated #One user in one iotlab slice : there should be no need #to remove/ add any user from/to a slice. @@ -475,23 +467,19 @@ class IotlabSlices: added_persons = [] # add new users - #requested_user_email is in existing_user_emails if len(added_user_emails) == 0: - slice_record['login'] = users_dict[requested_user_emails[0]]['uid'] - logger.debug(" SLABSLICE \tverify_person QUICK DIRTY %s" \ - %(slice_record)) - + logger.debug(" SLABSLICE \tverify_person QUICK DIRTY %s" + % (slice_record)) for added_user_email in added_user_emails: - #hrn, type = urn_to_hrn(added_user['urn']) added_user = users_dict[added_user_email] logger.debug(" SLABSLICE \r\n \r\n \t THE SECOND verify_person \ - added_user %s" %(added_user)) + added_user %s" % (added_user)) person = {} - person['peer_person_id'] = None - k_list = ['first_name', 'last_name','person_id'] + person['peer_person_id'] = None + k_list = ['first_name', 'last_name', 'person_id'] for k in k_list: if k in added_user: person[k] = added_user[k] @@ -499,26 +487,25 @@ class IotlabSlices: person['pkey'] = added_user['keys'][0] person['mail'] = added_user['email'] person['email'] = added_user['email'] - person['key_ids'] = added_user.get('key_ids', []) - #person['urn'] = added_user['urn'] + person['key_ids'] = added_user.get('key_ids', []) - #person['person_id'] = self.driver.iotlab_api.AddPerson(person) ret = self.driver.iotlab_api.AddPerson(person) - if type(ret) == int : + if type(ret) == int: person['uid'] = ret logger.debug(" SLABSLICE \r\n \r\n \t THE SECOND verify_person\ - personne %s" %(person)) + person %s" % (person)) #Update slice_Record with the id now known to LDAP slice_record['login'] = person['uid'] added_persons.append(person) - - return added_persons - #Unused + def verify_keys(self, persons, users, peer, options={}): + """ + .. warning:: unused + """ # existing keys key_ids = [] for person in persons: -- 2.47.0