From 53efc8dd9d863b39ece46813093fe5ac2f3c3252 Mon Sep 17 00:00:00 2001 From: Sandrine Avakian Date: Thu, 9 Jan 2014 17:20:06 +0100 Subject: [PATCH] Porting cortexlab to geni-v3. May try to factor the code in the future, as Iotlab and Cortexlab drivers are similar. --- sfa/cortexlab/cortexlabaggregate.py | 617 ++++++++++--- sfa/cortexlab/cortexlabdriver.py | 1236 +++++++++++++++++++++------ sfa/cortexlab/cortexlabshell.py | 758 ++-------------- 3 files changed, 1555 insertions(+), 1056 deletions(-) diff --git a/sfa/cortexlab/cortexlabaggregate.py b/sfa/cortexlab/cortexlabaggregate.py index f58acc66..d9cddf3b 100644 --- a/sfa/cortexlab/cortexlabaggregate.py +++ b/sfa/cortexlab/cortexlabaggregate.py @@ -3,55 +3,31 @@ File providing methods to generate valid RSpecs for the Iotlab testbed. Contains methods to get information on slice, slivers, nodes and leases, formatting them and turn it into a RSpec. """ -from sfa.util.xrn import hrn_to_urn, urn_to_hrn, get_authority +from sfa.util.xrn import hrn_to_urn, urn_to_hrn +from sfa.util.sfatime import utcparse, datetime_to_string +from sfa.iotlab.iotlabxrn import IotlabXrn, xrn_object from sfa.rspecs.rspec import RSpec + #from sfa.rspecs.elements.location import Location from sfa.rspecs.elements.hardware_type import HardwareType +from sfa.rspecs.elements.node import NodeElement from sfa.rspecs.elements.login import Login -from sfa.rspecs.elements.services import Services from sfa.rspecs.elements.sliver import Sliver from sfa.rspecs.elements.lease import Lease from sfa.rspecs.elements.granularity import Granularity from sfa.rspecs.version_manager import VersionManager - +from sfa.storage.model import SliverAllocation from sfa.rspecs.elements.versions.iotlabv1Node import IotlabPosition, \ - IotlabNode, IotlabLocation, IotlabMobility + IotlabLocation from sfa.util.sfalogging import logger from sfa.util.xrn import Xrn - -def cortexlab_xrn_to_hostname(xrn): - """Returns a node's hostname from its xrn. - :param xrn: The nodes xrn identifier. - :type xrn: Xrn (from sfa.util.xrn) - - :returns: node's hostname. - :rtype: string - - """ - return Xrn.unescape(Xrn(xrn=xrn, type='node').get_leaf()) - - -def cortexlab_xrn_object(root_auth, hostname): - """Creates a valid xrn object from the node's hostname and the authority - of the SFA server. - - :param hostname: the node's hostname. - :param root_auth: the SFA root authority. - :type hostname: string - :type root_auth: string - - :returns: the cortexlab node's xrn - :rtype: Xrn - - """ - return Xrn('.'.join([root_auth, Xrn.escape(hostname)]), type='node') - +import time class CortexlabAggregate: - """Aggregate manager class for Iotlab. """ + """Aggregate manager class for cortexlab. """ sites = {} nodes = {} @@ -98,14 +74,13 @@ class CortexlabAggregate: return (sfa_slice, slivers) slice_urn = hrn_to_urn(slice_xrn, 'slice') slice_hrn, _ = urn_to_hrn(slice_xrn) - slice_name = slice_hrn # GetSlices always returns a list, even if there is only one element - slices = self.driver.testbed_shell.GetSlices(slice_filter=str(slice_name), + slices = self.driver.GetSlices(slice_filter=str(slice_hrn), slice_filter_type='slice_hrn', login=login) - logger.debug("IotlabAggregate api \tget_slice_and_slivers \ + logger.debug("CortexlabAggregate api \tget_slice_and_slivers \ slice_hrn %s \r\n slices %s self.driver.hrn %s" % (slice_hrn, slices, self.driver.hrn)) if slices == []: @@ -152,6 +127,17 @@ class CortexlabAggregate: def find_ldap_username_from_slice(self, sfa_slice): + """ + Gets the ldap username of the user based on the information contained + in ist sfa_slice record. + + :param sfa_slice: the user's slice record. Must contain the + reg_researchers key. + :type sfa_slice: dictionary + :returns: ldap_username, the ldap user's login. + :rtype: string + """ + researchers = [sfa_slice['reg_researchers'][0].__dict__] # look in ldap: ldap_username = None @@ -163,7 +149,7 @@ class CortexlabAggregate: - def get_nodes(self, slices=None, slivers=[], options=None): + def get_nodes(self, options=None): """Returns the nodes in the slice using the rspec format, with all the nodes' properties. @@ -172,10 +158,6 @@ class CortexlabAggregate: it. If the slice does not have any job running or scheduled, that is it has no reserved nodes, then returns an empty list. - :param slices: list of slices (record dictionaries) - :param slivers: the list of slivers in all the slices - :type slices: list of dicts - :type slivers: list of Sliver object (dictionaries) :returns: An empty list if the slice has no reserved nodes, a rspec list with all the nodes and their properties (a dict per node) otherwise. @@ -184,98 +166,224 @@ class CortexlabAggregate: .. seealso:: get_slice_and_slivers """ - # NT: the semantic of this function is not clear to me : - # if slice is not defined, then all the nodes should be returned - # if slice is defined, we should return only the nodes that - # are part of this slice - # but what is the role of the slivers parameter ? - # So i assume that slice['node_ids'] will be the same as slivers for us - slice_nodes_list = [] - if slices is not None: - for one_slice in slices: - try: - slice_nodes_list = one_slice['node_ids'] - # if we are dealing with a slice that has no node just - # return an empty list. In cortexlab a slice can have multiple - # jobs scheduled, so it either has at least one lease or - # not at all. - except KeyError: - return [] + filter_nodes = None + if options: + geni_available = options.get('geni_available') + if geni_available == True: + filter_nodes['boot_state'] = ['Alive'] + + # slice_nodes_list = [] + # if slices is not None: + # for one_slice in slices: + # try: + # slice_nodes_list = one_slice['node_ids'] + # # if we are dealing with a slice that has no node just + # # return an empty list. In iotlab a slice can have multiple + # # jobs scheduled, so it either has at least one lease or + # # not at all. + # except KeyError: + # return [] # get the granularity in second for the reservation system - grain = self.driver.testbed_shell.GetLeaseGranularity() + # grain = self.driver.testbed_shell.GetLeaseGranularity() - nodes = self.driver.testbed_shell.GetNodes() + nodes = self.driver.testbed_shell.GetNodes(node_filter_dict = + filter_nodes) nodes_dict = {} #if slices, this means we got to list all the nodes given to this slice # Make a list of all the nodes in the slice before getting their #attributes - rspec_nodes = [] - logger.debug("CortexlabAggregate api get_nodes slices %s " - % (slices)) - - reserved_nodes = self.driver.testbed_shell.GetNodesCurrentlyInUse() - logger.debug("CortexlabAggregate api get_nodes slice_nodes_list %s " - % (slice_nodes_list)) for node in nodes: nodes_dict[node['node_id']] = node - if slice_nodes_list == [] or node['hostname'] in slice_nodes_list: - rspec_node = Node() + return nodes_dict + + + def node_to_rspec_node(self, node): + """ Creates a rspec node structure with the appropriate information + based on the node information that can be found in the node dictionary. + :param node: node data. this dict contains information about the node + and must have the following keys : mobile, radio, archi, hostname, + boot_state, site, x, y ,z (position). + :type node: dictionary. + :returns: node dictionary containing the following keys : mobile, archi, + radio, component_id, component_name, component_manager_id, + authority_id, boot_state, exclusive, hardware_types, location, + position, granularity, tags. + :rtype: dict + + """ - cortexlab_xrn = cortexlab_xrn_object(self.driver.testbed_shell.root_auth, + grain = self.driver.testbed_shell.GetLeaseGranularity() + rspec_node = NodeElement() + + # xxx how to retrieve site['login_base'] + #site_id=node['site_id'] + #site=sites_dict[site_id] + + rspec_node['mobile'] = node['mobile'] + rspec_node['archi'] = node['archi'] + rspec_node['radio'] = node['radio'] + cortexlab_xrn = xrn_object(self.driver.testbed_shell.root_auth, node['hostname']) - rspec_node['component_id'] = cortexlab_xrn.urn - rspec_node['component_name'] = node['hostname'] - rspec_node['component_manager_id'] = \ - hrn_to_urn(self.driver.testbed_shell.root_auth, - 'authority+sa') - - # Iotlab's nodes are federated : there is only one authority - # for all Iotlab sites, registered in SFA. - # Removing the part including the site - # in authority_id SA 27/07/12 - rspec_node['authority_id'] = rspec_node['component_manager_id'] - - - # boot state removed if you need it uncomment - # rspec_node['boot_state'] = node['boot_state'] - # if node['hostname'] in reserved_nodes: - # rspec_node['boot_state'] = "Reserved" - - rspec_node['exclusive'] = 'true' - rspec_node['hardware_types'] = [HardwareType({'name': - 'cortexlab-node'})] - - - # Location, mobility and position removed. If you need it go check - # get_nodes in iotlabaggregate.py - - # Granularity - granularity = Granularity({'grain': grain}) - rspec_node['granularity'] = granularity - rspec_node['tags'] = [] - if node['hostname'] in slivers: - # add sliver info - sliver = slivers[node['hostname']] - rspec_node['sliver_id'] = sliver['sliver_id'] - rspec_node['client_id'] = node['hostname'] - rspec_node['slivers'] = [sliver] - - # slivers always provide the ssh service - login = Login({'authentication': 'ssh-keys', - 'hostname': node['hostname'], 'port': '22', - 'username': sliver['name']}) - service = Services({'login': login}) - rspec_node['services'] = [service] - rspec_nodes.append(rspec_node) - return (rspec_nodes) + rspec_node['component_id'] = cortexlab_xrn.urn + rspec_node['component_name'] = node['hostname'] + rspec_node['component_manager_id'] = \ + hrn_to_urn(self.driver.testbed_shell.root_auth, + 'authority+sa') + + # Iotlab's nodes are federated : there is only one authority + # for all Iotlab sites, registered in SFA. + # Removing the part including the site + # in authority_id SA 27/07/12 + rspec_node['authority_id'] = rspec_node['component_manager_id'] + + # do not include boot state ( element) + #in the manifest rspec + + + rspec_node['boot_state'] = node['boot_state'] + # if node['hostname'] in reserved_nodes: + # rspec_node['boot_state'] = "Reserved" + rspec_node['exclusive'] = 'true' + rspec_node['hardware_types'] = [HardwareType({'name': \ + 'iotlab-node'})] + + location = IotlabLocation({'country':'France', 'site': \ + node['site']}) + rspec_node['location'] = location + + + position = IotlabPosition() + for field in position : + try: + position[field] = node[field] + except KeyError, error : + logger.log_exc("Cortexlabaggregate\t node_to_rspec_node \ + position %s "% (error)) + + rspec_node['position'] = position + + + # Granularity + granularity = Granularity({'grain': grain}) + rspec_node['granularity'] = granularity + rspec_node['tags'] = [] + # if node['hostname'] in slivers: + # # add sliver info + # sliver = slivers[node['hostname']] + # rspec_node['sliver_id'] = sliver['sliver_id'] + # rspec_node['client_id'] = node['hostname'] + # rspec_node['slivers'] = [sliver] + + # # slivers always provide the ssh service + # login = Login({'authentication': 'ssh-keys', \ + # 'hostname': node['hostname'], 'port':'22', \ + # 'username': sliver['name']}) + # service = Services({'login': login}) + # rspec_node['services'] = [service] + + return rspec_node + + + def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations = {}): + """Makes a geni sliver structure from all the nodes allocated + to slivers in the sliver_allocations dictionary. Returns the states + of the sliver. + + :param rspec_node: Node information contained in a rspec data structure + fashion. + :type rspec_node: dictionary + :param sliver_allocations: + :type sliver_allocations: dictionary + + :returns: Dictionary with the following keys: geni_sliver_urn, + geni_expires, geni_allocation_status, geni_operational_status, + geni_error. + + :rtype: dictionary + + .. seealso:: node_to_rspec_node + + """ + if rspec_node['sliver_id'] in sliver_allocations: + # set sliver allocation and operational status + sliver_allocation = sliver_allocations[rspec_node['sliver_id']] + if sliver_allocation: + allocation_status = sliver_allocation.allocation_state + if allocation_status == 'geni_allocated': + op_status = 'geni_pending_allocation' + elif allocation_status == 'geni_provisioned': + op_status = 'geni_ready' + else: + op_status = 'geni_unknown' + else: + allocation_status = 'geni_unallocated' + else: + allocation_status = 'geni_unallocated' + op_status = 'geni_failed' + # required fields + geni_sliver = {'geni_sliver_urn': rspec_node['sliver_id'], + 'geni_expires': rspec_node['expires'], + 'geni_allocation_status' : allocation_status, + 'geni_operational_status': op_status, + 'geni_error': '', + } + return geni_sliver + + def sliver_to_rspec_node(self, sliver, sliver_allocations): + """Used by describe to format node information into a rspec compliant + structure. + + Creates a node rspec compliant structure by calling node_to_rspec_node. + Adds slivers, if any, to rspec node structure. Returns the updated + rspec node struct. + + :param sliver: sliver dictionary. Contains keys: urn, slice_id, hostname + and slice_name. + :type sliver: dictionary + :param sliver_allocations: dictionary of slivers + :type sliver_allocations: dict + + :returns: Node dictionary with all necessary data. + + .. seealso:: node_to_rspec_node + """ + rspec_node = self.node_to_rspec_node(sliver) + rspec_node['expires'] = datetime_to_string(utcparse(sliver['expires'])) + # add sliver info + logger.debug("CORTEXLABAGGREGATE api \t sliver_to_rspec_node sliver \ + %s \r\nsliver_allocations %s" % (sliver, + sliver_allocations)) + rspec_sliver = Sliver({'sliver_id': sliver['urn'], + 'name': sliver['slice_id'], + 'type': 'iotlab-exclusive', + 'tags': []}) + rspec_node['sliver_id'] = rspec_sliver['sliver_id'] + + if sliver['urn'] in sliver_allocations: + rspec_node['client_id'] = sliver_allocations[ + sliver['urn']].client_id + if sliver_allocations[sliver['urn']].component_id: + rspec_node['component_id'] = sliver_allocations[ + sliver['urn']].component_id + rspec_node['slivers'] = [rspec_sliver] + + # slivers always provide the ssh service + login = Login({'authentication': 'ssh-keys', + 'hostname': sliver['hostname'], + 'port':'22', + 'username': sliver['slice_name'], + 'login': sliver['slice_name'] + }) + return rspec_node + def get_all_leases(self, ldap_username): """ @@ -285,10 +393,13 @@ class CortexlabAggregate: All the leases running or scheduled are returned. :param ldap_username: if ldap uid is not None, looks for the leases - belonging to this user. + belonging to this user. :type ldap_username: string :returns: rspec lease dictionary with keys lease_id, component_id, - slice_id, start_time, duration. + slice_id, start_time, duration where the lease_id is the oar job id, + component_id is the node's urn, slice_id is the slice urn, + start_time is the timestamp starting time and duration is expressed + in terms of the testbed's granularity. :rtype: dict .. note::There is no filtering of leases within a given time frame. @@ -298,14 +409,9 @@ class CortexlabAggregate: """ - #now = int(time.time()) - #lease_filter = {'clip': now } - - - logger.debug("CortexlabAggregate get_all_leases ldap_username %s " % (ldap_username)) - leases = self.driver.testbed_shell.GetLeases(login=ldap_username) + leases = self.driver.driver.GetLeases(login=ldap_username) grain = self.driver.testbed_shell.GetLeaseGranularity() # site_ids = [] rspec_leases = [] @@ -315,7 +421,7 @@ class CortexlabAggregate: rspec_lease = Lease() rspec_lease['lease_id'] = lease['lease_id'] - cortexlab_xrn = cortexlab_xrn_object( + cortexlab_xrn = xrn_object( self.driver.testbed_shell.root_auth, node) rspec_lease['component_id'] = cortexlab_xrn.urn #rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn,\ @@ -393,7 +499,7 @@ class CortexlabAggregate: slice_xrn %s slices %s\r\n \r\n" % (slice_xrn, slices)) - if options is not None and 'list_leases' in options: + if options is not None : lease_option = options['list_leases'] else: #If no options are specified, at least print the resources @@ -404,9 +510,7 @@ class CortexlabAggregate: if lease_option in ['all', 'resources']: #if not options.get('list_leases') or options.get('list_leases') #and options['list_leases'] != 'leases': - nodes = self.get_nodes(slices, slivers) - if slice_xrn and slices is None: - nodes = [] + nodes = self.get_nodes() logger.debug("\r\n") logger.debug("CortexlabAggregate \t lease_option %s \ get rspec ******* nodes %s" @@ -428,7 +532,8 @@ class CortexlabAggregate: logger.debug("CortexlabAggregate \tget_rspec **** \ version type %s ldap_ user %s \r\n" \ % (version.type, ldap_username)) - if version.type == "Iotlab": + #TODO : Change the version of Rspec here in case of pbm -SA 09/01/14 + if version.type in ["Cortexlab", "Iotlab"]: rspec.version.add_connection_information( ldap_username, sites_set) @@ -447,3 +552,257 @@ class CortexlabAggregate: logger.debug("CortexlabAggregate \tget_rspec **** \ FINAL RSPEC %s \r\n" % (rspec.toxml())) return rspec.toxml() + + + + def get_slivers(self, urns, options={}): + """Get slivers of the given slice urns. Slivers contains slice, node and + user information. + + For Iotlab, returns the leases with sliver ids and their allocation + status. + + :param urns: list of slice urns. + :type urns: list of strings + :param options: unused + :type options: unused + + .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3/CommonConcepts#urns + """ + + + slice_ids = set() + node_ids = [] + for urn in urns: + xrn = IotlabXrn(xrn=urn) + if xrn.type == 'sliver': + # id: slice_id-node_id + try: + sliver_id_parts = xrn.get_sliver_id_parts() + slice_id = int(sliver_id_parts[0]) + node_id = int(sliver_id_parts[1]) + slice_ids.add(slice_id) + node_ids.append(node_id) + except ValueError: + pass + else: + slice_names = set() + slice_names.add(xrn.hrn) + + + logger.debug("CortexlabAggregate \t get_slivers urns %s slice_ids %s \ + node_ids %s\r\n" % (urns, slice_ids, node_ids)) + logger.debug("CortexlabAggregate \t get_slivers xrn %s slice_names %s \ + \r\n" % (xrn, slice_names)) + filter_sliver = {} + if slice_names: + filter_sliver['slice_hrn'] = list(slice_names) + slice_hrn = filter_sliver['slice_hrn'][0] + + slice_filter_type = 'slice_hrn' + + # if slice_ids: + # filter['slice_id'] = list(slice_ids) + # # get slices + if slice_hrn: + slices = self.driver.GetSlices(slice_hrn, + slice_filter_type) + leases = self.driver.GetLeases({'slice_hrn':slice_hrn}) + logger.debug("CortexlabAggregate \t get_slivers \ + slices %s leases %s\r\n" % (slices, leases )) + if not slices: + return [] + + single_slice = slices[0] + # get sliver users + user = single_slice['reg_researchers'][0].__dict__ + logger.debug("CortexlabAggregate \t get_slivers user %s \ + \r\n" % (user)) + + # construct user key info + person = self.driver.testbed_shell.ldap.LdapFindUser(record=user) + logger.debug("CortexlabAggregate \t get_slivers person %s \ + \r\n" % (person)) + # name = person['last_name'] + user['login'] = person['uid'] + user['user_urn'] = hrn_to_urn(user['hrn'], 'user') + user['keys'] = person['pkey'] + + + try: + node_ids = single_slice['node_ids'] + node_list = self.driver.testbed_shell.GetNodes( + {'hostname':single_slice['node_ids']}) + node_by_hostname = dict([(node['hostname'], node) + for node in node_list]) + except KeyError: + logger.warning("\t get_slivers No slivers in slice") + # slice['node_ids'] = node_ids + # nodes_dict = self.get_slice_nodes(slice, options) + + slivers = [] + for current_lease in leases: + for hostname in current_lease['reserved_nodes']: + node = {} + node['slice_id'] = current_lease['slice_id'] + node['slice_hrn'] = current_lease['slice_hrn'] + slice_name = current_lease['slice_hrn'].split(".")[1] + node['slice_name'] = slice_name + index = current_lease['reserved_nodes'].index(hostname) + node_id = current_lease['resource_ids'][index] + # node['slice_name'] = user['login'] + # node.update(single_slice) + more_info = node_by_hostname[hostname] + node.update(more_info) + # oar_job_id is the slice_id (lease_id) + sliver_hrn = '%s.%s-%s' % (self.driver.hrn, + current_lease['lease_id'], node_id) + node['node_id'] = node_id + node['expires'] = current_lease['t_until'] + node['sliver_id'] = Xrn(sliver_hrn, type='sliver').urn + node['urn'] = node['sliver_id'] + node['services_user'] = [user] + + slivers.append(node) + return slivers + + + def list_resources(self, version = None, options={}): + """ + Returns an advertisement Rspec of available resources at this + aggregate. This Rspec contains a resource listing along with their + description, providing sufficient information for clients to be able to + select among available resources. + + :param options: various options. The valid options are: {boolean + geni_compressed ; struct geni_rspec_version { string type; + #case insensitive , string version; # case insensitive}} . The only + mandatory options if options is specified is geni_rspec_version. + :type options: dictionary + + :returns: On success, the value field of the return struct will contain + a geni.rspec advertisment RSpec + :rtype: Rspec advertisement in xml. + + .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3/CommonConcepts#RSpecdatatype + .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3#ListResources + """ + + version_manager = VersionManager() + version = version_manager.get_version(version) + rspec_version = version_manager._get_version(version.type, + version.version, 'ad') + rspec = RSpec(version=rspec_version, user_options=options) + # variable ldap_username to be compliant with get_all_leases + # prototype. Now unused in geni-v3 since we are getting all the leases + # here + ldap_username = None + if not options.get('list_leases') or options['list_leases'] != 'leases': + # get nodes + nodes_dict = self.get_nodes(options) + + # no interfaces on iotlab nodes + # convert nodes to rspec nodes + rspec_nodes = [] + for node_id in nodes_dict: + node = nodes_dict[node_id] + rspec_node = self.node_to_rspec_node(node) + rspec_nodes.append(rspec_node) + rspec.version.add_nodes(rspec_nodes) + + # add links + # links = self.get_links(sites, nodes_dict, interfaces) + # rspec.version.add_links(links) + + if not options.get('list_leases') or options.get('list_leases') \ + and options['list_leases'] != 'resources': + leases = self.get_all_leases(ldap_username) + rspec.version.add_leases(leases) + + return rspec.toxml() + + + def describe(self, urns, version=None, options={}): + """ + Retrieve a manifest RSpec describing the resources contained by the + named entities, e.g. a single slice or a set of the slivers in a slice. + This listing and description should be sufficiently descriptive to allow + experimenters to use the resources. + + :param urns: If a slice urn is supplied and there are no slivers in the + given slice at this aggregate, then geni_rspec shall be a valid + manifest RSpec, containing no node elements - no resources. + :type urns: list or strings + :param options: various options. the valid options are: {boolean + geni_compressed ; struct geni_rspec_version { string type; + #case insensitive , string version; # case insensitive}} + :type options: dictionary + + :returns: On success returns the following dictionary {geni_rspec: + , geni_urn: , geni_slivers:{ geni_sliver_urn: + , geni_expires: , + geni_allocation_status: , geni_operational_status: + , geni_error: } + + .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3#Describe + .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3/CommonConcepts#urns + """ + version_manager = VersionManager() + version = version_manager.get_version(version) + rspec_version = version_manager._get_version( + version.type, version.version, 'manifest') + rspec = RSpec(version=rspec_version, user_options=options) + + # get slivers + geni_slivers = [] + slivers = self.get_slivers(urns, options) + if slivers: + rspec_expires = datetime_to_string(utcparse(slivers[0]['expires'])) + else: + rspec_expires = datetime_to_string(utcparse(time.time())) + rspec.xml.set('expires', rspec_expires) + + # lookup the sliver allocations + geni_urn = urns[0] + sliver_ids = [sliver['sliver_id'] for sliver in slivers] + logger.debug(" Cortexlabaggregate.PY \tDescribe sliver_ids %s " + % (sliver_ids)) + constraint = SliverAllocation.sliver_id.in_(sliver_ids) + query = self.driver.api.dbsession().query(SliverAllocation) + sliver_allocations = query.filter((constraint)).all() + logger.debug(" Cortexlabaggregate.PY \tDescribe sliver_allocations %s " + % (sliver_allocations)) + sliver_allocation_dict = {} + for sliver_allocation in sliver_allocations: + geni_urn = sliver_allocation.slice_urn + sliver_allocation_dict[sliver_allocation.sliver_id] = \ + sliver_allocation + + # add slivers + nodes_dict = {} + for sliver in slivers: + nodes_dict[sliver['node_id']] = sliver + rspec_nodes = [] + for sliver in slivers: + rspec_node = self.sliver_to_rspec_node(sliver, + sliver_allocation_dict) + rspec_nodes.append(rspec_node) + logger.debug(" Cortexlabaggregate.PY \tDescribe sliver_allocation_dict %s " + % (sliver_allocation_dict)) + geni_sliver = self.rspec_node_to_geni_sliver(rspec_node, + sliver_allocation_dict) + geni_slivers.append(geni_sliver) + + logger.debug(" Cortexlabaggregate.PY \tDescribe rspec_nodes %s\ + rspec %s " + % (rspec_nodes, rspec)) + rspec.version.add_nodes(rspec_nodes) + + return {'geni_urn': geni_urn, + 'geni_rspec': rspec.toxml(), + 'geni_slivers': geni_slivers} \ No newline at end of file diff --git a/sfa/cortexlab/cortexlabdriver.py b/sfa/cortexlab/cortexlabdriver.py index 5fc26049..cd027655 100644 --- a/sfa/cortexlab/cortexlabdriver.py +++ b/sfa/cortexlab/cortexlabdriver.py @@ -1,25 +1,35 @@ """ Implements what a driver should provide for SFA to work. """ -from sfa.util.faults import SliverDoesNotExist, UnknownSfaType +from datetime import datetime +from sfa.util.sfatime import utcparse, datetime_to_string + +from sfa.util.faults import SliverDoesNotExist, Forbidden from sfa.util.sfalogging import logger -from sfa.storage.alchemy import dbsession -from sfa.storage.model import RegRecord + +from sfa.trust.hierarchy import Hierarchy +from sfa.trust.gid import create_uuid from sfa.managers.driver import Driver from sfa.rspecs.version_manager import VersionManager from sfa.rspecs.rspec import RSpec -from sfa.util.xrn import Xrn, hrn_to_urn, get_authority - from sfa.cortexlab.cortexlabaggregate import CortexlabAggregate, \ cortexlab_xrn_to_hostname from sfa.cortexlab.cortexlabslices import CortexlabSlices - from sfa.cortexlab.cortexlabshell import CortexlabShell +from sfa.iotlab.iotlabxrn import IotlabXrn, xrn_object +from sfa.util.xrn import Xrn, hrn_to_urn, get_authority, urn_to_hrn +from sfa.trust.certificate import Keypair, convert_public_key +from sfa.trust.credential import Credential +from sfa.storage.model import SliverAllocation + +from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey +from sfa.iotlab.iotlabpostgres import LeaseTableXP +from sqlalchemy.orm import joinedload class CortexlabDriver(Driver): """ Cortexlab Driver class inherited from Driver generic class. @@ -30,21 +40,681 @@ class CortexlabDriver(Driver): .. seealso::: Driver class """ - def __init__(self, config): + def __init__(self, api): """ - Sets the iotlab SFA config parameters, - instanciates the testbed api and the iotlab database. - - :param config: iotlab SFA configuration object - :type config: Config object + Sets the Cortexlab SFA config parameters, + instanciates the testbed api. + :param api: SfaApi configuration object. Holds reference to the + database. + :type api: SfaApi object """ - Driver.__init__(self, config) - self.config = config + + Driver.__init__(self, api) + self.api = api + config = api.config self.testbed_shell = CortexlabShell(config) self.cache = None + def GetPeers(self, peer_filter=None ): + """ Gathers registered authorities in SFA DB and looks for specific peer + if peer_filter is specified. + :param peer_filter: name of the site authority looked for. + :type peer_filter: string + :returns: list of records. + + """ + + existing_records = {} + existing_hrns_by_types = {} + logger.debug("CORTEXLAB_API \tGetPeers peer_filter %s " % (peer_filter)) + query = self.api.dbsession().query(RegRecord) + all_records = query.filter(RegRecord.type.like('%authority%')).all() + + for record in all_records: + existing_records[(record.hrn, record.type)] = record + if record.type not in existing_hrns_by_types: + existing_hrns_by_types[record.type] = [record.hrn] + else: + existing_hrns_by_types[record.type].append(record.hrn) + + logger.debug("CORTEXLAB_API \tGetPeer\texisting_hrns_by_types %s " + % (existing_hrns_by_types)) + records_list = [] + + try: + if peer_filter: + records_list.append(existing_records[(peer_filter, + 'authority')]) + else: + for hrn in existing_hrns_by_types['authority']: + records_list.append(existing_records[(hrn, 'authority')]) + + logger.debug("CORTEXLAB_API \tGetPeer \trecords_list %s " + % (records_list)) + + except KeyError: + pass + + return_records = records_list + logger.debug("CORTEXLAB_API \tGetPeer return_records %s " + % (return_records)) + return return_records + + def GetKeys(self, key_filter=None): + """Returns a dict of dict based on the key string. Each dict entry + contains the key id, the ssh key, the user's email and the + user's hrn. + If key_filter is specified and is an array of key identifiers, + only keys matching the filter will be returned. + + Admin may query all keys. Non-admins may only query their own keys. + FROM PLC API DOC + + :returns: dict with ssh key as key and dicts as value. + :rtype: dict + """ + query = self.api.dbsession().query(RegKey) + if key_filter is None: + keys = query.options(joinedload('reg_user')).all() + else: + constraint = RegKey.key.in_(key_filter) + keys = query.options(joinedload('reg_user')).filter(constraint).all() + + key_dict = {} + for key in keys: + key_dict[key.key] = {'key_id': key.key_id, 'key': key.key, + 'email': key.reg_user.email, + 'hrn': key.reg_user.hrn} + + #ldap_rslt = self.ldap.LdapSearch({'enabled']=True}) + #user_by_email = dict((user[1]['mail'][0], user[1]['sshPublicKey']) \ + #for user in ldap_rslt) + + logger.debug("CORTEXLAB_API GetKeys -key_dict %s \r\n " % (key_dict)) + return key_dict + + def AddPerson(self, record): + """ + + Adds a new account. Any fields specified in records are used, + otherwise defaults are used. Creates an appropriate login by calling + LdapAddUser. + + :param record: dictionary with the sfa user's properties. + :returns: a dicitonary with the status. If successful, the dictionary + boolean is set to True and there is a 'uid' key with the new login + added to LDAP, otherwise the bool is set to False and a key + 'message' is in the dictionary, with the error message. + :rtype: dict + + """ + ret = self.testbed_shell.ldap.LdapAddUser(record) + + if ret['bool'] is True: + record['hrn'] = self.testbed_shell.root_auth + '.' + ret['uid'] + logger.debug("Cortexlab api AddPerson return code %s record %s " + % (ret, record)) + self.__add_person_to_db(record) + return ret + + def __add_person_to_db(self, user_dict): + """ + Add a federated user straight to db when the user issues a lease + request with nodes and that he has not registered with cortexlab + yet (that is he does not have a LDAP entry yet). + Uses parts of the routines in CortexlabImport when importing user + from LDAP. + Called by AddPerson, right after LdapAddUser. + :param user_dict: Must contain email, hrn and pkey to get a GID + and be added to the SFA db. + :type user_dict: dict + + """ + request = self.api.dbsession().query(RegUser) + check_if_exists = \ + request.filter_by(email = user_dict['email']).first() + #user doesn't exists + if not check_if_exists: + logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \ + " %(user_dict)) + hrn = user_dict['hrn'] + person_urn = hrn_to_urn(hrn, 'user') + pubkey = user_dict['pkey'] + try: + pkey = convert_public_key(pubkey) + except TypeError: + #key not good. create another pkey + logger.warn('__add_person_to_db: unable to convert public \ + key for %s' %(hrn )) + pkey = Keypair(create=True) + + + if pubkey is not None and pkey is not None : + hierarchy = Hierarchy() + person_gid = hierarchy.create_gid(person_urn, create_uuid(), \ + pkey) + if user_dict['email']: + logger.debug("__add_person_to_db \r\n \r\n \ + IOTLAB IMPORTER PERSON EMAIL OK email %s "\ + %(user_dict['email'])) + person_gid.set_email(user_dict['email']) + + user_record = RegUser(hrn=hrn , pointer= '-1', \ + authority=get_authority(hrn), \ + email=user_dict['email'], gid = person_gid) + user_record.reg_keys = [RegKey(user_dict['pkey'])] + user_record.just_created() + self.api.dbsession().add (user_record) + self.api.dbsession().commit() + return + + + def _sql_get_slice_info(self, slice_filter): + """ + Get the slice record based on the slice hrn. Fetch the record of the + user associated with the slice by using joinedload based on the + reg_researcher relationship. + + :param slice_filter: the slice hrn we are looking for + :type slice_filter: string + :returns: the slice record enhanced with the user's information if the + slice was found, None it wasn't. + + :rtype: dict or None. + """ + #DO NOT USE RegSlice - reg_researchers to get the hrn + #of the user otherwise will mess up the RegRecord in + #Resolve, don't know why - SA 08/08/2012 + + #Only one entry for one user = one slice in testbed_xp table + #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() + request = self.api.dbsession().query(RegSlice) + raw_slicerec = request.options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first() + #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() + if raw_slicerec: + #load_reg_researcher + #raw_slicerec.reg_researchers + raw_slicerec = raw_slicerec.__dict__ + logger.debug(" CORTEXLAB_API \t _sql_get_slice_info slice_filter %s \ + raw_slicerec %s" % (slice_filter, raw_slicerec)) + slicerec = raw_slicerec + #only one researcher per slice so take the first one + #slicerec['reg_researchers'] = raw_slicerec['reg_researchers'] + #del slicerec['reg_researchers']['_sa_instance_state'] + return slicerec + + else: + return None + + def _sql_get_slice_info_from_user(self, slice_filter): + """ + Get the slice record based on the user recordid by using a joinedload + on the relationship reg_slices_as_researcher. Format the sql record + into a dict with the mandatory fields for user and slice. + :returns: dict with slice record and user record if the record was found + based on the user's id, None if not.. + :rtype:dict or None.. + """ + #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first() + request = self.api.dbsession().query(RegUser) + raw_slicerec = request.options(joinedload('reg_slices_as_researcher')).filter_by(record_id=slice_filter).first() + #raw_slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first() + #Put it in correct order + user_needed_fields = ['peer_authority', 'hrn', 'last_updated', + 'classtype', 'authority', 'gid', 'record_id', + 'date_created', 'type', 'email', 'pointer'] + slice_needed_fields = ['peer_authority', 'hrn', 'last_updated', + 'classtype', 'authority', 'gid', 'record_id', + 'date_created', 'type', 'pointer'] + if raw_slicerec: + #raw_slicerec.reg_slices_as_researcher + raw_slicerec = raw_slicerec.__dict__ + slicerec = {} + slicerec = \ + dict([(k, raw_slicerec[ + 'reg_slices_as_researcher'][0].__dict__[k]) + for k in slice_needed_fields]) + slicerec['reg_researchers'] = dict([(k, raw_slicerec[k]) + for k in user_needed_fields]) + #TODO Handle multiple slices for one user SA 10/12/12 + #for now only take the first slice record associated to the rec user + ##slicerec = raw_slicerec['reg_slices_as_researcher'][0].__dict__ + #del raw_slicerec['reg_slices_as_researcher'] + #slicerec['reg_researchers'] = raw_slicerec + ##del slicerec['_sa_instance_state'] + + return slicerec + + else: + return None + + + def _get_slice_records(self, slice_filter=None, + slice_filter_type=None): + """ + Get the slice record depending on the slice filter and its type. + :param slice_filter: Can be either the slice hrn or the user's record + id. + :type slice_filter: string + :param slice_filter_type: describes the slice filter type used, can be + slice_hrn or record_id_user + :type: string + :returns: the slice record + :rtype:dict + .. seealso::_sql_get_slice_info_from_user + .. seealso:: _sql_get_slice_info + """ + + #Get list of slices based on the slice hrn + if slice_filter_type == 'slice_hrn': + + #if get_authority(slice_filter) == self.root_auth: + #login = slice_filter.split(".")[1].split("_")[0] + + slicerec = self._sql_get_slice_info(slice_filter) + + if slicerec is None: + return None + #return login, None + + #Get slice based on user id + if slice_filter_type == 'record_id_user': + + slicerec = self._sql_get_slice_info_from_user(slice_filter) + + if slicerec: + fixed_slicerec_dict = slicerec + #At this point if there is no login it means + #record_id_user filter has been used for filtering + #if login is None : + ##If theslice record is from iotlab + #if fixed_slicerec_dict['peer_authority'] is None: + #login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0] + #return login, fixed_slicerec_dict + return fixed_slicerec_dict + else: + return None + + + + def GetSlices(self, slice_filter=None, slice_filter_type=None, + login=None): + """Get the slice records from the sfa db and add lease information + if any. + + :param slice_filter: can be the slice hrn or slice record id in the db + depending on the slice_filter_type. + :param slice_filter_type: defines the type of the filtering used, Can be + either 'slice_hrn' or 'record_id'. + :type slice_filter: string + :type slice_filter_type: string + :returns: a slice dict if slice_filter and slice_filter_type + are specified and a matching entry is found in the db. The result + is put into a list.Or a list of slice dictionnaries if no filters + arespecified. + + :rtype: list + + """ + #login = None + authorized_filter_types_list = ['slice_hrn', 'record_id_user'] + return_slicerec_dictlist = [] + + #First try to get information on the slice based on the filter provided + if slice_filter_type in authorized_filter_types_list: + fixed_slicerec_dict = self._get_slice_records(slice_filter, + slice_filter_type) + # if the slice was not found in the sfa db + if fixed_slicerec_dict is None: + return return_slicerec_dictlist + + slice_hrn = fixed_slicerec_dict['hrn'] + + logger.debug(" CORTEXLAB_API \tGetSlices login %s \ + slice record %s slice_filter %s \ + slice_filter_type %s " % (login, + fixed_slicerec_dict, slice_filter, + slice_filter_type)) + + + #Now we have the slice record fixed_slicerec_dict, get the + #jobs associated to this slice + leases_list = [] + + leases_list = self.GetLeases(login=login) + #If no job is running or no job scheduled + #return only the slice record + if leases_list == [] and fixed_slicerec_dict: + return_slicerec_dictlist.append(fixed_slicerec_dict) + + # if the jobs running don't belong to the user/slice we are looking + # for + leases_hrn = [lease['slice_hrn'] for lease in leases_list] + if slice_hrn not in leases_hrn: + return_slicerec_dictlist.append(fixed_slicerec_dict) + #If several experiments for one slice , put the slice record into + # each lease information dict + for lease in leases_list: + slicerec_dict = {} + logger.debug("CORTEXLAB_API.PY \tGetSlices slice_filter %s \ + \t lease['slice_hrn'] %s" + % (slice_filter, lease['slice_hrn'])) + if lease['slice_hrn'] == slice_hrn: + slicerec_dict['experiment_id'] = lease['lease_id'] + #Update lease dict with the slice record + if fixed_slicerec_dict: + fixed_slicerec_dict['experiment_id'] = [] + fixed_slicerec_dict['experiment_id'].append( + slicerec_dict['experiment_id']) + slicerec_dict.update(fixed_slicerec_dict) + + slicerec_dict['slice_hrn'] = lease['slice_hrn'] + slicerec_dict['hrn'] = lease['slice_hrn'] + slicerec_dict['user'] = lease['user'] + slicerec_dict.update( + {'list_node_ids': + {'hostname': lease['reserved_nodes']}}) + slicerec_dict.update({'node_ids': lease['reserved_nodes']}) + + + return_slicerec_dictlist.append(slicerec_dict) + + + logger.debug("CORTEXLAB_API.PY \tGetSlices \ + slicerec_dict %s return_slicerec_dictlist %s \ + lease['reserved_nodes'] \ + %s" % (slicerec_dict, return_slicerec_dictlist, + lease['reserved_nodes'])) + + logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \ + return_slicerec_dictlist %s" + % (return_slicerec_dictlist)) + + return return_slicerec_dictlist + + + else: + #Get all slices from the cortexlab sfa database , get the user info + # as well at the same time put them in dict format + request = self.api.dbsession().query(RegSlice) + query_slice_list = \ + request.options(joinedload('reg_researchers')).all() + + for record in query_slice_list: + tmp = record.__dict__ + tmp['reg_researchers'] = tmp['reg_researchers'][0].__dict__ + return_slicerec_dictlist.append(tmp) + + + #Get all the experiments reserved nodes + leases_list = self.testbed_shell.GetReservedNodes() + + for fixed_slicerec_dict in return_slicerec_dictlist: + slicerec_dict = {} + #Check if the slice belongs to a cortexlab user + if fixed_slicerec_dict['peer_authority'] is None: + owner = fixed_slicerec_dict['hrn'].split( + ".")[1].split("_")[0] + else: + owner = None + for lease in leases_list: + if owner == lease['user']: + slicerec_dict['experiment_id'] = lease['lease_id'] + + #for reserved_node in lease['reserved_nodes']: + logger.debug("CORTEXLAB_API.PY \tGetSlices lease %s " + % (lease)) + slicerec_dict.update(fixed_slicerec_dict) + slicerec_dict.update({'node_ids': + lease['reserved_nodes']}) + slicerec_dict.update({'list_node_ids': + {'hostname': + lease['reserved_nodes']}}) + + + fixed_slicerec_dict.update(slicerec_dict) + + logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \ + return_slicerec_dictlist %s \t slice_filter %s " \ + %(return_slicerec_dictlist, slice_filter)) + + return return_slicerec_dictlist + + def AddLeases(self, hostname_list, slice_record, + lease_start_time, lease_duration): + + """Creates an experiment on the testbed corresponding to the information + provided as parameters. Adds the experiment id and the slice hrn in the + lease table on the additional sfa database so that we are able to know + which slice has which nodes. + + :param hostname_list: list of nodes' OAR hostnames. + :param slice_record: sfa slice record, must contain login and hrn. + :param lease_start_time: starting time , unix timestamp format + :param lease_duration: duration in minutes + + :type hostname_list: list + :type slice_record: dict + :type lease_start_time: integer + :type lease_duration: integer + :returns: experiment_id, can be None if the job request failed. + + """ + logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases hostname_list %s \ + slice_record %s lease_start_time %s lease_duration %s "\ + %( hostname_list, slice_record , lease_start_time, \ + lease_duration)) + + username = slice_record['login'] + + experiment_id = self.testbed_shell.LaunchExperimentOnTestbed( + hostname_list, + slice_record['hrn'], + lease_start_time, lease_duration, + username) + if experiment_id is not None: + start_time = \ + datetime.fromtimestamp(int(lease_start_time)).\ + strftime(self.testbed_shell.time_format) + end_time = lease_start_time + lease_duration + + + logger.debug("CORTEXLAB_API \t AddLeases TURN ON LOGGING SQL \ + %s %s %s "%(slice_record['hrn'], experiment_id, end_time)) + + + logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases %s %s %s " \ + %(type(slice_record['hrn']), type(experiment_id), + type(end_time))) + + testbed_xp_row = LeaseTableXP(slice_hrn=slice_record['hrn'], + experiment_id=experiment_id, + end_time=end_time) + + logger.debug("CORTEXLAB_API \t AddLeases testbed_xp_row %s" \ + %(testbed_xp_row)) + self.api.dbsession().add(testbed_xp_row) + self.api.dbsession().commit() + + logger.debug("CORTEXLAB_API \t AddLeases hostname_list \ + start_time %s " %(start_time)) + + return experiment_id + + + def GetLeases(self, lease_filter_dict=None, login=None): + """ + Get the list of leases from testbed with complete information + about which slice owns which jobs and nodes. + Two purposes: + -Fetch all the experiment from the testbed (running, waiting..) + complete the reservation information with slice hrn + found in lease_table . If not available in the table, + assume it is a iotlab slice. + -Updates the iotlab table, deleting jobs when necessary. + + :returns: reservation_list, list of dictionaries with 'lease_id', + 'reserved_nodes','slice_id', 'state', 'user', 'component_id_list', + 'slice_hrn', 'resource_ids', 't_from', 't_until' + :rtype: list + + """ + + unfiltered_reservation_list = self.testbed_shell.GetReservedNodes(login) + + reservation_list = [] + #Find the slice associated with this user iotlab ldap uid + logger.debug(" CORTEXLAB \tGetLeases login %s\ + unfiltered_reservation_list %s " + % (login, unfiltered_reservation_list)) + #Create user dict first to avoid looking several times for + #the same user in LDAP SA 27/07/12 + experiment_id_list = [] + jobs_psql_query = self.api.dbsession().query(LeaseTableXP).all() + jobs_psql_dict = dict([(row.experiment_id, row.__dict__) + for row in jobs_psql_query]) + #jobs_psql_dict = jobs_psql_dict) + logger.debug("CORTEXLAB \tGetLeases jobs_psql_dict %s" + % (jobs_psql_dict)) + jobs_psql_id_list = [row.experiment_id for row in jobs_psql_query] + + for resa in unfiltered_reservation_list: + logger.debug("CORTEXLAB \tGetLeases USER %s" + % (resa['user'])) + #Construct list of jobs (runing, waiting..) from scheduler + experiment_id_list.append(resa['lease_id']) + #If there is information on the job in IOTLAB DB ] + #(slice used and job id) + if resa['lease_id'] in jobs_psql_dict: + job_info = jobs_psql_dict[resa['lease_id']] + logger.debug("CORTEXLAB \tGetLeases job_info %s" + % (job_info)) + resa['slice_hrn'] = job_info['slice_hrn'] + resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice') + + #otherwise, assume it is a iotlab slice: + else: + resa['slice_id'] = hrn_to_urn(self.testbed_shell.root_auth \ + + '.' + resa['user'] + "_slice", + 'slice') + resa['slice_hrn'] = Xrn(resa['slice_id']).get_hrn() + + resa['component_id_list'] = [] + #Transform the hostnames into urns (component ids) + for node in resa['reserved_nodes']: + + iotlab_xrn = xrn_object(self.testbed_shell.root_auth, node) + resa['component_id_list'].append(iotlab_xrn.urn) + + if lease_filter_dict: + logger.debug("CORTEXLAB \tGetLeases \ + \r\n leasefilter %s" % ( lease_filter_dict)) + + # filter_dict_functions = { + # 'slice_hrn' : IotlabShell.filter_lease_name, + # 't_from' : IotlabShell.filter_lease_start_time + # } + reservation_list = list(unfiltered_reservation_list) + for filter_type in lease_filter_dict: + logger.debug("CORTEXLAB \tGetLeases reservation_list %s" \ + % (reservation_list)) + reservation_list = self.testbed_shell.filter_lease( + reservation_list,filter_type, + lease_filter_dict[filter_type] ) + + # Filter the reservation list with a maximum timespan so that the + # leases and jobs running after this timestamp do not appear + # in the result leases. + # if 'start_time' in : + # if resa['start_time'] < lease_filter_dict['start_time']: + # reservation_list.append(resa) + + + # if 'name' in lease_filter_dict and \ + # lease_filter_dict['name'] == resa['slice_hrn']: + # reservation_list.append(resa) + + + if lease_filter_dict is None: + reservation_list = unfiltered_reservation_list + + self.update_experiments_in_lease_table(experiment_id_list, + jobs_psql_id_list) + + logger.debug(" CORTEXLAB.PY \tGetLeases reservation_list %s" + % (reservation_list)) + return reservation_list + + def update_experiments_in_lease_table(self, + experiment_list_from_testbed, experiment_list_in_db): + """Cleans the lease_table by deleting expired and cancelled jobs. + + Compares the list of experiment ids given by the testbed with the + experiment ids that are already in the database, deletes the + experiments that are no longer in the testbed experiment id list. + + :param experiment_list_from_testbed: list of experiment ids coming + from testbed + :type experiment_list_from_testbed: list + :param experiment_list_in_db: list of experiment ids from the sfa + additionnal database. + :type experiment_list_in_db: list + + :returns: None + """ + #Turn the list into a set + set_experiment_list_in_db = set(experiment_list_in_db) + + kept_experiments = set(experiment_list_from_testbed).intersection(set_experiment_list_in_db) + logger.debug("\r\n \t update_experiments_in_lease_table \ + experiment_list_in_db %s \r\n \ + experiment_list_from_testbed %s \ + kept_experiments %s " + % (set_experiment_list_in_db, + experiment_list_from_testbed, kept_experiments)) + deleted_experiments = set_experiment_list_in_db.difference( + kept_experiments) + deleted_experiments = list(deleted_experiments) + if len(deleted_experiments) > 0: + request = self.api.dbsession().query(LeaseTableXP) + request.filter(LeaseTableXP.experiment_id.in_(deleted_experiments)).delete(synchronize_session='fetch') + self.api.dbsession().commit() + return + + + def AddSlice(self, slice_record, user_record): + """ + + Add slice to the local cortexlab sfa tables if the slice comes + from a federated site and is not yet in the cortexlab sfa DB, + although the user has already a LDAP login. + Called by verify_slice during lease/sliver creation. + + :param slice_record: record of slice, must contain hrn, gid, slice_id + and authority of the slice. + :type slice_record: dictionary + :param user_record: record of the user + :type user_record: RegUser + + """ + + sfa_record = RegSlice(hrn=slice_record['hrn'], + gid=slice_record['gid'], + pointer=slice_record['slice_id'], + authority=slice_record['authority']) + logger.debug("CORTEXLAB_API.PY AddSlice sfa_record %s user_record %s" + % (sfa_record, user_record)) + sfa_record.just_created() + self.api.dbsession().add(sfa_record) + self.api.dbsession().commit() + #Update the reg-researcher dependance table + sfa_record.reg_researchers = [user_record] + self.api.dbsession().commit() + + return + def augment_records_with_testbed_info(self, record_list): """ @@ -118,8 +788,8 @@ class CortexlabDriver(Driver): # For client_helper.py compatibility 'key_ids': ''}) - #Get iotlab slice record and oar job id if any. - recslice_list = self.testbed_shell.GetSlices( + #Get slice record and job id if any. + recslice_list = self.GetSlices( slice_filter=str(record['hrn']), slice_filter_type='slice_hrn') @@ -147,7 +817,7 @@ class CortexlabDriver(Driver): #The record is a SFA user record. #Get the information about his slice from Iotlab's DB #and add it to the user record. - recslice_list = self.testbed_shell.GetSlices( + recslice_list = self.GetSlices( slice_filter=record['record_id'], slice_filter_type='record_id_user') @@ -166,7 +836,7 @@ class CortexlabDriver(Driver): recslice.update( {'PI': [recuser['hrn']], 'researcher': [recuser['hrn']], - 'name': record['hrn'], + 'name': recuser['hrn'], 'node_ids': [], 'experiment_id': [], 'person_ids': [recuser['record_id']]}) @@ -216,7 +886,7 @@ class CortexlabDriver(Driver): """ #First get the slice with the slice hrn - slice_list = self.testbed_shell.GetSlices(slice_filter=slice_hrn, + slice_list = self.GetSlices(slice_filter=slice_hrn, slice_filter_type='slice_hrn') if len(slice_list) == 0: @@ -294,7 +964,6 @@ class CortexlabDriver(Driver): % (resources, res)) return result - def get_user_record(self, hrn): """ @@ -306,7 +975,7 @@ class CortexlabDriver(Driver): :rtype: RegUser """ - return dbsession.query(RegRecord).filter_by(hrn=hrn).first() + return self.api.dbsession().query(RegRecord).filter_by(hrn=hrn).first() def testbed_name(self): """ @@ -318,28 +987,6 @@ class CortexlabDriver(Driver): """ return self.hrn - # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory - def aggregate_version(self): - """ - - Returns the testbed's supported rspec advertisement and request - versions. - :returns: rspec versions supported ad a dictionary. - :rtype: dict - - """ - version_manager = VersionManager() - ad_rspec_versions = [] - request_rspec_versions = [] - for rspec_version in version_manager.versions: - if rspec_version.content_type in ['*', 'ad']: - ad_rspec_versions.append(rspec_version.to_dict()) - if rspec_version.content_type in ['*', 'request']: - request_rspec_versions.append(rspec_version.to_dict()) - return { - 'testbed': self.testbed_name(), - 'geni_request_rspec_versions': request_rspec_versions, - 'geni_ad_rspec_versions': ad_rspec_versions} def _get_requested_leases_list(self, rspec): """ @@ -413,6 +1060,7 @@ class CortexlabDriver(Driver): return requested_xp_dict + def _process_requested_xp_dict(self, rspec): """ Turns the requested leases and information into a dictionary @@ -432,106 +1080,16 @@ class CortexlabDriver(Driver): return xp_dict - def create_sliver(self, slice_urn, slice_hrn, creds, rspec_string, - users, options): - """Answer to CreateSliver. - - Creates the leases and slivers for the users from the information - found in the rspec string. - Launch experiment on OAR if the requested leases is valid. Delete - no longer requested leases. - - - :param creds: user's credentials - :type creds: string - :param users: user record list - :type users: list - :param options: - :type options: - - :returns: a valid Rspec for the slice which has just been - modified. - :rtype: RSpec - """ - aggregate = CortexlabAggregate(self) - - slices = CortexlabSlices(self) - peer = slices.get_peer(slice_hrn) - sfa_peer = slices.get_sfa_peer(slice_hrn) - slice_record = None - - if not isinstance(creds, list): - creds = [creds] - - if users: - slice_record = users[0].get('slice_record', {}) - logger.debug("CORTEXLABDRIVER.PY \t ===============create_sliver \t\ - creds %s \r\n \r\n users %s" - % (creds, users)) - slice_record['user'] = {'keys': users[0]['keys'], - 'email': users[0]['email'], - 'hrn': slice_record['reg-researchers'][0]} - # parse rspec - rspec = RSpec(rspec_string) - logger.debug("CORTEXLABDRIVER.PY \t create_sliver \trspec.version \ - %s slice_record %s users %s" - % (rspec.version, slice_record, users)) - - # ensure site record exists? - # ensure slice record exists - #Removed options in verify_slice SA 14/08/12 - #Removed peer record in verify_slice SA 18/07/13 - sfa_slice = slices.verify_slice(slice_hrn, slice_record, sfa_peer) - - # ensure person records exists - #verify_persons returns added persons but the return value - #is not used - #Removed peer record and sfa_peer in verify_persons SA 18/07/13 - slices.verify_persons(slice_hrn, sfa_slice, users, options=options) - #requested_attributes returned by rspec.version.get_slice_attributes() - #unused, removed SA 13/08/12 - #rspec.version.get_slice_attributes() - - logger.debug("CORTEXLABDRIVER.PY create_sliver slice %s " % (sfa_slice)) - - # add/remove slice from nodes - - #requested_slivers = [node.get('component_id') \ - #for node in rspec.version.get_nodes_with_slivers()\ - #if node.get('authority_id') is self.testbed_shell.root_auth] - #l = [ node for node in rspec.version.get_nodes_with_slivers() ] - #logger.debug("SLADRIVER \tcreate_sliver requested_slivers \ - #requested_slivers %s listnodes %s" \ - #%(requested_slivers,l)) - #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12. - #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer) - - requested_xp_dict = self._process_requested_xp_dict(rspec) - - logger.debug("CORTEXLABDRIVER.PY \tcreate_sliver requested_xp_dict %s " - % (requested_xp_dict)) - #verify_slice_leases returns the leases , but the return value is unused - #here. Removed SA 13/08/12 - slices.verify_slice_leases(sfa_slice, - requested_xp_dict, peer) - - return aggregate.get_rspec(slice_xrn=slice_urn, - login=sfa_slice['login'], - version=rspec.version) - - def delete_sliver(self, slice_urn, slice_hrn, creds, options): + def delete(self, slice_urns, options={}): """ Deletes the lease associated with the slice hrn and the credentials if the slice belongs to iotlab. Answer to DeleteSliver. :param slice_urn: urn of the slice - :param slice_hrn: name of the slice - :param creds: slice credenials :type slice_urn: string - :type slice_hrn: string - :type creds: ? unused + :returns: 1 if the slice to delete was not found on iotlab, True if the deletion was successful, False otherwise otherwise. @@ -542,17 +1100,34 @@ class CortexlabDriver(Driver): .. note:: creds are unused, and are not used either in the dummy driver delete_sliver . """ - - sfa_slice_list = self.testbed_shell.GetSlices( - slice_filter=slice_hrn, - slice_filter_type='slice_hrn') + # collect sliver ids so we can update sliver allocation states after + # we remove the slivers. + aggregate = CortexlabAggregate(self) + slivers = aggregate.get_slivers(slice_urns) + if slivers: + # slice_id = slivers[0]['slice_id'] + node_ids = [] + sliver_ids = [] + sliver_jobs_dict = {} + for sliver in slivers: + node_ids.append(sliver['node_id']) + sliver_ids.append(sliver['sliver_id']) + job_id = sliver['sliver_id'].split('+')[-1].split('-')[0] + sliver_jobs_dict[job_id] = sliver['sliver_id'] + logger.debug("CORTEXLABDRIVER.PY delete_sliver slivers %s slice_urns %s" + % (slivers, slice_urns)) + slice_hrn = urn_to_hrn(slice_urns[0])[0] + + sfa_slice_list = self.GetSlices(slice_filter=slice_hrn, + slice_filter_type='slice_hrn') if not sfa_slice_list: return 1 #Delete all leases in the slice for sfa_slice in sfa_slice_list: - logger.debug("CORTEXLABDRIVER.PY delete_sliver slice %s" % (sfa_slice)) + logger.debug("CORTEXLABDRIVER.PY delete_sliver slice %s" \ + % (sfa_slice)) slices = CortexlabSlices(self) # determine if this is a peer slice @@ -560,80 +1135,29 @@ class CortexlabDriver(Driver): logger.debug("CORTEXLABDRIVER.PY delete_sliver peer %s \ \r\n \t sfa_slice %s " % (peer, sfa_slice)) + testbed_bool_ans = self.testbed_shell.DeleteSliceFromNodes(sfa_slice) + for job_id in testbed_bool_ans: + # if the job has not been successfully deleted + # don't delete the associated sliver + # remove it from the sliver list + if testbed_bool_ans[job_id] is False: + sliver = sliver_jobs_dict[job_id] + sliver_ids.remove(sliver) try: - self.testbed_shell.DeleteSliceFromNodes(sfa_slice) - return True - except: - return False - - def list_resources (self, slice_urn, slice_hrn, creds, options): - """ - - List resources from the iotlab aggregate and returns a Rspec - advertisement with resources found when slice_urn and slice_hrn are - None (in case of resource discovery). - If a slice hrn and urn are provided, list experiment's slice - nodes in a rspec format. Answer to ListResources. - Caching unused. - - :param slice_urn: urn of the slice - :param slice_hrn: name of the slice - :param creds: slice credenials - :type slice_urn: string - :type slice_hrn: string - :type creds: ? unused - :param options: options used when listing resources (list_leases, info, - geni_available) - :returns: rspec string in xml - :rtype: string - - .. note:: creds are unused - """ - - #cached_requested = options.get('cached', True) - - version_manager = VersionManager() - # get the rspec's return format from options - rspec_version = \ - version_manager.get_version(options.get('geni_rspec_version')) - version_string = "rspec_%s" % (rspec_version) - - #panos adding the info option to the caching key (can be improved) - if options.get('info'): - version_string = version_string + "_" + \ - options.get('info', 'default') - - # Adding the list_leases option to the caching key - if options.get('list_leases'): - version_string = version_string + "_" + \ - options.get('list_leases', 'default') - - # Adding geni_available to caching key - if options.get('geni_available'): - version_string = version_string + "_" + \ - str(options.get('geni_available')) - - # look in cache first - #if cached_requested and self.cache and not slice_hrn: - #rspec = self.cache.get(version_string) - #if rspec: - #logger.debug("IotlabDriver.ListResources: \ - #returning cached advertisement") - #return rspec - - #panos: passing user-defined options - aggregate = CortexlabAggregate(self) - rspec = aggregate.get_rspec(slice_xrn=slice_urn, - version=rspec_version, options=options) - - # cache the result - #if self.cache and not slice_hrn: - #logger.debug("Iotlab.ListResources: stores advertisement in cache") - #self.cache.add(version_string, rspec) - - return rspec + dbsession = self.api.dbsession() + SliverAllocation.delete_allocations(sliver_ids, dbsession) + except : + logger.log_exc("CORTEXLABDRIVER.PY delete error ") + # prepare return struct + geni_slivers = [] + for sliver in slivers: + geni_slivers.append( + {'geni_sliver_urn': sliver['sliver_id'], + 'geni_allocation_status': 'geni_unallocated', + 'geni_expires': datetime_to_string(utcparse(sliver['expires']))}) + return geni_slivers def list_slices(self, creds, options): """Answer to ListSlices. @@ -645,7 +1169,7 @@ class CortexlabDriver(Driver): :returns: slice urns list :rtype: list - .. note:: creds are unused + .. note:: creds are unused- SA 12/12/13 """ # look in cache first #if self.cache: @@ -656,7 +1180,7 @@ class CortexlabDriver(Driver): # get data from db - slices = self.testbed_shell.GetSlices() + slices = self.GetSlices() logger.debug("CORTEXLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n" % (slices)) slice_hrns = [iotlab_slice['hrn'] for iotlab_slice in slices] @@ -694,13 +1218,12 @@ class CortexlabDriver(Driver): """ return -1 - def update(self, old_sfa_record, new_sfa_record, hrn, new_key): """ - No site or node record update allowed in Cortexlab. The only modifications + No site or node record update allowed in Iotlab. The only modifications authorized here are key deletion/addition on an existing user and password change. On an existing user, CAN NOT BE MODIFIED: 'first_name', - 'last_name', 'email'. DOES NOT EXIST IN LDAP: 'phone', 'url', 'bio', + 'last_name', 'email'. DOES NOT EXIST IN SENSLAB: 'phone', 'url', 'bio', 'title', 'accepted_aup'. A slice is bound to its user, so modifying the user's ssh key should nmodify the slice's GID after an import procedure. @@ -714,48 +1237,52 @@ class CortexlabDriver(Driver): :type hrn: string TODO: needs review - .. seealso:: update in driver.py. - - """ - pointer = old_sfa_record['pointer'] - old_sfa_record_type = old_sfa_record['type'] - - # new_key implemented for users only - if new_key and old_sfa_record_type not in ['user']: - raise UnknownSfaType(old_sfa_record_type) - - if old_sfa_record_type == "user": - update_fields = {} - all_fields = new_sfa_record - for key in all_fields.keys(): - if key in ['key', 'password']: - update_fields[key] = all_fields[key] - - if new_key: - # must check this key against the previous one if it exists - persons = self.testbed_shell.GetPersons([old_sfa_record]) - person = persons[0] - keys = [person['pkey']] - #Get all the person's keys - keys_dict = self.testbed_shell.GetKeys(keys) - - # Delete all stale keys, meaning the user has only one key - #at a time - #TODO: do we really want to delete all the other keys? - #Is this a problem with the GID generation to have multiple - #keys? SA 30/05/13 - key_exists = False - if key in keys_dict: - key_exists = True - else: - #remove all the other keys - for key in keys_dict: - self.testbed_shell.DeleteKey(person, key) - self.testbed_shell.AddPersonKey( - person, {'sshPublicKey': person['pkey']}, - {'sshPublicKey': new_key}) + .. warning:: SA 12/12/13 - Removed. should be done in iotlabimporter + since users, keys and slice are managed by the LDAP. + + """ + # pointer = old_sfa_record['pointer'] + # old_sfa_record_type = old_sfa_record['type'] + + # # new_key implemented for users only + # if new_key and old_sfa_record_type not in ['user']: + # raise UnknownSfaType(old_sfa_record_type) + + # if old_sfa_record_type == "user": + # update_fields = {} + # all_fields = new_sfa_record + # for key in all_fields.keys(): + # if key in ['key', 'password']: + # update_fields[key] = all_fields[key] + + # if new_key: + # # must check this key against the previous one if it exists + # persons = self.testbed_shell.GetPersons([old_sfa_record]) + # person = persons[0] + # keys = [person['pkey']] + # #Get all the person's keys + # keys_dict = self.GetKeys(keys) + + # # Delete all stale keys, meaning the user has only one key + # #at a time + # #TODO: do we really want to delete all the other keys? + # #Is this a problem with the GID generation to have multiple + # #keys? SA 30/05/13 + # key_exists = False + # if key in keys_dict: + # key_exists = True + # else: + # #remove all the other keys + # for key in keys_dict: + # self.testbed_shell.DeleteKey(person, key) + # self.testbed_shell.AddPersonKey( + # person, {'sshPublicKey': person['pkey']}, + # {'sshPublicKey': new_key}) + logger.warning ("UNDEFINED - Update should be done by the \ + iotlabimporter") return True + def remove(self, sfa_record): """ @@ -789,7 +1316,200 @@ class CortexlabDriver(Driver): return self.testbed_shell.DeletePerson(sfa_record) elif sfa_record_type == 'slice': - if self.testbed_shell.GetSlices(slice_filter=hrn, - slice_filter_type='slice_hrn'): + if self.GetSlices(slice_filter=hrn, + slice_filter_type='slice_hrn'): ret = self.testbed_shell.DeleteSlice(sfa_record) return True + + def check_sliver_credentials(self, creds, urns): + """Check that the sliver urns belongs to the slice specified in the + credentials. + + :param urns: list of sliver urns. + :type urns: list. + :param creds: slice credentials. + :type creds: Credential object. + + + """ + # build list of cred object hrns + slice_cred_names = [] + for cred in creds: + slice_cred_hrn = Credential(cred=cred).get_gid_object().get_hrn() + slicename = IotlabXrn(xrn=slice_cred_hrn).iotlab_slicename() + slice_cred_names.append(slicename) + + # look up slice name of slivers listed in urns arg + + slice_ids = [] + for urn in urns: + sliver_id_parts = Xrn(xrn=urn).get_sliver_id_parts() + try: + slice_ids.append(int(sliver_id_parts[0])) + except ValueError: + pass + + if not slice_ids: + raise Forbidden("sliver urn not provided") + + slices = self.GetSlices(slice_ids) + sliver_names = [single_slice['name'] for single_slice in slices] + + # make sure we have a credential for every specified sliver + for sliver_name in sliver_names: + if sliver_name not in slice_cred_names: + msg = "Valid credential not found for target: %s" % sliver_name + raise Forbidden(msg) + + ######################################## + ########## aggregate oriented + ######################################## + + # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory + def aggregate_version(self): + """ + + Returns the testbed's supported rspec advertisement and request + versions. + :returns: rspec versions supported ad a dictionary. + :rtype: dict + + """ + version_manager = VersionManager() + ad_rspec_versions = [] + request_rspec_versions = [] + for rspec_version in version_manager.versions: + if rspec_version.content_type in ['*', 'ad']: + ad_rspec_versions.append(rspec_version.to_dict()) + if rspec_version.content_type in ['*', 'request']: + request_rspec_versions.append(rspec_version.to_dict()) + return { + 'testbed': self.testbed_name(), + 'geni_request_rspec_versions': request_rspec_versions, + 'geni_ad_rspec_versions': ad_rspec_versions} + + + + # first 2 args are None in case of resource discovery + def list_resources (self, version=None, options={}): + aggregate = CortexlabAggregate(self) + rspec = aggregate.list_resources(version=version, options=options) + return rspec + + + def describe(self, urns, version, options={}): + aggregate = CortexlabAggregate(self) + return aggregate.describe(urns, version=version, options=options) + + def status (self, urns, options={}): + aggregate = CortexlabAggregate(self) + desc = aggregate.describe(urns, version='GENI 3') + status = {'geni_urn': desc['geni_urn'], + 'geni_slivers': desc['geni_slivers']} + return status + + + def allocate (self, urn, rspec_string, expiration, options={}): + xrn = Xrn(urn) + aggregate = CortexlabAggregate(self) + + slices = CortexlabSlices(self) + peer = slices.get_peer(xrn.get_hrn()) + sfa_peer = slices.get_sfa_peer(xrn.get_hrn()) + + slice_record = None + users = options.get('geni_users', []) + + sfa_users = options.get('sfa_users', []) + if sfa_users: + slice_record = sfa_users[0].get('slice_record', []) + + # parse rspec + rspec = RSpec(rspec_string) + # requested_attributes = rspec.version.get_slice_attributes() + + # ensure site record exists + + # ensure slice record exists + + current_slice = slices.verify_slice(xrn.hrn, slice_record, sfa_peer) + logger.debug("IOTLABDRIVER.PY \t ===============allocate \t\ + \r\n \r\n current_slice %s" % (current_slice)) + # ensure person records exists + + # oui c'est degueulasse, le slice_record se retrouve modifie + # dans la methode avec les infos du user, els infos sont propagees + # dans verify_slice_leases + persons = slices.verify_persons(xrn.hrn, slice_record, users, + options=options) + # ensure slice attributes exists + # slices.verify_slice_attributes(slice, requested_attributes, + # options=options) + + # add/remove slice from nodes + requested_xp_dict = self._process_requested_xp_dict(rspec) + + logger.debug("IOTLABDRIVER.PY \tallocate requested_xp_dict %s " + % (requested_xp_dict)) + request_nodes = rspec.version.get_nodes_with_slivers() + nodes_list = [] + for start_time in requested_xp_dict: + lease = requested_xp_dict[start_time] + for hostname in lease['hostname']: + nodes_list.append(hostname) + + # nodes = slices.verify_slice_nodes(slice_record,request_nodes, peer) + logger.debug("IOTLABDRIVER.PY \tallocate nodes_list %s slice_record %s" + % (nodes_list, slice_record)) + + # add/remove leases + rspec_requested_leases = rspec.version.get_leases() + leases = slices.verify_slice_leases(slice_record, + requested_xp_dict, peer) + logger.debug("IOTLABDRIVER.PY \tallocate leases %s \ + rspec_requested_leases %s" % (leases, + rspec_requested_leases)) + # update sliver allocations + for hostname in nodes_list: + client_id = hostname + node_urn = xrn_object(self.testbed_shell.root_auth, hostname).urn + component_id = node_urn + slice_urn = current_slice['reg-urn'] + for lease in leases: + if hostname in lease['reserved_nodes']: + index = lease['reserved_nodes'].index(hostname) + sliver_hrn = '%s.%s-%s' % (self.hrn, lease['lease_id'], + lease['resource_ids'][index] ) + sliver_id = Xrn(sliver_hrn, type='sliver').urn + record = SliverAllocation(sliver_id=sliver_id, client_id=client_id, + component_id=component_id, + slice_urn = slice_urn, + allocation_state='geni_allocated') + record.sync(self.api.dbsession()) + + return aggregate.describe([xrn.get_urn()], version=rspec.version) + + def provision(self, urns, options={}): + # update users + slices = CortexlabSlices(self) + aggregate = CortexlabAggregate(self) + slivers = aggregate.get_slivers(urns) + current_slice = slivers[0] + peer = slices.get_peer(current_slice['hrn']) + sfa_peer = slices.get_sfa_peer(current_slice['hrn']) + users = options.get('geni_users', []) + # persons = slices.verify_persons(current_slice['hrn'], + # current_slice, users, peer, sfa_peer, options=options) + # slices.handle_peer(None, None, persons, peer) + # update sliver allocation states and set them to geni_provisioned + sliver_ids = [sliver['sliver_id'] for sliver in slivers] + dbsession = self.api.dbsession() + SliverAllocation.set_allocations(sliver_ids, 'geni_provisioned', + dbsession) + version_manager = VersionManager() + rspec_version = version_manager.get_version(options[ + 'geni_rspec_version']) + return self.describe(urns, rspec_version, options=options) + + + diff --git a/sfa/cortexlab/cortexlabshell.py b/sfa/cortexlab/cortexlabshell.py index 54c637a5..8c19ed6a 100644 --- a/sfa/cortexlab/cortexlabshell.py +++ b/sfa/cortexlab/cortexlabshell.py @@ -8,20 +8,13 @@ from datetime import datetime from sfa.util.sfalogging import logger -from sfa.storage.alchemy import dbsession -from sqlalchemy.orm import joinedload -from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey -from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB, LeaseTableXP +from sfa.iotlab.iotlabpostgres import LeaseTableXP from sfa.cortexlab.LDAPapi import LDAPapi -from sfa.util.xrn import Xrn, hrn_to_urn, get_authority -from sfa.trust.certificate import Keypair, convert_public_key -from sfa.trust.gid import create_uuid -from sfa.trust.hierarchy import Hierarchy -from sfa.iotlab.iotlabaggregate import iotlab_xrn_object +from sfa.iotlab.iotlabxrn import xrn_object from sfa.cortexlab.cortexlabnodes import CortexlabQueryNodes class CortexlabShell(): @@ -58,51 +51,6 @@ class CortexlabShell(): """ return CortexlabShell._MINIMUM_DURATION - - def GetPeers(self, peer_filter=None ): - """ Gathers registered authorities in SFA DB and looks for specific peer - if peer_filter is specified. - :param peer_filter: name of the site authority looked for. - :type peer_filter: string - :returns: list of records. - - """ - - existing_records = {} - existing_hrns_by_types = {} - logger.debug("CORTEXLAB_API \tGetPeers peer_filter %s " % (peer_filter)) - all_records = dbsession.query(RegRecord).filter(RegRecord.type.like('%authority%')).all() - - for record in all_records: - existing_records[(record.hrn, record.type)] = record - if record.type not in existing_hrns_by_types: - existing_hrns_by_types[record.type] = [record.hrn] - else: - existing_hrns_by_types[record.type].append(record.hrn) - - logger.debug("CORTEXLAB_API \tGetPeer\texisting_hrns_by_types %s " - % (existing_hrns_by_types)) - records_list = [] - - try: - if peer_filter: - records_list.append(existing_records[(peer_filter, - 'authority')]) - else: - for hrn in existing_hrns_by_types['authority']: - records_list.append(existing_records[(hrn, 'authority')]) - - logger.debug("CORTEXLAB_API \tGetPeer \trecords_list %s " - % (records_list)) - - except KeyError: - pass - - return_records = records_list - logger.debug("CORTEXLAB_API \tGetPeer return_records %s " - % (return_records)) - return return_records - #TODO : Handling OR request in make_ldap_filters_from_records #instead of the for loop #over the records' list @@ -276,36 +224,7 @@ class CortexlabShell(): return_node_list = node_list_dict return return_node_list - def AddSlice(self, slice_record, user_record): - """ - - Add slice to the local cortexlab sfa tables if the slice comes - from a federated site and is not yet in the cortexlab sfa DB, - although the user has already a LDAP login. - Called by verify_slice during lease/sliver creation. - - :param slice_record: record of slice, must contain hrn, gid, slice_id - and authority of the slice. - :type slice_record: dictionary - :param user_record: record of the user - :type user_record: RegUser - """ - - sfa_record = RegSlice(hrn=slice_record['hrn'], - gid=slice_record['gid'], - pointer=slice_record['slice_id'], - authority=slice_record['authority']) - logger.debug("CORTEXLAB_API.PY AddSlice sfa_record %s user_record %s" - % (sfa_record, user_record)) - sfa_record.just_created() - dbsession.add(sfa_record) - dbsession.commit() - #Update the reg-researcher dependance table - sfa_record.reg_researchers = [user_record] - dbsession.commit() - - return def GetSites(self, site_filter_name_list=None, return_fields_list=None): @@ -391,85 +310,6 @@ class CortexlabShell(): return delete_failed or True - def __add_person_to_db(self, user_dict): - """ - Add a federated user straight to db when the user issues a lease - request with iotlab nodes and that he has not registered with cortexlab - yet (that is he does not have a LDAP entry yet). - Uses parts of the routines in CortexlabImport when importing user - from LDAP. - Called by AddPerson, right after LdapAddUser. - :param user_dict: Must contain email, hrn and pkey to get a GID - and be added to the SFA db. - :type user_dict: dict - - """ - check_if_exists = \ - dbsession.query(RegUser).filter_by(email = user_dict['email']).first() - #user doesn't exists - if not check_if_exists: - logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \ - " %(user_dict)) - hrn = user_dict['hrn'] - person_urn = hrn_to_urn(hrn, 'user') - pubkey = user_dict['pkey'] - try: - pkey = convert_public_key(pubkey) - except TypeError: - #key not good. create another pkey - logger.warn('__add_person_to_db: unable to convert public \ - key for %s' %(hrn )) - pkey = Keypair(create=True) - - - if pubkey is not None and pkey is not None : - hierarchy = Hierarchy() - person_gid = hierarchy.create_gid(person_urn, create_uuid(), \ - pkey) - if user_dict['email']: - logger.debug("__add_person_to_db \r\n \r\n \ - IOTLAB IMPORTER PERSON EMAIL OK email %s "\ - %(user_dict['email'])) - person_gid.set_email(user_dict['email']) - - user_record = RegUser(hrn=hrn , pointer= '-1', \ - authority=get_authority(hrn), \ - email=user_dict['email'], gid = person_gid) - user_record.reg_keys = [RegKey(user_dict['pkey'])] - user_record.just_created() - dbsession.add (user_record) - dbsession.commit() - return - - - def AddPerson(self, record): - """ - - Adds a new account. Any fields specified in records are used, - otherwise defaults are used. Creates an appropriate login by calling - LdapAddUser. - - :param record: dictionary with the sfa user's properties. - :returns: a dicitonary with the status. If successful, the dictionary - boolean is set to True and there is a 'uid' key with the new login - added to LDAP, otherwise the bool is set to False and a key - 'message' is in the dictionary, with the error message. - :rtype: dict - - """ - ret = self.ldap.LdapAddUser(record) - - if ret['bool'] is True: - record['hrn'] = self.root_auth + '.' + ret['uid'] - logger.debug("CORTEXLAB_API AddPerson return code %s record %s " - % (ret, record)) - self.__add_person_to_db(record) - return ret - - - - - #TODO AddPersonKey 04/07/2012 SA def AddPersonKey(self, person_uid, old_attributes_dict, new_key_dict): """Adds a new key to the specified account. Adds the key to the @@ -492,6 +332,33 @@ class CortexlabShell(): logger.warning("CORTEXLAB_API AddPersonKey EMPTY - DO NOTHING \r\n ") return ret['bool'] + def DeleteLeases(self, leases_id_list, slice_hrn): + """ + + Deletes several leases, based on their experiment ids and the slice + they are associated with. Uses DeleteOneLease to delete the + experiment on the testbed. Note that one slice can contain multiple + experiments, and in this + case all the experiments in the leases_id_list MUST belong to this + same slice, since there is only one slice hrn provided here. + + :param leases_id_list: list of job ids that belong to the slice whose + slice hrn is provided. + :param slice_hrn: the slice hrn. + :type slice_hrn: string + + .. warning:: Does not have a return value since there was no easy + way to handle failure when dealing with multiple job delete. Plus, + there was no easy way to report it to the user. + + """ + logger.debug("CORTEXLAB_API DeleteLeases leases_id_list %s slice_hrn %s \ + \r\n " %(leases_id_list, slice_hrn)) + for experiment_id in leases_id_list: + self.DeleteOneLease(experiment_id, slice_hrn) + + return + @staticmethod def _process_walltime(duration): @@ -628,89 +495,7 @@ class CortexlabShell(): return experiment_id - def AddLeases(self, hostname_list, slice_record, - lease_start_time, lease_duration): - """Creates an experiment on the testbed corresponding to the information - provided as parameters. Adds the experiment id and the slice hrn in the - lease table on the additional sfa database so that we are able to know - which slice has which nodes. - - :param hostname_list: list of nodes' OAR hostnames. - :param slice_record: sfa slice record, must contain login and hrn. - :param lease_start_time: starting time , unix timestamp format - :param lease_duration: duration in minutes - - :type hostname_list: list - :type slice_record: dict - :type lease_start_time: integer - :type lease_duration: integer - - """ - logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases hostname_list %s \ - slice_record %s lease_start_time %s lease_duration %s "\ - %( hostname_list, slice_record , lease_start_time, \ - lease_duration)) - - username = slice_record['login'] - - experiment_id = self.LaunchExperimentOnTestbed(hostname_list, \ - slice_record['hrn'], \ - lease_start_time, lease_duration, \ - username) - start_time = \ - datetime.fromtimestamp(int(lease_start_time)).\ - strftime(self.time_format) - end_time = lease_start_time + lease_duration - - - logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases TURN ON LOGGING SQL \ - %s %s %s "%(slice_record['hrn'], experiment_id, end_time)) - - - logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases %s %s %s " \ - %(type(slice_record['hrn']), type(experiment_id), - type(end_time))) - - testbed_xp_row = LeaseTableXP(slice_hrn=slice_record['hrn'], - experiment_id=experiment_id, end_time=end_time) - - logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases testbed_xp_row %s" \ - %(testbed_xp_row)) - self.leases_db.testbed_session.add(testbed_xp_row) - self.leases_db.testbed_session.commit() - - logger.debug("CORTEXLAB_API \t AddLeases hostname_list start_time %s " \ - %(start_time)) - - return - - def DeleteLeases(self, leases_id_list, slice_hrn): - """ - - Deletes several leases, based on their experiment ids and the slice - they are associated with. Uses DeleteOneLease to delete the - experiment on the testbed. Note that one slice can contain multiple - experiments, and in this - case all the experiments in the leases_id_list MUST belong to this - same slice, since there is only one slice hrn provided here. - - :param leases_id_list: list of job ids that belong to the slice whose - slice hrn is provided. - :param slice_hrn: the slice hrn. - :type slice_hrn: string - - .. warning:: Does not have a return value since there was no easy - way to handle failure when dealing with multiple job delete. Plus, - there was no easy way to report it to the user. - - """ - logger.debug("CORTEXLAB_API DeleteLeases leases_id_list %s slice_hrn %s \ - \r\n " %(leases_id_list, slice_hrn)) - for experiment_id in leases_id_list: - self.DeleteOneLease(experiment_id, slice_hrn) - - return #Delete the jobs from job_iotlab table def DeleteSliceFromNodes(self, slice_record): @@ -754,56 +539,73 @@ class CortexlabShell(): Experiments which last less than 10 min (600 sec) are invalid""" return self.grain - - # @staticmethod - # def update_experiments_in_additional_sfa_db( job_oar_list, jobs_psql): - # """ Cleans the iotlab db by deleting expired and cancelled jobs. - # Compares the list of job ids given by OAR with the job ids that - # are already in the database, deletes the jobs that are no longer in - # the OAR job id list. - # :param job_oar_list: list of job ids coming from OAR - # :type job_oar_list: list - # :param job_psql: list of job ids cfrom the database. - # type job_psql: list - # """ - # #Turn the list into a set - # set_jobs_psql = set(jobs_psql) - - # kept_jobs = set(job_oar_list).intersection(set_jobs_psql) - # logger.debug ( "\r\n \t\ update_experiments_in_additional_sfa_db jobs_psql %s \r\n \t \ - # job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs)) - # deleted_jobs = set_jobs_psql.difference(kept_jobs) - # deleted_jobs = list(deleted_jobs) - # if len(deleted_jobs) > 0: - # self.leases_db.testbed_session.query(LeaseTableXP).filter(LeaseTableXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch') - # self.leases_db.testbed_session.commit() - - # return - @staticmethod - def filter_lease_name(reservation_list, filter_value): + def filter_lease(reservation_list, filter_type, filter_value ): + """Filters the lease reservation list by removing each lease whose + filter_type is not equal to the filter_value provided. Returns the list + of leases in one slice, defined by the slice_hrn if filter_type + is 'slice_hrn'. Otherwise, returns all leases scheduled starting from + the filter_value if filter_type is 't_from'. + + :param reservation_list: leases list + :type reservation_list: list of dictionary + :param filter_type: can be either 't_from' or 'slice hrn' + :type filter_type: string + :param filter_value: depending on the filter_type, can be the slice_hrn + or can be defining a timespan. + :type filter_value: if filter_type is 't_from', filter_value is int. + if filter_type is 'slice_hrn', filter_value is a string. + + + :returns: filtered_reservation_list, contains only leases running or + scheduled in the given slice (wanted_slice).Dict keys are + 'lease_id','reserved_nodes','slice_id', 'state', 'user', + 'component_id_list','slice_hrn', 'resource_ids', 't_from', 't_until' + :rtype: list of dict + + """ filtered_reservation_list = list(reservation_list) - logger.debug("CORTEXLAB_API \t filter_lease_name reservation_list %s" \ + logger.debug("IOTLAB_API \t filter_lease_name reservation_list %s" \ % (reservation_list)) - for reservation in reservation_list: - if 'slice_hrn' in reservation and \ - reservation['slice_hrn'] != filter_value: - filtered_reservation_list.remove(reservation) + try: + for reservation in reservation_list: + if \ + (filter_type is 'slice_hrn' and \ + reservation['slice_hrn'] != filter_value) or \ + (filter_type is 't_from' and \ + reservation['t_from'] > filter_value): + filtered_reservation_list.remove(reservation) + except TypeError: + logger.log_exc("Iotlabshell filter_lease : filter_type %s \ + filter_value %s not in lease" %(filter_type, + filter_value)) - logger.debug("CORTEXLAB_API \t filter_lease_name filtered_reservation_list %s" \ - % (filtered_reservation_list)) return filtered_reservation_list - @staticmethod - def filter_lease_start_time(reservation_list, filter_value): - filtered_reservation_list = list(reservation_list) + # @staticmethod + # def filter_lease_name(reservation_list, filter_value): + # filtered_reservation_list = list(reservation_list) + # logger.debug("CORTEXLAB_API \t filter_lease_name reservation_list %s" \ + # % (reservation_list)) + # for reservation in reservation_list: + # if 'slice_hrn' in reservation and \ + # reservation['slice_hrn'] != filter_value: + # filtered_reservation_list.remove(reservation) + + # logger.debug("CORTEXLAB_API \t filter_lease_name filtered_reservation_list %s" \ + # % (filtered_reservation_list)) + # return filtered_reservation_list + + # @staticmethod + # def filter_lease_start_time(reservation_list, filter_value): + # filtered_reservation_list = list(reservation_list) - for reservation in reservation_list: - if 't_from' in reservation and \ - reservation['t_from'] > filter_value: - filtered_reservation_list.remove(reservation) + # for reservation in reservation_list: + # if 't_from' in reservation and \ + # reservation['t_from'] > filter_value: + # filtered_reservation_list.remove(reservation) - return filtered_reservation_list + # return filtered_reservation_list def complete_leases_info(self, unfiltered_reservation_list, db_xp_dict): @@ -852,99 +654,11 @@ class CortexlabShell(): #Transform the hostnames into urns (component ids) for node in resa['reserved_nodes']: - iotlab_xrn = iotlab_xrn_object(self.root_auth, node) + iotlab_xrn = xrn_object(self.root_auth, node) resa['component_id_list'].append(iotlab_xrn.urn) return local_unfiltered_reservation_list, testbed_xp_list - def GetLeases(self, lease_filter_dict=None, login=None): - """ - - Get the list of leases from the testbed with complete information - about in which slice is running which experiment ans which nodes are - involved. - Two purposes: - -Fetch all the experiments from the testbed (running, waiting..) - complete the reservation information with slice hrn - found in testbed_xp table. If not available in the table, - assume it is a cortexlab slice. - -Updates the cortexlab table, deleting jobs when necessary. - - :returns: reservation_list, list of dictionaries with 'lease_id', - 'reserved_nodes','slice_id','user', 'component_id_list', - 'slice_hrn', 'resource_ids', 't_from', 't_until'. Other - keys can be returned if necessary, such as the 'state' of the lease, - if the information has been added in GetReservedNodes. - :rtype: list - - """ - - unfiltered_reservation_list = self.GetReservedNodes(login) - - reservation_list = [] - #Find the slice associated with this user ldap uid - logger.debug(" CORTEXLAB_API.PY \tGetLeases login %s\ - unfiltered_reservation_list %s " - % (login, unfiltered_reservation_list)) - #Create user dict first to avoid looking several times for - #the same user in LDAP SA 27/07/12 - - - db_xp_query = self.leases_db.testbed_session.query(LeaseTableXP).all() - db_xp_dict = dict([(row.experiment_id, row.__dict__) - for row in db_xp_query]) - - logger.debug("CORTEXLAB_API \tGetLeases db_xp_dict %s" - % (db_xp_dict)) - db_xp_id_list = [row.experiment_id for row in db_xp_query] - - required_fiels_in_leases = ['lease_id', - 'reserved_nodes','slice_id', 'user', 'component_id_list', - 'slice_hrn', 'resource_ids', 't_from', 't_until'] - - # Add any missing information on the leases with complete_leases_info - unfiltered_reservation_list, testbed_xp_list = \ - self.complete_leases_info(unfiltered_reservation_list, - db_xp_dict) - # Check that the list of leases is complete and have the mandatory - # information - format_status = self.ensure_format_is_valid(unfiltered_reservation_list, - required_fiels_in_leases) - - if not format_status: - logger.log_exc("\tCortexlabapi \t GetLeases : Missing fields in \ - reservation list") - raise KeyError, "GetLeases : Missing fields in reservation list " - - if lease_filter_dict: - logger.debug("CORTEXLAB_API \tGetLeases \ - \r\n leasefilter %s" % ( lease_filter_dict)) - - filter_dict_functions = { - 'slice_hrn' : CortexlabShell.filter_lease_name, - 't_from' : CortexlabShell.filter_lease_start_time - } - - reservation_list = list(unfiltered_reservation_list) - for filter_type in lease_filter_dict: - logger.debug("CORTEXLAB_API \tGetLeases reservation_list %s" \ - % (reservation_list)) - reservation_list = filter_dict_functions[filter_type](\ - reservation_list,lease_filter_dict[filter_type] ) - - - if lease_filter_dict is None: - reservation_list = unfiltered_reservation_list - - self.leases_db.update_experiments_in_additional_sfa_db( - testbed_xp_list, db_xp_id_list) - - logger.debug(" CORTEXLAB_API.PY \tGetLeases reservation_list %s" - % (reservation_list)) - return reservation_list - - - #TODO FUNCTIONS SECTION 04/07/2012 SA @@ -1025,36 +739,7 @@ class CortexlabShell(): #return - def GetKeys(self, key_filter=None): - """Returns a dict of dict based on the key string. Each dict entry - contains the key id, the ssh key, the user's email and the - user's hrn. - If key_filter is specified and is an array of key identifiers, - only keys matching the filter will be returned. - - Admin may query all keys. Non-admins may only query their own keys. - FROM PLC API DOC - - :returns: dict with ssh key as key and dicts as value. - :rtype: dict - """ - if key_filter is None: - keys = dbsession.query(RegKey).options(joinedload('reg_user')).all() - else: - keys = dbsession.query(RegKey).options(joinedload('reg_user')).filter(RegKey.key.in_(key_filter)).all() - - key_dict = {} - for key in keys: - key_dict[key.key] = {'key_id': key.key_id, 'key': key.key, - 'email': key.reg_user.email, - 'hrn': key.reg_user.hrn} - - #ldap_rslt = self.ldap.LdapSearch({'enabled']=True}) - #user_by_email = dict((user[1]['mail'][0], user[1]['sshPublicKey']) \ - #for user in ldap_rslt) - logger.debug("CORTEXLAB_API GetKeys -key_dict %s \r\n " % (key_dict)) - return key_dict #TODO : test def DeleteKey(self, user_record, key_string): @@ -1079,273 +764,8 @@ class CortexlabShell(): return ret['bool'] - def _sql_get_slice_info(self, slice_filter): - """ - Get the slice record based on the slice hrn. Fetch the record of the - user associated with the slice by using joinedload based on the - reg_researcher relationship. - - :param slice_filter: the slice hrn we are looking for - :type slice_filter: string - :returns: the slice record enhanced with the user's information if the - slice was found, None it wasn't. - - :rtype: dict or None. - """ - #DO NOT USE RegSlice - reg_researchers to get the hrn - #of the user otherwise will mess up the RegRecord in - #Resolve, don't know why - SA 08/08/2012 - - #Only one entry for one user = one slice in testbed_xp table - #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() - raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first() - #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first() - if raw_slicerec: - #load_reg_researcher - #raw_slicerec.reg_researchers - raw_slicerec = raw_slicerec.__dict__ - logger.debug(" CORTEXLAB_API \t _sql_get_slice_info slice_filter %s \ - raw_slicerec %s" % (slice_filter, raw_slicerec)) - slicerec = raw_slicerec - #only one researcher per slice so take the first one - #slicerec['reg_researchers'] = raw_slicerec['reg_researchers'] - #del slicerec['reg_researchers']['_sa_instance_state'] - return slicerec - - else: - return None - - - def _sql_get_slice_info_from_user(self, slice_filter): - """ - Get the slice record based on the user recordid by using a joinedload - on the relationship reg_slices_as_researcher. Format the sql record - into a dict with the mandatory fields for user and slice. - :returns: dict with slice record and user record if the record was found - based on the user's id, None if not.. - :rtype:dict or None.. - """ - #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first() - raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id=slice_filter).first() - #raw_slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first() - #Put it in correct order - user_needed_fields = ['peer_authority', 'hrn', 'last_updated', - 'classtype', 'authority', 'gid', 'record_id', - 'date_created', 'type', 'email', 'pointer'] - slice_needed_fields = ['peer_authority', 'hrn', 'last_updated', - 'classtype', 'authority', 'gid', 'record_id', - 'date_created', 'type', 'pointer'] - if raw_slicerec: - #raw_slicerec.reg_slices_as_researcher - raw_slicerec = raw_slicerec.__dict__ - slicerec = {} - slicerec = \ - dict([(k, raw_slicerec[ - 'reg_slices_as_researcher'][0].__dict__[k]) - for k in slice_needed_fields]) - slicerec['reg_researchers'] = dict([(k, raw_slicerec[k]) - for k in user_needed_fields]) - #TODO Handle multiple slices for one user SA 10/12/12 - #for now only take the first slice record associated to the rec user - ##slicerec = raw_slicerec['reg_slices_as_researcher'][0].__dict__ - #del raw_slicerec['reg_slices_as_researcher'] - #slicerec['reg_researchers'] = raw_slicerec - ##del slicerec['_sa_instance_state'] - - return slicerec - - else: - return None - - def _get_slice_records(self, slice_filter=None, - slice_filter_type=None): - """ - Get the slice record depending on the slice filter and its type. - :param slice_filter: Can be either the slice hrn or the user's record - id. - :type slice_filter: string - :param slice_filter_type: describes the slice filter type used, can be - slice_hrn or record_id_user - :type: string - :returns: the slice record - :rtype:dict - .. seealso::_sql_get_slice_info_from_user - .. seealso:: _sql_get_slice_info - """ - - #Get list of slices based on the slice hrn - if slice_filter_type == 'slice_hrn': - - #if get_authority(slice_filter) == self.root_auth: - #login = slice_filter.split(".")[1].split("_")[0] - - slicerec = self._sql_get_slice_info(slice_filter) - if slicerec is None: - return None - #return login, None - #Get slice based on user id - if slice_filter_type == 'record_id_user': - - slicerec = self._sql_get_slice_info_from_user(slice_filter) - - if slicerec: - fixed_slicerec_dict = slicerec - #At this point if there is no login it means - #record_id_user filter has been used for filtering - #if login is None : - ##If theslice record is from iotlab - #if fixed_slicerec_dict['peer_authority'] is None: - #login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0] - #return login, fixed_slicerec_dict - return fixed_slicerec_dict - else: - return None - - - def GetSlices(self, slice_filter=None, slice_filter_type=None, - login=None): - """Get the slice records from the sfa db and add lease information - if any. - - :param slice_filter: can be the slice hrn or slice record id in the db - depending on the slice_filter_type. - :param slice_filter_type: defines the type of the filtering used, Can be - either 'slice_hrn' or 'record_id'. - :type slice_filter: string - :type slice_filter_type: string - :returns: a slice dict if slice_filter and slice_filter_type - are specified and a matching entry is found in the db. The result - is put into a list.Or a list of slice dictionnaries if no filters - arespecified. - - :rtype: list - - """ - #login = None - authorized_filter_types_list = ['slice_hrn', 'record_id_user'] - return_slicerec_dictlist = [] - - #First try to get information on the slice based on the filter provided - if slice_filter_type in authorized_filter_types_list: - fixed_slicerec_dict = self._get_slice_records(slice_filter, - slice_filter_type) - # if the slice was not found in the sfa db - if fixed_slicerec_dict is None: - return return_slicerec_dictlist - - slice_hrn = fixed_slicerec_dict['hrn'] - - logger.debug(" CORTEXLAB_API \tGetSlices login %s \ - slice record %s slice_filter %s \ - slice_filter_type %s " % (login, - fixed_slicerec_dict, slice_filter, - slice_filter_type)) - - - #Now we have the slice record fixed_slicerec_dict, get the - #jobs associated to this slice - leases_list = [] - - leases_list = self.GetLeases(login=login) - #If no job is running or no job scheduled - #return only the slice record - if leases_list == [] and fixed_slicerec_dict: - return_slicerec_dictlist.append(fixed_slicerec_dict) - - # if the jobs running don't belong to the user/slice we are looking - # for - leases_hrn = [lease['slice_hrn'] for lease in leases_list] - if slice_hrn not in leases_hrn: - return_slicerec_dictlist.append(fixed_slicerec_dict) - #If several experiments for one slice , put the slice record into - # each lease information dict - for lease in leases_list: - slicerec_dict = {} - logger.debug("CORTEXLAB_API.PY \tGetSlices slice_filter %s \ - \t lease['slice_hrn'] %s" - % (slice_filter, lease['slice_hrn'])) - if lease['slice_hrn'] == slice_hrn: - slicerec_dict['experiment_id'] = lease['lease_id'] - #Update lease dict with the slice record - if fixed_slicerec_dict: - fixed_slicerec_dict['experiment_id'] = [] - fixed_slicerec_dict['experiment_id'].append( - slicerec_dict['experiment_id']) - slicerec_dict.update(fixed_slicerec_dict) - - slicerec_dict['slice_hrn'] = lease['slice_hrn'] - slicerec_dict['hrn'] = lease['slice_hrn'] - slicerec_dict['user'] = lease['user'] - slicerec_dict.update( - {'list_node_ids': - {'hostname': lease['reserved_nodes']}}) - slicerec_dict.update({'node_ids': lease['reserved_nodes']}) - - - return_slicerec_dictlist.append(slicerec_dict) - - - logger.debug("CORTEXLAB_API.PY \tGetSlices \ - slicerec_dict %s return_slicerec_dictlist %s \ - lease['reserved_nodes'] \ - %s" % (slicerec_dict, return_slicerec_dictlist, - lease['reserved_nodes'])) - - logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \ - return_slicerec_dictlist %s" - % (return_slicerec_dictlist)) - - return return_slicerec_dictlist - - - else: - #Get all slices from the cortexlab sfa database , get the user info - # as well at the same time put them in dict format - - query_slice_list = \ - dbsession.query(RegSlice).options(joinedload('reg_researchers')).all() - - for record in query_slice_list: - tmp = record.__dict__ - tmp['reg_researchers'] = tmp['reg_researchers'][0].__dict__ - return_slicerec_dictlist.append(tmp) - - - #Get all the experiments reserved nodes - leases_list = self.GetReservedNodes() - - for fixed_slicerec_dict in return_slicerec_dictlist: - slicerec_dict = {} - #Check if the slice belongs to a cortexlab user - if fixed_slicerec_dict['peer_authority'] is None: - owner = fixed_slicerec_dict['hrn'].split( - ".")[1].split("_")[0] - else: - owner = None - for lease in leases_list: - if owner == lease['user']: - slicerec_dict['experiment_id'] = lease['lease_id'] - - #for reserved_node in lease['reserved_nodes']: - logger.debug("CORTEXLAB_API.PY \tGetSlices lease %s " - % (lease)) - slicerec_dict.update(fixed_slicerec_dict) - slicerec_dict.update({'node_ids': - lease['reserved_nodes']}) - slicerec_dict.update({'list_node_ids': - {'hostname': - lease['reserved_nodes']}}) - - - fixed_slicerec_dict.update(slicerec_dict) - - logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \ - return_slicerec_dictlist %s \slice_filter %s " \ - %(return_slicerec_dictlist, slice_filter)) - - return return_slicerec_dictlist -- 2.43.0