May try to factor the code in the future, as Iotlab and Cortexlab drivers are similar.
Contains methods to get information on slice, slivers, nodes and leases,
formatting them and turn it into a RSpec.
"""
-from sfa.util.xrn import hrn_to_urn, urn_to_hrn, get_authority
+from sfa.util.xrn import hrn_to_urn, urn_to_hrn
+from sfa.util.sfatime import utcparse, datetime_to_string
+from sfa.iotlab.iotlabxrn import IotlabXrn, xrn_object
from sfa.rspecs.rspec import RSpec
+
#from sfa.rspecs.elements.location import Location
from sfa.rspecs.elements.hardware_type import HardwareType
+from sfa.rspecs.elements.node import NodeElement
from sfa.rspecs.elements.login import Login
-from sfa.rspecs.elements.services import Services
from sfa.rspecs.elements.sliver import Sliver
from sfa.rspecs.elements.lease import Lease
from sfa.rspecs.elements.granularity import Granularity
from sfa.rspecs.version_manager import VersionManager
-
+from sfa.storage.model import SliverAllocation
from sfa.rspecs.elements.versions.iotlabv1Node import IotlabPosition, \
- IotlabNode, IotlabLocation, IotlabMobility
+ IotlabLocation
from sfa.util.sfalogging import logger
from sfa.util.xrn import Xrn
-
-def cortexlab_xrn_to_hostname(xrn):
- """Returns a node's hostname from its xrn.
- :param xrn: The nodes xrn identifier.
- :type xrn: Xrn (from sfa.util.xrn)
-
- :returns: node's hostname.
- :rtype: string
-
- """
- return Xrn.unescape(Xrn(xrn=xrn, type='node').get_leaf())
-
-
-def cortexlab_xrn_object(root_auth, hostname):
- """Creates a valid xrn object from the node's hostname and the authority
- of the SFA server.
-
- :param hostname: the node's hostname.
- :param root_auth: the SFA root authority.
- :type hostname: string
- :type root_auth: string
-
- :returns: the cortexlab node's xrn
- :rtype: Xrn
-
- """
- return Xrn('.'.join([root_auth, Xrn.escape(hostname)]), type='node')
-
+import time
class CortexlabAggregate:
- """Aggregate manager class for Iotlab. """
+ """Aggregate manager class for cortexlab. """
sites = {}
nodes = {}
return (sfa_slice, slivers)
slice_urn = hrn_to_urn(slice_xrn, 'slice')
slice_hrn, _ = urn_to_hrn(slice_xrn)
- slice_name = slice_hrn
# GetSlices always returns a list, even if there is only one element
- slices = self.driver.testbed_shell.GetSlices(slice_filter=str(slice_name),
+ slices = self.driver.GetSlices(slice_filter=str(slice_hrn),
slice_filter_type='slice_hrn',
login=login)
- logger.debug("IotlabAggregate api \tget_slice_and_slivers \
+ logger.debug("CortexlabAggregate api \tget_slice_and_slivers \
slice_hrn %s \r\n slices %s self.driver.hrn %s"
% (slice_hrn, slices, self.driver.hrn))
if slices == []:
def find_ldap_username_from_slice(self, sfa_slice):
+ """
+ Gets the ldap username of the user based on the information contained
+ in ist sfa_slice record.
+
+ :param sfa_slice: the user's slice record. Must contain the
+ reg_researchers key.
+ :type sfa_slice: dictionary
+ :returns: ldap_username, the ldap user's login.
+ :rtype: string
+ """
+
researchers = [sfa_slice['reg_researchers'][0].__dict__]
# look in ldap:
ldap_username = None
- def get_nodes(self, slices=None, slivers=[], options=None):
+ def get_nodes(self, options=None):
"""Returns the nodes in the slice using the rspec format, with all the
nodes' properties.
it. If the slice does not have any job running or scheduled, that is
it has no reserved nodes, then returns an empty list.
- :param slices: list of slices (record dictionaries)
- :param slivers: the list of slivers in all the slices
- :type slices: list of dicts
- :type slivers: list of Sliver object (dictionaries)
:returns: An empty list if the slice has no reserved nodes, a rspec
list with all the nodes and their properties (a dict per node)
otherwise.
.. seealso:: get_slice_and_slivers
"""
- # NT: the semantic of this function is not clear to me :
- # if slice is not defined, then all the nodes should be returned
- # if slice is defined, we should return only the nodes that
- # are part of this slice
- # but what is the role of the slivers parameter ?
- # So i assume that slice['node_ids'] will be the same as slivers for us
- slice_nodes_list = []
- if slices is not None:
- for one_slice in slices:
- try:
- slice_nodes_list = one_slice['node_ids']
- # if we are dealing with a slice that has no node just
- # return an empty list. In cortexlab a slice can have multiple
- # jobs scheduled, so it either has at least one lease or
- # not at all.
- except KeyError:
- return []
+ filter_nodes = None
+ if options:
+ geni_available = options.get('geni_available')
+ if geni_available == True:
+ filter_nodes['boot_state'] = ['Alive']
+
+ # slice_nodes_list = []
+ # if slices is not None:
+ # for one_slice in slices:
+ # try:
+ # slice_nodes_list = one_slice['node_ids']
+ # # if we are dealing with a slice that has no node just
+ # # return an empty list. In iotlab a slice can have multiple
+ # # jobs scheduled, so it either has at least one lease or
+ # # not at all.
+ # except KeyError:
+ # return []
# get the granularity in second for the reservation system
- grain = self.driver.testbed_shell.GetLeaseGranularity()
+ # grain = self.driver.testbed_shell.GetLeaseGranularity()
- nodes = self.driver.testbed_shell.GetNodes()
+ nodes = self.driver.testbed_shell.GetNodes(node_filter_dict =
+ filter_nodes)
nodes_dict = {}
#if slices, this means we got to list all the nodes given to this slice
# Make a list of all the nodes in the slice before getting their
#attributes
- rspec_nodes = []
- logger.debug("CortexlabAggregate api get_nodes slices %s "
- % (slices))
-
- reserved_nodes = self.driver.testbed_shell.GetNodesCurrentlyInUse()
- logger.debug("CortexlabAggregate api get_nodes slice_nodes_list %s "
- % (slice_nodes_list))
for node in nodes:
nodes_dict[node['node_id']] = node
- if slice_nodes_list == [] or node['hostname'] in slice_nodes_list:
- rspec_node = Node()
+ return nodes_dict
+
+
+ def node_to_rspec_node(self, node):
+ """ Creates a rspec node structure with the appropriate information
+ based on the node information that can be found in the node dictionary.
+ :param node: node data. this dict contains information about the node
+ and must have the following keys : mobile, radio, archi, hostname,
+ boot_state, site, x, y ,z (position).
+ :type node: dictionary.
+ :returns: node dictionary containing the following keys : mobile, archi,
+ radio, component_id, component_name, component_manager_id,
+ authority_id, boot_state, exclusive, hardware_types, location,
+ position, granularity, tags.
+ :rtype: dict
+
+ """
- cortexlab_xrn = cortexlab_xrn_object(self.driver.testbed_shell.root_auth,
+ grain = self.driver.testbed_shell.GetLeaseGranularity()
+ rspec_node = NodeElement()
+
+ # xxx how to retrieve site['login_base']
+ #site_id=node['site_id']
+ #site=sites_dict[site_id]
+
+ rspec_node['mobile'] = node['mobile']
+ rspec_node['archi'] = node['archi']
+ rspec_node['radio'] = node['radio']
+ cortexlab_xrn = xrn_object(self.driver.testbed_shell.root_auth,
node['hostname'])
- rspec_node['component_id'] = cortexlab_xrn.urn
- rspec_node['component_name'] = node['hostname']
- rspec_node['component_manager_id'] = \
- hrn_to_urn(self.driver.testbed_shell.root_auth,
- 'authority+sa')
-
- # Iotlab's nodes are federated : there is only one authority
- # for all Iotlab sites, registered in SFA.
- # Removing the part including the site
- # in authority_id SA 27/07/12
- rspec_node['authority_id'] = rspec_node['component_manager_id']
-
-
- # boot state removed if you need it uncomment
- # rspec_node['boot_state'] = node['boot_state']
- # if node['hostname'] in reserved_nodes:
- # rspec_node['boot_state'] = "Reserved"
-
- rspec_node['exclusive'] = 'true'
- rspec_node['hardware_types'] = [HardwareType({'name':
- 'cortexlab-node'})]
-
-
- # Location, mobility and position removed. If you need it go check
- # get_nodes in iotlabaggregate.py
-
- # Granularity
- granularity = Granularity({'grain': grain})
- rspec_node['granularity'] = granularity
- rspec_node['tags'] = []
- if node['hostname'] in slivers:
- # add sliver info
- sliver = slivers[node['hostname']]
- rspec_node['sliver_id'] = sliver['sliver_id']
- rspec_node['client_id'] = node['hostname']
- rspec_node['slivers'] = [sliver]
-
- # slivers always provide the ssh service
- login = Login({'authentication': 'ssh-keys',
- 'hostname': node['hostname'], 'port': '22',
- 'username': sliver['name']})
- service = Services({'login': login})
- rspec_node['services'] = [service]
- rspec_nodes.append(rspec_node)
- return (rspec_nodes)
+ rspec_node['component_id'] = cortexlab_xrn.urn
+ rspec_node['component_name'] = node['hostname']
+ rspec_node['component_manager_id'] = \
+ hrn_to_urn(self.driver.testbed_shell.root_auth,
+ 'authority+sa')
+
+ # Iotlab's nodes are federated : there is only one authority
+ # for all Iotlab sites, registered in SFA.
+ # Removing the part including the site
+ # in authority_id SA 27/07/12
+ rspec_node['authority_id'] = rspec_node['component_manager_id']
+
+ # do not include boot state (<available> element)
+ #in the manifest rspec
+
+
+ rspec_node['boot_state'] = node['boot_state']
+ # if node['hostname'] in reserved_nodes:
+ # rspec_node['boot_state'] = "Reserved"
+ rspec_node['exclusive'] = 'true'
+ rspec_node['hardware_types'] = [HardwareType({'name': \
+ 'iotlab-node'})]
+
+ location = IotlabLocation({'country':'France', 'site': \
+ node['site']})
+ rspec_node['location'] = location
+
+
+ position = IotlabPosition()
+ for field in position :
+ try:
+ position[field] = node[field]
+ except KeyError, error :
+ logger.log_exc("Cortexlabaggregate\t node_to_rspec_node \
+ position %s "% (error))
+
+ rspec_node['position'] = position
+
+
+ # Granularity
+ granularity = Granularity({'grain': grain})
+ rspec_node['granularity'] = granularity
+ rspec_node['tags'] = []
+ # if node['hostname'] in slivers:
+ # # add sliver info
+ # sliver = slivers[node['hostname']]
+ # rspec_node['sliver_id'] = sliver['sliver_id']
+ # rspec_node['client_id'] = node['hostname']
+ # rspec_node['slivers'] = [sliver]
+
+ # # slivers always provide the ssh service
+ # login = Login({'authentication': 'ssh-keys', \
+ # 'hostname': node['hostname'], 'port':'22', \
+ # 'username': sliver['name']})
+ # service = Services({'login': login})
+ # rspec_node['services'] = [service]
+
+ return rspec_node
+
+
+ def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations = {}):
+ """Makes a geni sliver structure from all the nodes allocated
+ to slivers in the sliver_allocations dictionary. Returns the states
+ of the sliver.
+
+ :param rspec_node: Node information contained in a rspec data structure
+ fashion.
+ :type rspec_node: dictionary
+ :param sliver_allocations:
+ :type sliver_allocations: dictionary
+
+ :returns: Dictionary with the following keys: geni_sliver_urn,
+ geni_expires, geni_allocation_status, geni_operational_status,
+ geni_error.
+
+ :rtype: dictionary
+
+ .. seealso:: node_to_rspec_node
+
+ """
+ if rspec_node['sliver_id'] in sliver_allocations:
+ # set sliver allocation and operational status
+ sliver_allocation = sliver_allocations[rspec_node['sliver_id']]
+ if sliver_allocation:
+ allocation_status = sliver_allocation.allocation_state
+ if allocation_status == 'geni_allocated':
+ op_status = 'geni_pending_allocation'
+ elif allocation_status == 'geni_provisioned':
+ op_status = 'geni_ready'
+ else:
+ op_status = 'geni_unknown'
+ else:
+ allocation_status = 'geni_unallocated'
+ else:
+ allocation_status = 'geni_unallocated'
+ op_status = 'geni_failed'
+ # required fields
+ geni_sliver = {'geni_sliver_urn': rspec_node['sliver_id'],
+ 'geni_expires': rspec_node['expires'],
+ 'geni_allocation_status' : allocation_status,
+ 'geni_operational_status': op_status,
+ 'geni_error': '',
+ }
+ return geni_sliver
+
+ def sliver_to_rspec_node(self, sliver, sliver_allocations):
+ """Used by describe to format node information into a rspec compliant
+ structure.
+
+ Creates a node rspec compliant structure by calling node_to_rspec_node.
+ Adds slivers, if any, to rspec node structure. Returns the updated
+ rspec node struct.
+
+ :param sliver: sliver dictionary. Contains keys: urn, slice_id, hostname
+ and slice_name.
+ :type sliver: dictionary
+ :param sliver_allocations: dictionary of slivers
+ :type sliver_allocations: dict
+
+ :returns: Node dictionary with all necessary data.
+
+ .. seealso:: node_to_rspec_node
+ """
+ rspec_node = self.node_to_rspec_node(sliver)
+ rspec_node['expires'] = datetime_to_string(utcparse(sliver['expires']))
+ # add sliver info
+ logger.debug("CORTEXLABAGGREGATE api \t sliver_to_rspec_node sliver \
+ %s \r\nsliver_allocations %s" % (sliver,
+ sliver_allocations))
+ rspec_sliver = Sliver({'sliver_id': sliver['urn'],
+ 'name': sliver['slice_id'],
+ 'type': 'iotlab-exclusive',
+ 'tags': []})
+ rspec_node['sliver_id'] = rspec_sliver['sliver_id']
+
+ if sliver['urn'] in sliver_allocations:
+ rspec_node['client_id'] = sliver_allocations[
+ sliver['urn']].client_id
+ if sliver_allocations[sliver['urn']].component_id:
+ rspec_node['component_id'] = sliver_allocations[
+ sliver['urn']].component_id
+ rspec_node['slivers'] = [rspec_sliver]
+
+ # slivers always provide the ssh service
+ login = Login({'authentication': 'ssh-keys',
+ 'hostname': sliver['hostname'],
+ 'port':'22',
+ 'username': sliver['slice_name'],
+ 'login': sliver['slice_name']
+ })
+ return rspec_node
+
def get_all_leases(self, ldap_username):
"""
All the leases running or scheduled are returned.
:param ldap_username: if ldap uid is not None, looks for the leases
- belonging to this user.
+ belonging to this user.
:type ldap_username: string
:returns: rspec lease dictionary with keys lease_id, component_id,
- slice_id, start_time, duration.
+ slice_id, start_time, duration where the lease_id is the oar job id,
+ component_id is the node's urn, slice_id is the slice urn,
+ start_time is the timestamp starting time and duration is expressed
+ in terms of the testbed's granularity.
:rtype: dict
.. note::There is no filtering of leases within a given time frame.
"""
- #now = int(time.time())
- #lease_filter = {'clip': now }
-
-
-
logger.debug("CortexlabAggregate get_all_leases ldap_username %s "
% (ldap_username))
- leases = self.driver.testbed_shell.GetLeases(login=ldap_username)
+ leases = self.driver.driver.GetLeases(login=ldap_username)
grain = self.driver.testbed_shell.GetLeaseGranularity()
# site_ids = []
rspec_leases = []
rspec_lease = Lease()
rspec_lease['lease_id'] = lease['lease_id']
- cortexlab_xrn = cortexlab_xrn_object(
+ cortexlab_xrn = xrn_object(
self.driver.testbed_shell.root_auth, node)
rspec_lease['component_id'] = cortexlab_xrn.urn
#rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn,\
slice_xrn %s slices %s\r\n \r\n"
% (slice_xrn, slices))
- if options is not None and 'list_leases' in options:
+ if options is not None :
lease_option = options['list_leases']
else:
#If no options are specified, at least print the resources
if lease_option in ['all', 'resources']:
#if not options.get('list_leases') or options.get('list_leases')
#and options['list_leases'] != 'leases':
- nodes = self.get_nodes(slices, slivers)
- if slice_xrn and slices is None:
- nodes = []
+ nodes = self.get_nodes()
logger.debug("\r\n")
logger.debug("CortexlabAggregate \t lease_option %s \
get rspec ******* nodes %s"
logger.debug("CortexlabAggregate \tget_rspec **** \
version type %s ldap_ user %s \r\n" \
% (version.type, ldap_username))
- if version.type == "Iotlab":
+ #TODO : Change the version of Rspec here in case of pbm -SA 09/01/14
+ if version.type in ["Cortexlab", "Iotlab"]:
rspec.version.add_connection_information(
ldap_username, sites_set)
logger.debug("CortexlabAggregate \tget_rspec **** \
FINAL RSPEC %s \r\n" % (rspec.toxml()))
return rspec.toxml()
+
+
+
+ def get_slivers(self, urns, options={}):
+ """Get slivers of the given slice urns. Slivers contains slice, node and
+ user information.
+
+ For Iotlab, returns the leases with sliver ids and their allocation
+ status.
+
+ :param urns: list of slice urns.
+ :type urns: list of strings
+ :param options: unused
+ :type options: unused
+
+ .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3/CommonConcepts#urns
+ """
+
+
+ slice_ids = set()
+ node_ids = []
+ for urn in urns:
+ xrn = IotlabXrn(xrn=urn)
+ if xrn.type == 'sliver':
+ # id: slice_id-node_id
+ try:
+ sliver_id_parts = xrn.get_sliver_id_parts()
+ slice_id = int(sliver_id_parts[0])
+ node_id = int(sliver_id_parts[1])
+ slice_ids.add(slice_id)
+ node_ids.append(node_id)
+ except ValueError:
+ pass
+ else:
+ slice_names = set()
+ slice_names.add(xrn.hrn)
+
+
+ logger.debug("CortexlabAggregate \t get_slivers urns %s slice_ids %s \
+ node_ids %s\r\n" % (urns, slice_ids, node_ids))
+ logger.debug("CortexlabAggregate \t get_slivers xrn %s slice_names %s \
+ \r\n" % (xrn, slice_names))
+ filter_sliver = {}
+ if slice_names:
+ filter_sliver['slice_hrn'] = list(slice_names)
+ slice_hrn = filter_sliver['slice_hrn'][0]
+
+ slice_filter_type = 'slice_hrn'
+
+ # if slice_ids:
+ # filter['slice_id'] = list(slice_ids)
+ # # get slices
+ if slice_hrn:
+ slices = self.driver.GetSlices(slice_hrn,
+ slice_filter_type)
+ leases = self.driver.GetLeases({'slice_hrn':slice_hrn})
+ logger.debug("CortexlabAggregate \t get_slivers \
+ slices %s leases %s\r\n" % (slices, leases ))
+ if not slices:
+ return []
+
+ single_slice = slices[0]
+ # get sliver users
+ user = single_slice['reg_researchers'][0].__dict__
+ logger.debug("CortexlabAggregate \t get_slivers user %s \
+ \r\n" % (user))
+
+ # construct user key info
+ person = self.driver.testbed_shell.ldap.LdapFindUser(record=user)
+ logger.debug("CortexlabAggregate \t get_slivers person %s \
+ \r\n" % (person))
+ # name = person['last_name']
+ user['login'] = person['uid']
+ user['user_urn'] = hrn_to_urn(user['hrn'], 'user')
+ user['keys'] = person['pkey']
+
+
+ try:
+ node_ids = single_slice['node_ids']
+ node_list = self.driver.testbed_shell.GetNodes(
+ {'hostname':single_slice['node_ids']})
+ node_by_hostname = dict([(node['hostname'], node)
+ for node in node_list])
+ except KeyError:
+ logger.warning("\t get_slivers No slivers in slice")
+ # slice['node_ids'] = node_ids
+ # nodes_dict = self.get_slice_nodes(slice, options)
+
+ slivers = []
+ for current_lease in leases:
+ for hostname in current_lease['reserved_nodes']:
+ node = {}
+ node['slice_id'] = current_lease['slice_id']
+ node['slice_hrn'] = current_lease['slice_hrn']
+ slice_name = current_lease['slice_hrn'].split(".")[1]
+ node['slice_name'] = slice_name
+ index = current_lease['reserved_nodes'].index(hostname)
+ node_id = current_lease['resource_ids'][index]
+ # node['slice_name'] = user['login']
+ # node.update(single_slice)
+ more_info = node_by_hostname[hostname]
+ node.update(more_info)
+ # oar_job_id is the slice_id (lease_id)
+ sliver_hrn = '%s.%s-%s' % (self.driver.hrn,
+ current_lease['lease_id'], node_id)
+ node['node_id'] = node_id
+ node['expires'] = current_lease['t_until']
+ node['sliver_id'] = Xrn(sliver_hrn, type='sliver').urn
+ node['urn'] = node['sliver_id']
+ node['services_user'] = [user]
+
+ slivers.append(node)
+ return slivers
+
+
+ def list_resources(self, version = None, options={}):
+ """
+ Returns an advertisement Rspec of available resources at this
+ aggregate. This Rspec contains a resource listing along with their
+ description, providing sufficient information for clients to be able to
+ select among available resources.
+
+ :param options: various options. The valid options are: {boolean
+ geni_compressed <optional>; struct geni_rspec_version { string type;
+ #case insensitive , string version; # case insensitive}} . The only
+ mandatory options if options is specified is geni_rspec_version.
+ :type options: dictionary
+
+ :returns: On success, the value field of the return struct will contain
+ a geni.rspec advertisment RSpec
+ :rtype: Rspec advertisement in xml.
+
+ .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3/CommonConcepts#RSpecdatatype
+ .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3#ListResources
+ """
+
+ version_manager = VersionManager()
+ version = version_manager.get_version(version)
+ rspec_version = version_manager._get_version(version.type,
+ version.version, 'ad')
+ rspec = RSpec(version=rspec_version, user_options=options)
+ # variable ldap_username to be compliant with get_all_leases
+ # prototype. Now unused in geni-v3 since we are getting all the leases
+ # here
+ ldap_username = None
+ if not options.get('list_leases') or options['list_leases'] != 'leases':
+ # get nodes
+ nodes_dict = self.get_nodes(options)
+
+ # no interfaces on iotlab nodes
+ # convert nodes to rspec nodes
+ rspec_nodes = []
+ for node_id in nodes_dict:
+ node = nodes_dict[node_id]
+ rspec_node = self.node_to_rspec_node(node)
+ rspec_nodes.append(rspec_node)
+ rspec.version.add_nodes(rspec_nodes)
+
+ # add links
+ # links = self.get_links(sites, nodes_dict, interfaces)
+ # rspec.version.add_links(links)
+
+ if not options.get('list_leases') or options.get('list_leases') \
+ and options['list_leases'] != 'resources':
+ leases = self.get_all_leases(ldap_username)
+ rspec.version.add_leases(leases)
+
+ return rspec.toxml()
+
+
+ def describe(self, urns, version=None, options={}):
+ """
+ Retrieve a manifest RSpec describing the resources contained by the
+ named entities, e.g. a single slice or a set of the slivers in a slice.
+ This listing and description should be sufficiently descriptive to allow
+ experimenters to use the resources.
+
+ :param urns: If a slice urn is supplied and there are no slivers in the
+ given slice at this aggregate, then geni_rspec shall be a valid
+ manifest RSpec, containing no node elements - no resources.
+ :type urns: list or strings
+ :param options: various options. the valid options are: {boolean
+ geni_compressed <optional>; struct geni_rspec_version { string type;
+ #case insensitive , string version; # case insensitive}}
+ :type options: dictionary
+
+ :returns: On success returns the following dictionary {geni_rspec:
+ <geni.rspec, a Manifest RSpec>, geni_urn: <string slice urn of the
+ containing slice>, geni_slivers:{ geni_sliver_urn:
+ <string sliver urn>, geni_expires: <dateTime.rfc3339 allocation
+ expiration string, as in geni_expires from SliversStatus>,
+ geni_allocation_status: <string sliver state - e.g. geni_allocated
+ or geni_provisioned >, geni_operational_status:
+ <string sliver operational state>, geni_error: <optional string.
+ The field may be omitted entirely but may not be null/None,
+ explaining any failure for a sliver.>}
+
+ .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3#Describe
+ .. seealso:: http://groups.geni.net/geni/wiki/GAPI_AM_API_V3/CommonConcepts#urns
+ """
+ version_manager = VersionManager()
+ version = version_manager.get_version(version)
+ rspec_version = version_manager._get_version(
+ version.type, version.version, 'manifest')
+ rspec = RSpec(version=rspec_version, user_options=options)
+
+ # get slivers
+ geni_slivers = []
+ slivers = self.get_slivers(urns, options)
+ if slivers:
+ rspec_expires = datetime_to_string(utcparse(slivers[0]['expires']))
+ else:
+ rspec_expires = datetime_to_string(utcparse(time.time()))
+ rspec.xml.set('expires', rspec_expires)
+
+ # lookup the sliver allocations
+ geni_urn = urns[0]
+ sliver_ids = [sliver['sliver_id'] for sliver in slivers]
+ logger.debug(" Cortexlabaggregate.PY \tDescribe sliver_ids %s "
+ % (sliver_ids))
+ constraint = SliverAllocation.sliver_id.in_(sliver_ids)
+ query = self.driver.api.dbsession().query(SliverAllocation)
+ sliver_allocations = query.filter((constraint)).all()
+ logger.debug(" Cortexlabaggregate.PY \tDescribe sliver_allocations %s "
+ % (sliver_allocations))
+ sliver_allocation_dict = {}
+ for sliver_allocation in sliver_allocations:
+ geni_urn = sliver_allocation.slice_urn
+ sliver_allocation_dict[sliver_allocation.sliver_id] = \
+ sliver_allocation
+
+ # add slivers
+ nodes_dict = {}
+ for sliver in slivers:
+ nodes_dict[sliver['node_id']] = sliver
+ rspec_nodes = []
+ for sliver in slivers:
+ rspec_node = self.sliver_to_rspec_node(sliver,
+ sliver_allocation_dict)
+ rspec_nodes.append(rspec_node)
+ logger.debug(" Cortexlabaggregate.PY \tDescribe sliver_allocation_dict %s "
+ % (sliver_allocation_dict))
+ geni_sliver = self.rspec_node_to_geni_sliver(rspec_node,
+ sliver_allocation_dict)
+ geni_slivers.append(geni_sliver)
+
+ logger.debug(" Cortexlabaggregate.PY \tDescribe rspec_nodes %s\
+ rspec %s "
+ % (rspec_nodes, rspec))
+ rspec.version.add_nodes(rspec_nodes)
+
+ return {'geni_urn': geni_urn,
+ 'geni_rspec': rspec.toxml(),
+ 'geni_slivers': geni_slivers}
\ No newline at end of file
"""
Implements what a driver should provide for SFA to work.
"""
-from sfa.util.faults import SliverDoesNotExist, UnknownSfaType
+from datetime import datetime
+from sfa.util.sfatime import utcparse, datetime_to_string
+
+from sfa.util.faults import SliverDoesNotExist, Forbidden
from sfa.util.sfalogging import logger
-from sfa.storage.alchemy import dbsession
-from sfa.storage.model import RegRecord
+
+from sfa.trust.hierarchy import Hierarchy
+from sfa.trust.gid import create_uuid
from sfa.managers.driver import Driver
from sfa.rspecs.version_manager import VersionManager
from sfa.rspecs.rspec import RSpec
-from sfa.util.xrn import Xrn, hrn_to_urn, get_authority
-
from sfa.cortexlab.cortexlabaggregate import CortexlabAggregate, \
cortexlab_xrn_to_hostname
from sfa.cortexlab.cortexlabslices import CortexlabSlices
-
from sfa.cortexlab.cortexlabshell import CortexlabShell
+from sfa.iotlab.iotlabxrn import IotlabXrn, xrn_object
+from sfa.util.xrn import Xrn, hrn_to_urn, get_authority, urn_to_hrn
+from sfa.trust.certificate import Keypair, convert_public_key
+from sfa.trust.credential import Credential
+from sfa.storage.model import SliverAllocation
+
+from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey
+from sfa.iotlab.iotlabpostgres import LeaseTableXP
+from sqlalchemy.orm import joinedload
class CortexlabDriver(Driver):
""" Cortexlab Driver class inherited from Driver generic class.
.. seealso::: Driver class
"""
- def __init__(self, config):
+ def __init__(self, api):
"""
- Sets the iotlab SFA config parameters,
- instanciates the testbed api and the iotlab database.
-
- :param config: iotlab SFA configuration object
- :type config: Config object
+ Sets the Cortexlab SFA config parameters,
+ instanciates the testbed api.
+ :param api: SfaApi configuration object. Holds reference to the
+ database.
+ :type api: SfaApi object
"""
- Driver.__init__(self, config)
- self.config = config
+
+ Driver.__init__(self, api)
+ self.api = api
+ config = api.config
self.testbed_shell = CortexlabShell(config)
self.cache = None
+ def GetPeers(self, peer_filter=None ):
+ """ Gathers registered authorities in SFA DB and looks for specific peer
+ if peer_filter is specified.
+ :param peer_filter: name of the site authority looked for.
+ :type peer_filter: string
+ :returns: list of records.
+
+ """
+
+ existing_records = {}
+ existing_hrns_by_types = {}
+ logger.debug("CORTEXLAB_API \tGetPeers peer_filter %s " % (peer_filter))
+ query = self.api.dbsession().query(RegRecord)
+ all_records = query.filter(RegRecord.type.like('%authority%')).all()
+
+ for record in all_records:
+ existing_records[(record.hrn, record.type)] = record
+ if record.type not in existing_hrns_by_types:
+ existing_hrns_by_types[record.type] = [record.hrn]
+ else:
+ existing_hrns_by_types[record.type].append(record.hrn)
+
+ logger.debug("CORTEXLAB_API \tGetPeer\texisting_hrns_by_types %s "
+ % (existing_hrns_by_types))
+ records_list = []
+
+ try:
+ if peer_filter:
+ records_list.append(existing_records[(peer_filter,
+ 'authority')])
+ else:
+ for hrn in existing_hrns_by_types['authority']:
+ records_list.append(existing_records[(hrn, 'authority')])
+
+ logger.debug("CORTEXLAB_API \tGetPeer \trecords_list %s "
+ % (records_list))
+
+ except KeyError:
+ pass
+
+ return_records = records_list
+ logger.debug("CORTEXLAB_API \tGetPeer return_records %s "
+ % (return_records))
+ return return_records
+
+ def GetKeys(self, key_filter=None):
+ """Returns a dict of dict based on the key string. Each dict entry
+ contains the key id, the ssh key, the user's email and the
+ user's hrn.
+ If key_filter is specified and is an array of key identifiers,
+ only keys matching the filter will be returned.
+
+ Admin may query all keys. Non-admins may only query their own keys.
+ FROM PLC API DOC
+
+ :returns: dict with ssh key as key and dicts as value.
+ :rtype: dict
+ """
+ query = self.api.dbsession().query(RegKey)
+ if key_filter is None:
+ keys = query.options(joinedload('reg_user')).all()
+ else:
+ constraint = RegKey.key.in_(key_filter)
+ keys = query.options(joinedload('reg_user')).filter(constraint).all()
+
+ key_dict = {}
+ for key in keys:
+ key_dict[key.key] = {'key_id': key.key_id, 'key': key.key,
+ 'email': key.reg_user.email,
+ 'hrn': key.reg_user.hrn}
+
+ #ldap_rslt = self.ldap.LdapSearch({'enabled']=True})
+ #user_by_email = dict((user[1]['mail'][0], user[1]['sshPublicKey']) \
+ #for user in ldap_rslt)
+
+ logger.debug("CORTEXLAB_API GetKeys -key_dict %s \r\n " % (key_dict))
+ return key_dict
+
+ def AddPerson(self, record):
+ """
+
+ Adds a new account. Any fields specified in records are used,
+ otherwise defaults are used. Creates an appropriate login by calling
+ LdapAddUser.
+
+ :param record: dictionary with the sfa user's properties.
+ :returns: a dicitonary with the status. If successful, the dictionary
+ boolean is set to True and there is a 'uid' key with the new login
+ added to LDAP, otherwise the bool is set to False and a key
+ 'message' is in the dictionary, with the error message.
+ :rtype: dict
+
+ """
+ ret = self.testbed_shell.ldap.LdapAddUser(record)
+
+ if ret['bool'] is True:
+ record['hrn'] = self.testbed_shell.root_auth + '.' + ret['uid']
+ logger.debug("Cortexlab api AddPerson return code %s record %s "
+ % (ret, record))
+ self.__add_person_to_db(record)
+ return ret
+
+ def __add_person_to_db(self, user_dict):
+ """
+ Add a federated user straight to db when the user issues a lease
+ request with nodes and that he has not registered with cortexlab
+ yet (that is he does not have a LDAP entry yet).
+ Uses parts of the routines in CortexlabImport when importing user
+ from LDAP.
+ Called by AddPerson, right after LdapAddUser.
+ :param user_dict: Must contain email, hrn and pkey to get a GID
+ and be added to the SFA db.
+ :type user_dict: dict
+
+ """
+ request = self.api.dbsession().query(RegUser)
+ check_if_exists = \
+ request.filter_by(email = user_dict['email']).first()
+ #user doesn't exists
+ if not check_if_exists:
+ logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \
+ " %(user_dict))
+ hrn = user_dict['hrn']
+ person_urn = hrn_to_urn(hrn, 'user')
+ pubkey = user_dict['pkey']
+ try:
+ pkey = convert_public_key(pubkey)
+ except TypeError:
+ #key not good. create another pkey
+ logger.warn('__add_person_to_db: unable to convert public \
+ key for %s' %(hrn ))
+ pkey = Keypair(create=True)
+
+
+ if pubkey is not None and pkey is not None :
+ hierarchy = Hierarchy()
+ person_gid = hierarchy.create_gid(person_urn, create_uuid(), \
+ pkey)
+ if user_dict['email']:
+ logger.debug("__add_person_to_db \r\n \r\n \
+ IOTLAB IMPORTER PERSON EMAIL OK email %s "\
+ %(user_dict['email']))
+ person_gid.set_email(user_dict['email'])
+
+ user_record = RegUser(hrn=hrn , pointer= '-1', \
+ authority=get_authority(hrn), \
+ email=user_dict['email'], gid = person_gid)
+ user_record.reg_keys = [RegKey(user_dict['pkey'])]
+ user_record.just_created()
+ self.api.dbsession().add (user_record)
+ self.api.dbsession().commit()
+ return
+
+
+ def _sql_get_slice_info(self, slice_filter):
+ """
+ Get the slice record based on the slice hrn. Fetch the record of the
+ user associated with the slice by using joinedload based on the
+ reg_researcher relationship.
+
+ :param slice_filter: the slice hrn we are looking for
+ :type slice_filter: string
+ :returns: the slice record enhanced with the user's information if the
+ slice was found, None it wasn't.
+
+ :rtype: dict or None.
+ """
+ #DO NOT USE RegSlice - reg_researchers to get the hrn
+ #of the user otherwise will mess up the RegRecord in
+ #Resolve, don't know why - SA 08/08/2012
+
+ #Only one entry for one user = one slice in testbed_xp table
+ #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
+ request = self.api.dbsession().query(RegSlice)
+ raw_slicerec = request.options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first()
+ #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
+ if raw_slicerec:
+ #load_reg_researcher
+ #raw_slicerec.reg_researchers
+ raw_slicerec = raw_slicerec.__dict__
+ logger.debug(" CORTEXLAB_API \t _sql_get_slice_info slice_filter %s \
+ raw_slicerec %s" % (slice_filter, raw_slicerec))
+ slicerec = raw_slicerec
+ #only one researcher per slice so take the first one
+ #slicerec['reg_researchers'] = raw_slicerec['reg_researchers']
+ #del slicerec['reg_researchers']['_sa_instance_state']
+ return slicerec
+
+ else:
+ return None
+
+ def _sql_get_slice_info_from_user(self, slice_filter):
+ """
+ Get the slice record based on the user recordid by using a joinedload
+ on the relationship reg_slices_as_researcher. Format the sql record
+ into a dict with the mandatory fields for user and slice.
+ :returns: dict with slice record and user record if the record was found
+ based on the user's id, None if not..
+ :rtype:dict or None..
+ """
+ #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
+ request = self.api.dbsession().query(RegUser)
+ raw_slicerec = request.options(joinedload('reg_slices_as_researcher')).filter_by(record_id=slice_filter).first()
+ #raw_slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
+ #Put it in correct order
+ user_needed_fields = ['peer_authority', 'hrn', 'last_updated',
+ 'classtype', 'authority', 'gid', 'record_id',
+ 'date_created', 'type', 'email', 'pointer']
+ slice_needed_fields = ['peer_authority', 'hrn', 'last_updated',
+ 'classtype', 'authority', 'gid', 'record_id',
+ 'date_created', 'type', 'pointer']
+ if raw_slicerec:
+ #raw_slicerec.reg_slices_as_researcher
+ raw_slicerec = raw_slicerec.__dict__
+ slicerec = {}
+ slicerec = \
+ dict([(k, raw_slicerec[
+ 'reg_slices_as_researcher'][0].__dict__[k])
+ for k in slice_needed_fields])
+ slicerec['reg_researchers'] = dict([(k, raw_slicerec[k])
+ for k in user_needed_fields])
+ #TODO Handle multiple slices for one user SA 10/12/12
+ #for now only take the first slice record associated to the rec user
+ ##slicerec = raw_slicerec['reg_slices_as_researcher'][0].__dict__
+ #del raw_slicerec['reg_slices_as_researcher']
+ #slicerec['reg_researchers'] = raw_slicerec
+ ##del slicerec['_sa_instance_state']
+
+ return slicerec
+
+ else:
+ return None
+
+
+ def _get_slice_records(self, slice_filter=None,
+ slice_filter_type=None):
+ """
+ Get the slice record depending on the slice filter and its type.
+ :param slice_filter: Can be either the slice hrn or the user's record
+ id.
+ :type slice_filter: string
+ :param slice_filter_type: describes the slice filter type used, can be
+ slice_hrn or record_id_user
+ :type: string
+ :returns: the slice record
+ :rtype:dict
+ .. seealso::_sql_get_slice_info_from_user
+ .. seealso:: _sql_get_slice_info
+ """
+
+ #Get list of slices based on the slice hrn
+ if slice_filter_type == 'slice_hrn':
+
+ #if get_authority(slice_filter) == self.root_auth:
+ #login = slice_filter.split(".")[1].split("_")[0]
+
+ slicerec = self._sql_get_slice_info(slice_filter)
+
+ if slicerec is None:
+ return None
+ #return login, None
+
+ #Get slice based on user id
+ if slice_filter_type == 'record_id_user':
+
+ slicerec = self._sql_get_slice_info_from_user(slice_filter)
+
+ if slicerec:
+ fixed_slicerec_dict = slicerec
+ #At this point if there is no login it means
+ #record_id_user filter has been used for filtering
+ #if login is None :
+ ##If theslice record is from iotlab
+ #if fixed_slicerec_dict['peer_authority'] is None:
+ #login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0]
+ #return login, fixed_slicerec_dict
+ return fixed_slicerec_dict
+ else:
+ return None
+
+
+
+ def GetSlices(self, slice_filter=None, slice_filter_type=None,
+ login=None):
+ """Get the slice records from the sfa db and add lease information
+ if any.
+
+ :param slice_filter: can be the slice hrn or slice record id in the db
+ depending on the slice_filter_type.
+ :param slice_filter_type: defines the type of the filtering used, Can be
+ either 'slice_hrn' or 'record_id'.
+ :type slice_filter: string
+ :type slice_filter_type: string
+ :returns: a slice dict if slice_filter and slice_filter_type
+ are specified and a matching entry is found in the db. The result
+ is put into a list.Or a list of slice dictionnaries if no filters
+ arespecified.
+
+ :rtype: list
+
+ """
+ #login = None
+ authorized_filter_types_list = ['slice_hrn', 'record_id_user']
+ return_slicerec_dictlist = []
+
+ #First try to get information on the slice based on the filter provided
+ if slice_filter_type in authorized_filter_types_list:
+ fixed_slicerec_dict = self._get_slice_records(slice_filter,
+ slice_filter_type)
+ # if the slice was not found in the sfa db
+ if fixed_slicerec_dict is None:
+ return return_slicerec_dictlist
+
+ slice_hrn = fixed_slicerec_dict['hrn']
+
+ logger.debug(" CORTEXLAB_API \tGetSlices login %s \
+ slice record %s slice_filter %s \
+ slice_filter_type %s " % (login,
+ fixed_slicerec_dict, slice_filter,
+ slice_filter_type))
+
+
+ #Now we have the slice record fixed_slicerec_dict, get the
+ #jobs associated to this slice
+ leases_list = []
+
+ leases_list = self.GetLeases(login=login)
+ #If no job is running or no job scheduled
+ #return only the slice record
+ if leases_list == [] and fixed_slicerec_dict:
+ return_slicerec_dictlist.append(fixed_slicerec_dict)
+
+ # if the jobs running don't belong to the user/slice we are looking
+ # for
+ leases_hrn = [lease['slice_hrn'] for lease in leases_list]
+ if slice_hrn not in leases_hrn:
+ return_slicerec_dictlist.append(fixed_slicerec_dict)
+ #If several experiments for one slice , put the slice record into
+ # each lease information dict
+ for lease in leases_list:
+ slicerec_dict = {}
+ logger.debug("CORTEXLAB_API.PY \tGetSlices slice_filter %s \
+ \t lease['slice_hrn'] %s"
+ % (slice_filter, lease['slice_hrn']))
+ if lease['slice_hrn'] == slice_hrn:
+ slicerec_dict['experiment_id'] = lease['lease_id']
+ #Update lease dict with the slice record
+ if fixed_slicerec_dict:
+ fixed_slicerec_dict['experiment_id'] = []
+ fixed_slicerec_dict['experiment_id'].append(
+ slicerec_dict['experiment_id'])
+ slicerec_dict.update(fixed_slicerec_dict)
+
+ slicerec_dict['slice_hrn'] = lease['slice_hrn']
+ slicerec_dict['hrn'] = lease['slice_hrn']
+ slicerec_dict['user'] = lease['user']
+ slicerec_dict.update(
+ {'list_node_ids':
+ {'hostname': lease['reserved_nodes']}})
+ slicerec_dict.update({'node_ids': lease['reserved_nodes']})
+
+
+ return_slicerec_dictlist.append(slicerec_dict)
+
+
+ logger.debug("CORTEXLAB_API.PY \tGetSlices \
+ slicerec_dict %s return_slicerec_dictlist %s \
+ lease['reserved_nodes'] \
+ %s" % (slicerec_dict, return_slicerec_dictlist,
+ lease['reserved_nodes']))
+
+ logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \
+ return_slicerec_dictlist %s"
+ % (return_slicerec_dictlist))
+
+ return return_slicerec_dictlist
+
+
+ else:
+ #Get all slices from the cortexlab sfa database , get the user info
+ # as well at the same time put them in dict format
+ request = self.api.dbsession().query(RegSlice)
+ query_slice_list = \
+ request.options(joinedload('reg_researchers')).all()
+
+ for record in query_slice_list:
+ tmp = record.__dict__
+ tmp['reg_researchers'] = tmp['reg_researchers'][0].__dict__
+ return_slicerec_dictlist.append(tmp)
+
+
+ #Get all the experiments reserved nodes
+ leases_list = self.testbed_shell.GetReservedNodes()
+
+ for fixed_slicerec_dict in return_slicerec_dictlist:
+ slicerec_dict = {}
+ #Check if the slice belongs to a cortexlab user
+ if fixed_slicerec_dict['peer_authority'] is None:
+ owner = fixed_slicerec_dict['hrn'].split(
+ ".")[1].split("_")[0]
+ else:
+ owner = None
+ for lease in leases_list:
+ if owner == lease['user']:
+ slicerec_dict['experiment_id'] = lease['lease_id']
+
+ #for reserved_node in lease['reserved_nodes']:
+ logger.debug("CORTEXLAB_API.PY \tGetSlices lease %s "
+ % (lease))
+ slicerec_dict.update(fixed_slicerec_dict)
+ slicerec_dict.update({'node_ids':
+ lease['reserved_nodes']})
+ slicerec_dict.update({'list_node_ids':
+ {'hostname':
+ lease['reserved_nodes']}})
+
+
+ fixed_slicerec_dict.update(slicerec_dict)
+
+ logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \
+ return_slicerec_dictlist %s \t slice_filter %s " \
+ %(return_slicerec_dictlist, slice_filter))
+
+ return return_slicerec_dictlist
+
+ def AddLeases(self, hostname_list, slice_record,
+ lease_start_time, lease_duration):
+
+ """Creates an experiment on the testbed corresponding to the information
+ provided as parameters. Adds the experiment id and the slice hrn in the
+ lease table on the additional sfa database so that we are able to know
+ which slice has which nodes.
+
+ :param hostname_list: list of nodes' OAR hostnames.
+ :param slice_record: sfa slice record, must contain login and hrn.
+ :param lease_start_time: starting time , unix timestamp format
+ :param lease_duration: duration in minutes
+
+ :type hostname_list: list
+ :type slice_record: dict
+ :type lease_start_time: integer
+ :type lease_duration: integer
+ :returns: experiment_id, can be None if the job request failed.
+
+ """
+ logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases hostname_list %s \
+ slice_record %s lease_start_time %s lease_duration %s "\
+ %( hostname_list, slice_record , lease_start_time, \
+ lease_duration))
+
+ username = slice_record['login']
+
+ experiment_id = self.testbed_shell.LaunchExperimentOnTestbed(
+ hostname_list,
+ slice_record['hrn'],
+ lease_start_time, lease_duration,
+ username)
+ if experiment_id is not None:
+ start_time = \
+ datetime.fromtimestamp(int(lease_start_time)).\
+ strftime(self.testbed_shell.time_format)
+ end_time = lease_start_time + lease_duration
+
+
+ logger.debug("CORTEXLAB_API \t AddLeases TURN ON LOGGING SQL \
+ %s %s %s "%(slice_record['hrn'], experiment_id, end_time))
+
+
+ logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases %s %s %s " \
+ %(type(slice_record['hrn']), type(experiment_id),
+ type(end_time)))
+
+ testbed_xp_row = LeaseTableXP(slice_hrn=slice_record['hrn'],
+ experiment_id=experiment_id,
+ end_time=end_time)
+
+ logger.debug("CORTEXLAB_API \t AddLeases testbed_xp_row %s" \
+ %(testbed_xp_row))
+ self.api.dbsession().add(testbed_xp_row)
+ self.api.dbsession().commit()
+
+ logger.debug("CORTEXLAB_API \t AddLeases hostname_list \
+ start_time %s " %(start_time))
+
+ return experiment_id
+
+
+ def GetLeases(self, lease_filter_dict=None, login=None):
+ """
+ Get the list of leases from testbed with complete information
+ about which slice owns which jobs and nodes.
+ Two purposes:
+ -Fetch all the experiment from the testbed (running, waiting..)
+ complete the reservation information with slice hrn
+ found in lease_table . If not available in the table,
+ assume it is a iotlab slice.
+ -Updates the iotlab table, deleting jobs when necessary.
+
+ :returns: reservation_list, list of dictionaries with 'lease_id',
+ 'reserved_nodes','slice_id', 'state', 'user', 'component_id_list',
+ 'slice_hrn', 'resource_ids', 't_from', 't_until'
+ :rtype: list
+
+ """
+
+ unfiltered_reservation_list = self.testbed_shell.GetReservedNodes(login)
+
+ reservation_list = []
+ #Find the slice associated with this user iotlab ldap uid
+ logger.debug(" CORTEXLAB \tGetLeases login %s\
+ unfiltered_reservation_list %s "
+ % (login, unfiltered_reservation_list))
+ #Create user dict first to avoid looking several times for
+ #the same user in LDAP SA 27/07/12
+ experiment_id_list = []
+ jobs_psql_query = self.api.dbsession().query(LeaseTableXP).all()
+ jobs_psql_dict = dict([(row.experiment_id, row.__dict__)
+ for row in jobs_psql_query])
+ #jobs_psql_dict = jobs_psql_dict)
+ logger.debug("CORTEXLAB \tGetLeases jobs_psql_dict %s"
+ % (jobs_psql_dict))
+ jobs_psql_id_list = [row.experiment_id for row in jobs_psql_query]
+
+ for resa in unfiltered_reservation_list:
+ logger.debug("CORTEXLAB \tGetLeases USER %s"
+ % (resa['user']))
+ #Construct list of jobs (runing, waiting..) from scheduler
+ experiment_id_list.append(resa['lease_id'])
+ #If there is information on the job in IOTLAB DB ]
+ #(slice used and job id)
+ if resa['lease_id'] in jobs_psql_dict:
+ job_info = jobs_psql_dict[resa['lease_id']]
+ logger.debug("CORTEXLAB \tGetLeases job_info %s"
+ % (job_info))
+ resa['slice_hrn'] = job_info['slice_hrn']
+ resa['slice_id'] = hrn_to_urn(resa['slice_hrn'], 'slice')
+
+ #otherwise, assume it is a iotlab slice:
+ else:
+ resa['slice_id'] = hrn_to_urn(self.testbed_shell.root_auth \
+ + '.' + resa['user'] + "_slice",
+ 'slice')
+ resa['slice_hrn'] = Xrn(resa['slice_id']).get_hrn()
+
+ resa['component_id_list'] = []
+ #Transform the hostnames into urns (component ids)
+ for node in resa['reserved_nodes']:
+
+ iotlab_xrn = xrn_object(self.testbed_shell.root_auth, node)
+ resa['component_id_list'].append(iotlab_xrn.urn)
+
+ if lease_filter_dict:
+ logger.debug("CORTEXLAB \tGetLeases \
+ \r\n leasefilter %s" % ( lease_filter_dict))
+
+ # filter_dict_functions = {
+ # 'slice_hrn' : IotlabShell.filter_lease_name,
+ # 't_from' : IotlabShell.filter_lease_start_time
+ # }
+ reservation_list = list(unfiltered_reservation_list)
+ for filter_type in lease_filter_dict:
+ logger.debug("CORTEXLAB \tGetLeases reservation_list %s" \
+ % (reservation_list))
+ reservation_list = self.testbed_shell.filter_lease(
+ reservation_list,filter_type,
+ lease_filter_dict[filter_type] )
+
+ # Filter the reservation list with a maximum timespan so that the
+ # leases and jobs running after this timestamp do not appear
+ # in the result leases.
+ # if 'start_time' in :
+ # if resa['start_time'] < lease_filter_dict['start_time']:
+ # reservation_list.append(resa)
+
+
+ # if 'name' in lease_filter_dict and \
+ # lease_filter_dict['name'] == resa['slice_hrn']:
+ # reservation_list.append(resa)
+
+
+ if lease_filter_dict is None:
+ reservation_list = unfiltered_reservation_list
+
+ self.update_experiments_in_lease_table(experiment_id_list,
+ jobs_psql_id_list)
+
+ logger.debug(" CORTEXLAB.PY \tGetLeases reservation_list %s"
+ % (reservation_list))
+ return reservation_list
+
+ def update_experiments_in_lease_table(self,
+ experiment_list_from_testbed, experiment_list_in_db):
+ """Cleans the lease_table by deleting expired and cancelled jobs.
+
+ Compares the list of experiment ids given by the testbed with the
+ experiment ids that are already in the database, deletes the
+ experiments that are no longer in the testbed experiment id list.
+
+ :param experiment_list_from_testbed: list of experiment ids coming
+ from testbed
+ :type experiment_list_from_testbed: list
+ :param experiment_list_in_db: list of experiment ids from the sfa
+ additionnal database.
+ :type experiment_list_in_db: list
+
+ :returns: None
+ """
+ #Turn the list into a set
+ set_experiment_list_in_db = set(experiment_list_in_db)
+
+ kept_experiments = set(experiment_list_from_testbed).intersection(set_experiment_list_in_db)
+ logger.debug("\r\n \t update_experiments_in_lease_table \
+ experiment_list_in_db %s \r\n \
+ experiment_list_from_testbed %s \
+ kept_experiments %s "
+ % (set_experiment_list_in_db,
+ experiment_list_from_testbed, kept_experiments))
+ deleted_experiments = set_experiment_list_in_db.difference(
+ kept_experiments)
+ deleted_experiments = list(deleted_experiments)
+ if len(deleted_experiments) > 0:
+ request = self.api.dbsession().query(LeaseTableXP)
+ request.filter(LeaseTableXP.experiment_id.in_(deleted_experiments)).delete(synchronize_session='fetch')
+ self.api.dbsession().commit()
+ return
+
+
+ def AddSlice(self, slice_record, user_record):
+ """
+
+ Add slice to the local cortexlab sfa tables if the slice comes
+ from a federated site and is not yet in the cortexlab sfa DB,
+ although the user has already a LDAP login.
+ Called by verify_slice during lease/sliver creation.
+
+ :param slice_record: record of slice, must contain hrn, gid, slice_id
+ and authority of the slice.
+ :type slice_record: dictionary
+ :param user_record: record of the user
+ :type user_record: RegUser
+
+ """
+
+ sfa_record = RegSlice(hrn=slice_record['hrn'],
+ gid=slice_record['gid'],
+ pointer=slice_record['slice_id'],
+ authority=slice_record['authority'])
+ logger.debug("CORTEXLAB_API.PY AddSlice sfa_record %s user_record %s"
+ % (sfa_record, user_record))
+ sfa_record.just_created()
+ self.api.dbsession().add(sfa_record)
+ self.api.dbsession().commit()
+ #Update the reg-researcher dependance table
+ sfa_record.reg_researchers = [user_record]
+ self.api.dbsession().commit()
+
+ return
+
def augment_records_with_testbed_info(self, record_list):
"""
# For client_helper.py compatibility
'key_ids': ''})
- #Get iotlab slice record and oar job id if any.
- recslice_list = self.testbed_shell.GetSlices(
+ #Get slice record and job id if any.
+ recslice_list = self.GetSlices(
slice_filter=str(record['hrn']),
slice_filter_type='slice_hrn')
#The record is a SFA user record.
#Get the information about his slice from Iotlab's DB
#and add it to the user record.
- recslice_list = self.testbed_shell.GetSlices(
+ recslice_list = self.GetSlices(
slice_filter=record['record_id'],
slice_filter_type='record_id_user')
recslice.update(
{'PI': [recuser['hrn']],
'researcher': [recuser['hrn']],
- 'name': record['hrn'],
+ 'name': recuser['hrn'],
'node_ids': [],
'experiment_id': [],
'person_ids': [recuser['record_id']]})
"""
#First get the slice with the slice hrn
- slice_list = self.testbed_shell.GetSlices(slice_filter=slice_hrn,
+ slice_list = self.GetSlices(slice_filter=slice_hrn,
slice_filter_type='slice_hrn')
if len(slice_list) == 0:
% (resources, res))
return result
-
def get_user_record(self, hrn):
"""
:rtype: RegUser
"""
- return dbsession.query(RegRecord).filter_by(hrn=hrn).first()
+ return self.api.dbsession().query(RegRecord).filter_by(hrn=hrn).first()
def testbed_name(self):
"""
"""
return self.hrn
- # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
- def aggregate_version(self):
- """
-
- Returns the testbed's supported rspec advertisement and request
- versions.
- :returns: rspec versions supported ad a dictionary.
- :rtype: dict
-
- """
- version_manager = VersionManager()
- ad_rspec_versions = []
- request_rspec_versions = []
- for rspec_version in version_manager.versions:
- if rspec_version.content_type in ['*', 'ad']:
- ad_rspec_versions.append(rspec_version.to_dict())
- if rspec_version.content_type in ['*', 'request']:
- request_rspec_versions.append(rspec_version.to_dict())
- return {
- 'testbed': self.testbed_name(),
- 'geni_request_rspec_versions': request_rspec_versions,
- 'geni_ad_rspec_versions': ad_rspec_versions}
def _get_requested_leases_list(self, rspec):
"""
return requested_xp_dict
+
def _process_requested_xp_dict(self, rspec):
"""
Turns the requested leases and information into a dictionary
return xp_dict
- def create_sliver(self, slice_urn, slice_hrn, creds, rspec_string,
- users, options):
- """Answer to CreateSliver.
-
- Creates the leases and slivers for the users from the information
- found in the rspec string.
- Launch experiment on OAR if the requested leases is valid. Delete
- no longer requested leases.
-
-
- :param creds: user's credentials
- :type creds: string
- :param users: user record list
- :type users: list
- :param options:
- :type options:
-
- :returns: a valid Rspec for the slice which has just been
- modified.
- :rtype: RSpec
- """
- aggregate = CortexlabAggregate(self)
-
- slices = CortexlabSlices(self)
- peer = slices.get_peer(slice_hrn)
- sfa_peer = slices.get_sfa_peer(slice_hrn)
- slice_record = None
-
- if not isinstance(creds, list):
- creds = [creds]
-
- if users:
- slice_record = users[0].get('slice_record', {})
- logger.debug("CORTEXLABDRIVER.PY \t ===============create_sliver \t\
- creds %s \r\n \r\n users %s"
- % (creds, users))
- slice_record['user'] = {'keys': users[0]['keys'],
- 'email': users[0]['email'],
- 'hrn': slice_record['reg-researchers'][0]}
- # parse rspec
- rspec = RSpec(rspec_string)
- logger.debug("CORTEXLABDRIVER.PY \t create_sliver \trspec.version \
- %s slice_record %s users %s"
- % (rspec.version, slice_record, users))
-
- # ensure site record exists?
- # ensure slice record exists
- #Removed options in verify_slice SA 14/08/12
- #Removed peer record in verify_slice SA 18/07/13
- sfa_slice = slices.verify_slice(slice_hrn, slice_record, sfa_peer)
-
- # ensure person records exists
- #verify_persons returns added persons but the return value
- #is not used
- #Removed peer record and sfa_peer in verify_persons SA 18/07/13
- slices.verify_persons(slice_hrn, sfa_slice, users, options=options)
- #requested_attributes returned by rspec.version.get_slice_attributes()
- #unused, removed SA 13/08/12
- #rspec.version.get_slice_attributes()
-
- logger.debug("CORTEXLABDRIVER.PY create_sliver slice %s " % (sfa_slice))
-
- # add/remove slice from nodes
-
- #requested_slivers = [node.get('component_id') \
- #for node in rspec.version.get_nodes_with_slivers()\
- #if node.get('authority_id') is self.testbed_shell.root_auth]
- #l = [ node for node in rspec.version.get_nodes_with_slivers() ]
- #logger.debug("SLADRIVER \tcreate_sliver requested_slivers \
- #requested_slivers %s listnodes %s" \
- #%(requested_slivers,l))
- #verify_slice_nodes returns nodes, but unused here. Removed SA 13/08/12.
- #slices.verify_slice_nodes(sfa_slice, requested_slivers, peer)
-
- requested_xp_dict = self._process_requested_xp_dict(rspec)
-
- logger.debug("CORTEXLABDRIVER.PY \tcreate_sliver requested_xp_dict %s "
- % (requested_xp_dict))
- #verify_slice_leases returns the leases , but the return value is unused
- #here. Removed SA 13/08/12
- slices.verify_slice_leases(sfa_slice,
- requested_xp_dict, peer)
-
- return aggregate.get_rspec(slice_xrn=slice_urn,
- login=sfa_slice['login'],
- version=rspec.version)
-
- def delete_sliver(self, slice_urn, slice_hrn, creds, options):
+ def delete(self, slice_urns, options={}):
"""
Deletes the lease associated with the slice hrn and the credentials
if the slice belongs to iotlab. Answer to DeleteSliver.
:param slice_urn: urn of the slice
- :param slice_hrn: name of the slice
- :param creds: slice credenials
:type slice_urn: string
- :type slice_hrn: string
- :type creds: ? unused
+
:returns: 1 if the slice to delete was not found on iotlab,
True if the deletion was successful, False otherwise otherwise.
.. note:: creds are unused, and are not used either in the dummy driver
delete_sliver .
"""
-
- sfa_slice_list = self.testbed_shell.GetSlices(
- slice_filter=slice_hrn,
- slice_filter_type='slice_hrn')
+ # collect sliver ids so we can update sliver allocation states after
+ # we remove the slivers.
+ aggregate = CortexlabAggregate(self)
+ slivers = aggregate.get_slivers(slice_urns)
+ if slivers:
+ # slice_id = slivers[0]['slice_id']
+ node_ids = []
+ sliver_ids = []
+ sliver_jobs_dict = {}
+ for sliver in slivers:
+ node_ids.append(sliver['node_id'])
+ sliver_ids.append(sliver['sliver_id'])
+ job_id = sliver['sliver_id'].split('+')[-1].split('-')[0]
+ sliver_jobs_dict[job_id] = sliver['sliver_id']
+ logger.debug("CORTEXLABDRIVER.PY delete_sliver slivers %s slice_urns %s"
+ % (slivers, slice_urns))
+ slice_hrn = urn_to_hrn(slice_urns[0])[0]
+
+ sfa_slice_list = self.GetSlices(slice_filter=slice_hrn,
+ slice_filter_type='slice_hrn')
if not sfa_slice_list:
return 1
#Delete all leases in the slice
for sfa_slice in sfa_slice_list:
- logger.debug("CORTEXLABDRIVER.PY delete_sliver slice %s" % (sfa_slice))
+ logger.debug("CORTEXLABDRIVER.PY delete_sliver slice %s" \
+ % (sfa_slice))
slices = CortexlabSlices(self)
# determine if this is a peer slice
logger.debug("CORTEXLABDRIVER.PY delete_sliver peer %s \
\r\n \t sfa_slice %s " % (peer, sfa_slice))
+ testbed_bool_ans = self.testbed_shell.DeleteSliceFromNodes(sfa_slice)
+ for job_id in testbed_bool_ans:
+ # if the job has not been successfully deleted
+ # don't delete the associated sliver
+ # remove it from the sliver list
+ if testbed_bool_ans[job_id] is False:
+ sliver = sliver_jobs_dict[job_id]
+ sliver_ids.remove(sliver)
try:
- self.testbed_shell.DeleteSliceFromNodes(sfa_slice)
- return True
- except:
- return False
-
- def list_resources (self, slice_urn, slice_hrn, creds, options):
- """
-
- List resources from the iotlab aggregate and returns a Rspec
- advertisement with resources found when slice_urn and slice_hrn are
- None (in case of resource discovery).
- If a slice hrn and urn are provided, list experiment's slice
- nodes in a rspec format. Answer to ListResources.
- Caching unused.
-
- :param slice_urn: urn of the slice
- :param slice_hrn: name of the slice
- :param creds: slice credenials
- :type slice_urn: string
- :type slice_hrn: string
- :type creds: ? unused
- :param options: options used when listing resources (list_leases, info,
- geni_available)
- :returns: rspec string in xml
- :rtype: string
-
- .. note:: creds are unused
- """
-
- #cached_requested = options.get('cached', True)
-
- version_manager = VersionManager()
- # get the rspec's return format from options
- rspec_version = \
- version_manager.get_version(options.get('geni_rspec_version'))
- version_string = "rspec_%s" % (rspec_version)
-
- #panos adding the info option to the caching key (can be improved)
- if options.get('info'):
- version_string = version_string + "_" + \
- options.get('info', 'default')
-
- # Adding the list_leases option to the caching key
- if options.get('list_leases'):
- version_string = version_string + "_" + \
- options.get('list_leases', 'default')
-
- # Adding geni_available to caching key
- if options.get('geni_available'):
- version_string = version_string + "_" + \
- str(options.get('geni_available'))
-
- # look in cache first
- #if cached_requested and self.cache and not slice_hrn:
- #rspec = self.cache.get(version_string)
- #if rspec:
- #logger.debug("IotlabDriver.ListResources: \
- #returning cached advertisement")
- #return rspec
-
- #panos: passing user-defined options
- aggregate = CortexlabAggregate(self)
- rspec = aggregate.get_rspec(slice_xrn=slice_urn,
- version=rspec_version, options=options)
-
- # cache the result
- #if self.cache and not slice_hrn:
- #logger.debug("Iotlab.ListResources: stores advertisement in cache")
- #self.cache.add(version_string, rspec)
-
- return rspec
+ dbsession = self.api.dbsession()
+ SliverAllocation.delete_allocations(sliver_ids, dbsession)
+ except :
+ logger.log_exc("CORTEXLABDRIVER.PY delete error ")
+ # prepare return struct
+ geni_slivers = []
+ for sliver in slivers:
+ geni_slivers.append(
+ {'geni_sliver_urn': sliver['sliver_id'],
+ 'geni_allocation_status': 'geni_unallocated',
+ 'geni_expires': datetime_to_string(utcparse(sliver['expires']))})
+ return geni_slivers
def list_slices(self, creds, options):
"""Answer to ListSlices.
:returns: slice urns list
:rtype: list
- .. note:: creds are unused
+ .. note:: creds are unused- SA 12/12/13
"""
# look in cache first
#if self.cache:
# get data from db
- slices = self.testbed_shell.GetSlices()
+ slices = self.GetSlices()
logger.debug("CORTEXLABDRIVER.PY \tlist_slices hrn %s \r\n \r\n"
% (slices))
slice_hrns = [iotlab_slice['hrn'] for iotlab_slice in slices]
"""
return -1
-
def update(self, old_sfa_record, new_sfa_record, hrn, new_key):
"""
- No site or node record update allowed in Cortexlab. The only modifications
+ No site or node record update allowed in Iotlab. The only modifications
authorized here are key deletion/addition on an existing user and
password change. On an existing user, CAN NOT BE MODIFIED: 'first_name',
- 'last_name', 'email'. DOES NOT EXIST IN LDAP: 'phone', 'url', 'bio',
+ 'last_name', 'email'. DOES NOT EXIST IN SENSLAB: 'phone', 'url', 'bio',
'title', 'accepted_aup'. A slice is bound to its user, so modifying the
user's ssh key should nmodify the slice's GID after an import procedure.
:type hrn: string
TODO: needs review
- .. seealso:: update in driver.py.
-
- """
- pointer = old_sfa_record['pointer']
- old_sfa_record_type = old_sfa_record['type']
-
- # new_key implemented for users only
- if new_key and old_sfa_record_type not in ['user']:
- raise UnknownSfaType(old_sfa_record_type)
-
- if old_sfa_record_type == "user":
- update_fields = {}
- all_fields = new_sfa_record
- for key in all_fields.keys():
- if key in ['key', 'password']:
- update_fields[key] = all_fields[key]
-
- if new_key:
- # must check this key against the previous one if it exists
- persons = self.testbed_shell.GetPersons([old_sfa_record])
- person = persons[0]
- keys = [person['pkey']]
- #Get all the person's keys
- keys_dict = self.testbed_shell.GetKeys(keys)
-
- # Delete all stale keys, meaning the user has only one key
- #at a time
- #TODO: do we really want to delete all the other keys?
- #Is this a problem with the GID generation to have multiple
- #keys? SA 30/05/13
- key_exists = False
- if key in keys_dict:
- key_exists = True
- else:
- #remove all the other keys
- for key in keys_dict:
- self.testbed_shell.DeleteKey(person, key)
- self.testbed_shell.AddPersonKey(
- person, {'sshPublicKey': person['pkey']},
- {'sshPublicKey': new_key})
+ .. warning:: SA 12/12/13 - Removed. should be done in iotlabimporter
+ since users, keys and slice are managed by the LDAP.
+
+ """
+ # pointer = old_sfa_record['pointer']
+ # old_sfa_record_type = old_sfa_record['type']
+
+ # # new_key implemented for users only
+ # if new_key and old_sfa_record_type not in ['user']:
+ # raise UnknownSfaType(old_sfa_record_type)
+
+ # if old_sfa_record_type == "user":
+ # update_fields = {}
+ # all_fields = new_sfa_record
+ # for key in all_fields.keys():
+ # if key in ['key', 'password']:
+ # update_fields[key] = all_fields[key]
+
+ # if new_key:
+ # # must check this key against the previous one if it exists
+ # persons = self.testbed_shell.GetPersons([old_sfa_record])
+ # person = persons[0]
+ # keys = [person['pkey']]
+ # #Get all the person's keys
+ # keys_dict = self.GetKeys(keys)
+
+ # # Delete all stale keys, meaning the user has only one key
+ # #at a time
+ # #TODO: do we really want to delete all the other keys?
+ # #Is this a problem with the GID generation to have multiple
+ # #keys? SA 30/05/13
+ # key_exists = False
+ # if key in keys_dict:
+ # key_exists = True
+ # else:
+ # #remove all the other keys
+ # for key in keys_dict:
+ # self.testbed_shell.DeleteKey(person, key)
+ # self.testbed_shell.AddPersonKey(
+ # person, {'sshPublicKey': person['pkey']},
+ # {'sshPublicKey': new_key})
+ logger.warning ("UNDEFINED - Update should be done by the \
+ iotlabimporter")
return True
+
def remove(self, sfa_record):
"""
return self.testbed_shell.DeletePerson(sfa_record)
elif sfa_record_type == 'slice':
- if self.testbed_shell.GetSlices(slice_filter=hrn,
- slice_filter_type='slice_hrn'):
+ if self.GetSlices(slice_filter=hrn,
+ slice_filter_type='slice_hrn'):
ret = self.testbed_shell.DeleteSlice(sfa_record)
return True
+
+ def check_sliver_credentials(self, creds, urns):
+ """Check that the sliver urns belongs to the slice specified in the
+ credentials.
+
+ :param urns: list of sliver urns.
+ :type urns: list.
+ :param creds: slice credentials.
+ :type creds: Credential object.
+
+
+ """
+ # build list of cred object hrns
+ slice_cred_names = []
+ for cred in creds:
+ slice_cred_hrn = Credential(cred=cred).get_gid_object().get_hrn()
+ slicename = IotlabXrn(xrn=slice_cred_hrn).iotlab_slicename()
+ slice_cred_names.append(slicename)
+
+ # look up slice name of slivers listed in urns arg
+
+ slice_ids = []
+ for urn in urns:
+ sliver_id_parts = Xrn(xrn=urn).get_sliver_id_parts()
+ try:
+ slice_ids.append(int(sliver_id_parts[0]))
+ except ValueError:
+ pass
+
+ if not slice_ids:
+ raise Forbidden("sliver urn not provided")
+
+ slices = self.GetSlices(slice_ids)
+ sliver_names = [single_slice['name'] for single_slice in slices]
+
+ # make sure we have a credential for every specified sliver
+ for sliver_name in sliver_names:
+ if sliver_name not in slice_cred_names:
+ msg = "Valid credential not found for target: %s" % sliver_name
+ raise Forbidden(msg)
+
+ ########################################
+ ########## aggregate oriented
+ ########################################
+
+ # 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
+ def aggregate_version(self):
+ """
+
+ Returns the testbed's supported rspec advertisement and request
+ versions.
+ :returns: rspec versions supported ad a dictionary.
+ :rtype: dict
+
+ """
+ version_manager = VersionManager()
+ ad_rspec_versions = []
+ request_rspec_versions = []
+ for rspec_version in version_manager.versions:
+ if rspec_version.content_type in ['*', 'ad']:
+ ad_rspec_versions.append(rspec_version.to_dict())
+ if rspec_version.content_type in ['*', 'request']:
+ request_rspec_versions.append(rspec_version.to_dict())
+ return {
+ 'testbed': self.testbed_name(),
+ 'geni_request_rspec_versions': request_rspec_versions,
+ 'geni_ad_rspec_versions': ad_rspec_versions}
+
+
+
+ # first 2 args are None in case of resource discovery
+ def list_resources (self, version=None, options={}):
+ aggregate = CortexlabAggregate(self)
+ rspec = aggregate.list_resources(version=version, options=options)
+ return rspec
+
+
+ def describe(self, urns, version, options={}):
+ aggregate = CortexlabAggregate(self)
+ return aggregate.describe(urns, version=version, options=options)
+
+ def status (self, urns, options={}):
+ aggregate = CortexlabAggregate(self)
+ desc = aggregate.describe(urns, version='GENI 3')
+ status = {'geni_urn': desc['geni_urn'],
+ 'geni_slivers': desc['geni_slivers']}
+ return status
+
+
+ def allocate (self, urn, rspec_string, expiration, options={}):
+ xrn = Xrn(urn)
+ aggregate = CortexlabAggregate(self)
+
+ slices = CortexlabSlices(self)
+ peer = slices.get_peer(xrn.get_hrn())
+ sfa_peer = slices.get_sfa_peer(xrn.get_hrn())
+
+ slice_record = None
+ users = options.get('geni_users', [])
+
+ sfa_users = options.get('sfa_users', [])
+ if sfa_users:
+ slice_record = sfa_users[0].get('slice_record', [])
+
+ # parse rspec
+ rspec = RSpec(rspec_string)
+ # requested_attributes = rspec.version.get_slice_attributes()
+
+ # ensure site record exists
+
+ # ensure slice record exists
+
+ current_slice = slices.verify_slice(xrn.hrn, slice_record, sfa_peer)
+ logger.debug("IOTLABDRIVER.PY \t ===============allocate \t\
+ \r\n \r\n current_slice %s" % (current_slice))
+ # ensure person records exists
+
+ # oui c'est degueulasse, le slice_record se retrouve modifie
+ # dans la methode avec les infos du user, els infos sont propagees
+ # dans verify_slice_leases
+ persons = slices.verify_persons(xrn.hrn, slice_record, users,
+ options=options)
+ # ensure slice attributes exists
+ # slices.verify_slice_attributes(slice, requested_attributes,
+ # options=options)
+
+ # add/remove slice from nodes
+ requested_xp_dict = self._process_requested_xp_dict(rspec)
+
+ logger.debug("IOTLABDRIVER.PY \tallocate requested_xp_dict %s "
+ % (requested_xp_dict))
+ request_nodes = rspec.version.get_nodes_with_slivers()
+ nodes_list = []
+ for start_time in requested_xp_dict:
+ lease = requested_xp_dict[start_time]
+ for hostname in lease['hostname']:
+ nodes_list.append(hostname)
+
+ # nodes = slices.verify_slice_nodes(slice_record,request_nodes, peer)
+ logger.debug("IOTLABDRIVER.PY \tallocate nodes_list %s slice_record %s"
+ % (nodes_list, slice_record))
+
+ # add/remove leases
+ rspec_requested_leases = rspec.version.get_leases()
+ leases = slices.verify_slice_leases(slice_record,
+ requested_xp_dict, peer)
+ logger.debug("IOTLABDRIVER.PY \tallocate leases %s \
+ rspec_requested_leases %s" % (leases,
+ rspec_requested_leases))
+ # update sliver allocations
+ for hostname in nodes_list:
+ client_id = hostname
+ node_urn = xrn_object(self.testbed_shell.root_auth, hostname).urn
+ component_id = node_urn
+ slice_urn = current_slice['reg-urn']
+ for lease in leases:
+ if hostname in lease['reserved_nodes']:
+ index = lease['reserved_nodes'].index(hostname)
+ sliver_hrn = '%s.%s-%s' % (self.hrn, lease['lease_id'],
+ lease['resource_ids'][index] )
+ sliver_id = Xrn(sliver_hrn, type='sliver').urn
+ record = SliverAllocation(sliver_id=sliver_id, client_id=client_id,
+ component_id=component_id,
+ slice_urn = slice_urn,
+ allocation_state='geni_allocated')
+ record.sync(self.api.dbsession())
+
+ return aggregate.describe([xrn.get_urn()], version=rspec.version)
+
+ def provision(self, urns, options={}):
+ # update users
+ slices = CortexlabSlices(self)
+ aggregate = CortexlabAggregate(self)
+ slivers = aggregate.get_slivers(urns)
+ current_slice = slivers[0]
+ peer = slices.get_peer(current_slice['hrn'])
+ sfa_peer = slices.get_sfa_peer(current_slice['hrn'])
+ users = options.get('geni_users', [])
+ # persons = slices.verify_persons(current_slice['hrn'],
+ # current_slice, users, peer, sfa_peer, options=options)
+ # slices.handle_peer(None, None, persons, peer)
+ # update sliver allocation states and set them to geni_provisioned
+ sliver_ids = [sliver['sliver_id'] for sliver in slivers]
+ dbsession = self.api.dbsession()
+ SliverAllocation.set_allocations(sliver_ids, 'geni_provisioned',
+ dbsession)
+ version_manager = VersionManager()
+ rspec_version = version_manager.get_version(options[
+ 'geni_rspec_version'])
+ return self.describe(urns, rspec_version, options=options)
+
+
+
from sfa.util.sfalogging import logger
-from sfa.storage.alchemy import dbsession
-from sqlalchemy.orm import joinedload
-from sfa.storage.model import RegRecord, RegUser, RegSlice, RegKey
-from sfa.iotlab.iotlabpostgres import TestbedAdditionalSfaDB, LeaseTableXP
+from sfa.iotlab.iotlabpostgres import LeaseTableXP
from sfa.cortexlab.LDAPapi import LDAPapi
-from sfa.util.xrn import Xrn, hrn_to_urn, get_authority
-from sfa.trust.certificate import Keypair, convert_public_key
-from sfa.trust.gid import create_uuid
-from sfa.trust.hierarchy import Hierarchy
-from sfa.iotlab.iotlabaggregate import iotlab_xrn_object
+from sfa.iotlab.iotlabxrn import xrn_object
from sfa.cortexlab.cortexlabnodes import CortexlabQueryNodes
class CortexlabShell():
"""
return CortexlabShell._MINIMUM_DURATION
-
- def GetPeers(self, peer_filter=None ):
- """ Gathers registered authorities in SFA DB and looks for specific peer
- if peer_filter is specified.
- :param peer_filter: name of the site authority looked for.
- :type peer_filter: string
- :returns: list of records.
-
- """
-
- existing_records = {}
- existing_hrns_by_types = {}
- logger.debug("CORTEXLAB_API \tGetPeers peer_filter %s " % (peer_filter))
- all_records = dbsession.query(RegRecord).filter(RegRecord.type.like('%authority%')).all()
-
- for record in all_records:
- existing_records[(record.hrn, record.type)] = record
- if record.type not in existing_hrns_by_types:
- existing_hrns_by_types[record.type] = [record.hrn]
- else:
- existing_hrns_by_types[record.type].append(record.hrn)
-
- logger.debug("CORTEXLAB_API \tGetPeer\texisting_hrns_by_types %s "
- % (existing_hrns_by_types))
- records_list = []
-
- try:
- if peer_filter:
- records_list.append(existing_records[(peer_filter,
- 'authority')])
- else:
- for hrn in existing_hrns_by_types['authority']:
- records_list.append(existing_records[(hrn, 'authority')])
-
- logger.debug("CORTEXLAB_API \tGetPeer \trecords_list %s "
- % (records_list))
-
- except KeyError:
- pass
-
- return_records = records_list
- logger.debug("CORTEXLAB_API \tGetPeer return_records %s "
- % (return_records))
- return return_records
-
#TODO : Handling OR request in make_ldap_filters_from_records
#instead of the for loop
#over the records' list
return_node_list = node_list_dict
return return_node_list
- def AddSlice(self, slice_record, user_record):
- """
-
- Add slice to the local cortexlab sfa tables if the slice comes
- from a federated site and is not yet in the cortexlab sfa DB,
- although the user has already a LDAP login.
- Called by verify_slice during lease/sliver creation.
-
- :param slice_record: record of slice, must contain hrn, gid, slice_id
- and authority of the slice.
- :type slice_record: dictionary
- :param user_record: record of the user
- :type user_record: RegUser
- """
-
- sfa_record = RegSlice(hrn=slice_record['hrn'],
- gid=slice_record['gid'],
- pointer=slice_record['slice_id'],
- authority=slice_record['authority'])
- logger.debug("CORTEXLAB_API.PY AddSlice sfa_record %s user_record %s"
- % (sfa_record, user_record))
- sfa_record.just_created()
- dbsession.add(sfa_record)
- dbsession.commit()
- #Update the reg-researcher dependance table
- sfa_record.reg_researchers = [user_record]
- dbsession.commit()
-
- return
def GetSites(self, site_filter_name_list=None, return_fields_list=None):
return delete_failed or True
- def __add_person_to_db(self, user_dict):
- """
- Add a federated user straight to db when the user issues a lease
- request with iotlab nodes and that he has not registered with cortexlab
- yet (that is he does not have a LDAP entry yet).
- Uses parts of the routines in CortexlabImport when importing user
- from LDAP.
- Called by AddPerson, right after LdapAddUser.
- :param user_dict: Must contain email, hrn and pkey to get a GID
- and be added to the SFA db.
- :type user_dict: dict
-
- """
- check_if_exists = \
- dbsession.query(RegUser).filter_by(email = user_dict['email']).first()
- #user doesn't exists
- if not check_if_exists:
- logger.debug("__add_person_to_db \t Adding %s \r\n \r\n \
- " %(user_dict))
- hrn = user_dict['hrn']
- person_urn = hrn_to_urn(hrn, 'user')
- pubkey = user_dict['pkey']
- try:
- pkey = convert_public_key(pubkey)
- except TypeError:
- #key not good. create another pkey
- logger.warn('__add_person_to_db: unable to convert public \
- key for %s' %(hrn ))
- pkey = Keypair(create=True)
-
-
- if pubkey is not None and pkey is not None :
- hierarchy = Hierarchy()
- person_gid = hierarchy.create_gid(person_urn, create_uuid(), \
- pkey)
- if user_dict['email']:
- logger.debug("__add_person_to_db \r\n \r\n \
- IOTLAB IMPORTER PERSON EMAIL OK email %s "\
- %(user_dict['email']))
- person_gid.set_email(user_dict['email'])
-
- user_record = RegUser(hrn=hrn , pointer= '-1', \
- authority=get_authority(hrn), \
- email=user_dict['email'], gid = person_gid)
- user_record.reg_keys = [RegKey(user_dict['pkey'])]
- user_record.just_created()
- dbsession.add (user_record)
- dbsession.commit()
- return
-
-
- def AddPerson(self, record):
- """
-
- Adds a new account. Any fields specified in records are used,
- otherwise defaults are used. Creates an appropriate login by calling
- LdapAddUser.
-
- :param record: dictionary with the sfa user's properties.
- :returns: a dicitonary with the status. If successful, the dictionary
- boolean is set to True and there is a 'uid' key with the new login
- added to LDAP, otherwise the bool is set to False and a key
- 'message' is in the dictionary, with the error message.
- :rtype: dict
-
- """
- ret = self.ldap.LdapAddUser(record)
-
- if ret['bool'] is True:
- record['hrn'] = self.root_auth + '.' + ret['uid']
- logger.debug("CORTEXLAB_API AddPerson return code %s record %s "
- % (ret, record))
- self.__add_person_to_db(record)
- return ret
-
-
-
-
-
#TODO AddPersonKey 04/07/2012 SA
def AddPersonKey(self, person_uid, old_attributes_dict, new_key_dict):
"""Adds a new key to the specified account. Adds the key to the
logger.warning("CORTEXLAB_API AddPersonKey EMPTY - DO NOTHING \r\n ")
return ret['bool']
+ def DeleteLeases(self, leases_id_list, slice_hrn):
+ """
+
+ Deletes several leases, based on their experiment ids and the slice
+ they are associated with. Uses DeleteOneLease to delete the
+ experiment on the testbed. Note that one slice can contain multiple
+ experiments, and in this
+ case all the experiments in the leases_id_list MUST belong to this
+ same slice, since there is only one slice hrn provided here.
+
+ :param leases_id_list: list of job ids that belong to the slice whose
+ slice hrn is provided.
+ :param slice_hrn: the slice hrn.
+ :type slice_hrn: string
+
+ .. warning:: Does not have a return value since there was no easy
+ way to handle failure when dealing with multiple job delete. Plus,
+ there was no easy way to report it to the user.
+
+ """
+ logger.debug("CORTEXLAB_API DeleteLeases leases_id_list %s slice_hrn %s \
+ \r\n " %(leases_id_list, slice_hrn))
+ for experiment_id in leases_id_list:
+ self.DeleteOneLease(experiment_id, slice_hrn)
+
+ return
+
@staticmethod
def _process_walltime(duration):
return experiment_id
- def AddLeases(self, hostname_list, slice_record,
- lease_start_time, lease_duration):
- """Creates an experiment on the testbed corresponding to the information
- provided as parameters. Adds the experiment id and the slice hrn in the
- lease table on the additional sfa database so that we are able to know
- which slice has which nodes.
-
- :param hostname_list: list of nodes' OAR hostnames.
- :param slice_record: sfa slice record, must contain login and hrn.
- :param lease_start_time: starting time , unix timestamp format
- :param lease_duration: duration in minutes
-
- :type hostname_list: list
- :type slice_record: dict
- :type lease_start_time: integer
- :type lease_duration: integer
-
- """
- logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases hostname_list %s \
- slice_record %s lease_start_time %s lease_duration %s "\
- %( hostname_list, slice_record , lease_start_time, \
- lease_duration))
-
- username = slice_record['login']
-
- experiment_id = self.LaunchExperimentOnTestbed(hostname_list, \
- slice_record['hrn'], \
- lease_start_time, lease_duration, \
- username)
- start_time = \
- datetime.fromtimestamp(int(lease_start_time)).\
- strftime(self.time_format)
- end_time = lease_start_time + lease_duration
-
-
- logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases TURN ON LOGGING SQL \
- %s %s %s "%(slice_record['hrn'], experiment_id, end_time))
-
-
- logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases %s %s %s " \
- %(type(slice_record['hrn']), type(experiment_id),
- type(end_time)))
-
- testbed_xp_row = LeaseTableXP(slice_hrn=slice_record['hrn'],
- experiment_id=experiment_id, end_time=end_time)
-
- logger.debug("CORTEXLAB_API \r\n \r\n \t AddLeases testbed_xp_row %s" \
- %(testbed_xp_row))
- self.leases_db.testbed_session.add(testbed_xp_row)
- self.leases_db.testbed_session.commit()
-
- logger.debug("CORTEXLAB_API \t AddLeases hostname_list start_time %s " \
- %(start_time))
-
- return
-
- def DeleteLeases(self, leases_id_list, slice_hrn):
- """
-
- Deletes several leases, based on their experiment ids and the slice
- they are associated with. Uses DeleteOneLease to delete the
- experiment on the testbed. Note that one slice can contain multiple
- experiments, and in this
- case all the experiments in the leases_id_list MUST belong to this
- same slice, since there is only one slice hrn provided here.
-
- :param leases_id_list: list of job ids that belong to the slice whose
- slice hrn is provided.
- :param slice_hrn: the slice hrn.
- :type slice_hrn: string
-
- .. warning:: Does not have a return value since there was no easy
- way to handle failure when dealing with multiple job delete. Plus,
- there was no easy way to report it to the user.
-
- """
- logger.debug("CORTEXLAB_API DeleteLeases leases_id_list %s slice_hrn %s \
- \r\n " %(leases_id_list, slice_hrn))
- for experiment_id in leases_id_list:
- self.DeleteOneLease(experiment_id, slice_hrn)
-
- return
#Delete the jobs from job_iotlab table
def DeleteSliceFromNodes(self, slice_record):
Experiments which last less than 10 min (600 sec) are invalid"""
return self.grain
-
- # @staticmethod
- # def update_experiments_in_additional_sfa_db( job_oar_list, jobs_psql):
- # """ Cleans the iotlab db by deleting expired and cancelled jobs.
- # Compares the list of job ids given by OAR with the job ids that
- # are already in the database, deletes the jobs that are no longer in
- # the OAR job id list.
- # :param job_oar_list: list of job ids coming from OAR
- # :type job_oar_list: list
- # :param job_psql: list of job ids cfrom the database.
- # type job_psql: list
- # """
- # #Turn the list into a set
- # set_jobs_psql = set(jobs_psql)
-
- # kept_jobs = set(job_oar_list).intersection(set_jobs_psql)
- # logger.debug ( "\r\n \t\ update_experiments_in_additional_sfa_db jobs_psql %s \r\n \t \
- # job_oar_list %s kept_jobs %s "%(set_jobs_psql, job_oar_list, kept_jobs))
- # deleted_jobs = set_jobs_psql.difference(kept_jobs)
- # deleted_jobs = list(deleted_jobs)
- # if len(deleted_jobs) > 0:
- # self.leases_db.testbed_session.query(LeaseTableXP).filter(LeaseTableXP.job_id.in_(deleted_jobs)).delete(synchronize_session='fetch')
- # self.leases_db.testbed_session.commit()
-
- # return
-
@staticmethod
- def filter_lease_name(reservation_list, filter_value):
+ def filter_lease(reservation_list, filter_type, filter_value ):
+ """Filters the lease reservation list by removing each lease whose
+ filter_type is not equal to the filter_value provided. Returns the list
+ of leases in one slice, defined by the slice_hrn if filter_type
+ is 'slice_hrn'. Otherwise, returns all leases scheduled starting from
+ the filter_value if filter_type is 't_from'.
+
+ :param reservation_list: leases list
+ :type reservation_list: list of dictionary
+ :param filter_type: can be either 't_from' or 'slice hrn'
+ :type filter_type: string
+ :param filter_value: depending on the filter_type, can be the slice_hrn
+ or can be defining a timespan.
+ :type filter_value: if filter_type is 't_from', filter_value is int.
+ if filter_type is 'slice_hrn', filter_value is a string.
+
+
+ :returns: filtered_reservation_list, contains only leases running or
+ scheduled in the given slice (wanted_slice).Dict keys are
+ 'lease_id','reserved_nodes','slice_id', 'state', 'user',
+ 'component_id_list','slice_hrn', 'resource_ids', 't_from', 't_until'
+ :rtype: list of dict
+
+ """
filtered_reservation_list = list(reservation_list)
- logger.debug("CORTEXLAB_API \t filter_lease_name reservation_list %s" \
+ logger.debug("IOTLAB_API \t filter_lease_name reservation_list %s" \
% (reservation_list))
- for reservation in reservation_list:
- if 'slice_hrn' in reservation and \
- reservation['slice_hrn'] != filter_value:
- filtered_reservation_list.remove(reservation)
+ try:
+ for reservation in reservation_list:
+ if \
+ (filter_type is 'slice_hrn' and \
+ reservation['slice_hrn'] != filter_value) or \
+ (filter_type is 't_from' and \
+ reservation['t_from'] > filter_value):
+ filtered_reservation_list.remove(reservation)
+ except TypeError:
+ logger.log_exc("Iotlabshell filter_lease : filter_type %s \
+ filter_value %s not in lease" %(filter_type,
+ filter_value))
- logger.debug("CORTEXLAB_API \t filter_lease_name filtered_reservation_list %s" \
- % (filtered_reservation_list))
return filtered_reservation_list
- @staticmethod
- def filter_lease_start_time(reservation_list, filter_value):
- filtered_reservation_list = list(reservation_list)
+ # @staticmethod
+ # def filter_lease_name(reservation_list, filter_value):
+ # filtered_reservation_list = list(reservation_list)
+ # logger.debug("CORTEXLAB_API \t filter_lease_name reservation_list %s" \
+ # % (reservation_list))
+ # for reservation in reservation_list:
+ # if 'slice_hrn' in reservation and \
+ # reservation['slice_hrn'] != filter_value:
+ # filtered_reservation_list.remove(reservation)
+
+ # logger.debug("CORTEXLAB_API \t filter_lease_name filtered_reservation_list %s" \
+ # % (filtered_reservation_list))
+ # return filtered_reservation_list
+
+ # @staticmethod
+ # def filter_lease_start_time(reservation_list, filter_value):
+ # filtered_reservation_list = list(reservation_list)
- for reservation in reservation_list:
- if 't_from' in reservation and \
- reservation['t_from'] > filter_value:
- filtered_reservation_list.remove(reservation)
+ # for reservation in reservation_list:
+ # if 't_from' in reservation and \
+ # reservation['t_from'] > filter_value:
+ # filtered_reservation_list.remove(reservation)
- return filtered_reservation_list
+ # return filtered_reservation_list
def complete_leases_info(self, unfiltered_reservation_list, db_xp_dict):
#Transform the hostnames into urns (component ids)
for node in resa['reserved_nodes']:
- iotlab_xrn = iotlab_xrn_object(self.root_auth, node)
+ iotlab_xrn = xrn_object(self.root_auth, node)
resa['component_id_list'].append(iotlab_xrn.urn)
return local_unfiltered_reservation_list, testbed_xp_list
- def GetLeases(self, lease_filter_dict=None, login=None):
- """
-
- Get the list of leases from the testbed with complete information
- about in which slice is running which experiment ans which nodes are
- involved.
- Two purposes:
- -Fetch all the experiments from the testbed (running, waiting..)
- complete the reservation information with slice hrn
- found in testbed_xp table. If not available in the table,
- assume it is a cortexlab slice.
- -Updates the cortexlab table, deleting jobs when necessary.
-
- :returns: reservation_list, list of dictionaries with 'lease_id',
- 'reserved_nodes','slice_id','user', 'component_id_list',
- 'slice_hrn', 'resource_ids', 't_from', 't_until'. Other
- keys can be returned if necessary, such as the 'state' of the lease,
- if the information has been added in GetReservedNodes.
- :rtype: list
-
- """
-
- unfiltered_reservation_list = self.GetReservedNodes(login)
-
- reservation_list = []
- #Find the slice associated with this user ldap uid
- logger.debug(" CORTEXLAB_API.PY \tGetLeases login %s\
- unfiltered_reservation_list %s "
- % (login, unfiltered_reservation_list))
- #Create user dict first to avoid looking several times for
- #the same user in LDAP SA 27/07/12
-
-
- db_xp_query = self.leases_db.testbed_session.query(LeaseTableXP).all()
- db_xp_dict = dict([(row.experiment_id, row.__dict__)
- for row in db_xp_query])
-
- logger.debug("CORTEXLAB_API \tGetLeases db_xp_dict %s"
- % (db_xp_dict))
- db_xp_id_list = [row.experiment_id for row in db_xp_query]
-
- required_fiels_in_leases = ['lease_id',
- 'reserved_nodes','slice_id', 'user', 'component_id_list',
- 'slice_hrn', 'resource_ids', 't_from', 't_until']
-
- # Add any missing information on the leases with complete_leases_info
- unfiltered_reservation_list, testbed_xp_list = \
- self.complete_leases_info(unfiltered_reservation_list,
- db_xp_dict)
- # Check that the list of leases is complete and have the mandatory
- # information
- format_status = self.ensure_format_is_valid(unfiltered_reservation_list,
- required_fiels_in_leases)
-
- if not format_status:
- logger.log_exc("\tCortexlabapi \t GetLeases : Missing fields in \
- reservation list")
- raise KeyError, "GetLeases : Missing fields in reservation list "
-
- if lease_filter_dict:
- logger.debug("CORTEXLAB_API \tGetLeases \
- \r\n leasefilter %s" % ( lease_filter_dict))
-
- filter_dict_functions = {
- 'slice_hrn' : CortexlabShell.filter_lease_name,
- 't_from' : CortexlabShell.filter_lease_start_time
- }
-
- reservation_list = list(unfiltered_reservation_list)
- for filter_type in lease_filter_dict:
- logger.debug("CORTEXLAB_API \tGetLeases reservation_list %s" \
- % (reservation_list))
- reservation_list = filter_dict_functions[filter_type](\
- reservation_list,lease_filter_dict[filter_type] )
-
-
- if lease_filter_dict is None:
- reservation_list = unfiltered_reservation_list
-
- self.leases_db.update_experiments_in_additional_sfa_db(
- testbed_xp_list, db_xp_id_list)
-
- logger.debug(" CORTEXLAB_API.PY \tGetLeases reservation_list %s"
- % (reservation_list))
- return reservation_list
-
-
-
#TODO FUNCTIONS SECTION 04/07/2012 SA
#return
- def GetKeys(self, key_filter=None):
- """Returns a dict of dict based on the key string. Each dict entry
- contains the key id, the ssh key, the user's email and the
- user's hrn.
- If key_filter is specified and is an array of key identifiers,
- only keys matching the filter will be returned.
-
- Admin may query all keys. Non-admins may only query their own keys.
- FROM PLC API DOC
-
- :returns: dict with ssh key as key and dicts as value.
- :rtype: dict
- """
- if key_filter is None:
- keys = dbsession.query(RegKey).options(joinedload('reg_user')).all()
- else:
- keys = dbsession.query(RegKey).options(joinedload('reg_user')).filter(RegKey.key.in_(key_filter)).all()
-
- key_dict = {}
- for key in keys:
- key_dict[key.key] = {'key_id': key.key_id, 'key': key.key,
- 'email': key.reg_user.email,
- 'hrn': key.reg_user.hrn}
-
- #ldap_rslt = self.ldap.LdapSearch({'enabled']=True})
- #user_by_email = dict((user[1]['mail'][0], user[1]['sshPublicKey']) \
- #for user in ldap_rslt)
- logger.debug("CORTEXLAB_API GetKeys -key_dict %s \r\n " % (key_dict))
- return key_dict
#TODO : test
def DeleteKey(self, user_record, key_string):
return ret['bool']
- def _sql_get_slice_info(self, slice_filter):
- """
- Get the slice record based on the slice hrn. Fetch the record of the
- user associated with the slice by using joinedload based on the
- reg_researcher relationship.
-
- :param slice_filter: the slice hrn we are looking for
- :type slice_filter: string
- :returns: the slice record enhanced with the user's information if the
- slice was found, None it wasn't.
-
- :rtype: dict or None.
- """
- #DO NOT USE RegSlice - reg_researchers to get the hrn
- #of the user otherwise will mess up the RegRecord in
- #Resolve, don't know why - SA 08/08/2012
-
- #Only one entry for one user = one slice in testbed_xp table
- #slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
- raw_slicerec = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn=slice_filter).first()
- #raw_slicerec = dbsession.query(RegRecord).filter_by(hrn = slice_filter).first()
- if raw_slicerec:
- #load_reg_researcher
- #raw_slicerec.reg_researchers
- raw_slicerec = raw_slicerec.__dict__
- logger.debug(" CORTEXLAB_API \t _sql_get_slice_info slice_filter %s \
- raw_slicerec %s" % (slice_filter, raw_slicerec))
- slicerec = raw_slicerec
- #only one researcher per slice so take the first one
- #slicerec['reg_researchers'] = raw_slicerec['reg_researchers']
- #del slicerec['reg_researchers']['_sa_instance_state']
- return slicerec
-
- else:
- return None
-
-
- def _sql_get_slice_info_from_user(self, slice_filter):
- """
- Get the slice record based on the user recordid by using a joinedload
- on the relationship reg_slices_as_researcher. Format the sql record
- into a dict with the mandatory fields for user and slice.
- :returns: dict with slice record and user record if the record was found
- based on the user's id, None if not..
- :rtype:dict or None..
- """
- #slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
- raw_slicerec = dbsession.query(RegUser).options(joinedload('reg_slices_as_researcher')).filter_by(record_id=slice_filter).first()
- #raw_slicerec = dbsession.query(RegRecord).filter_by(record_id = slice_filter).first()
- #Put it in correct order
- user_needed_fields = ['peer_authority', 'hrn', 'last_updated',
- 'classtype', 'authority', 'gid', 'record_id',
- 'date_created', 'type', 'email', 'pointer']
- slice_needed_fields = ['peer_authority', 'hrn', 'last_updated',
- 'classtype', 'authority', 'gid', 'record_id',
- 'date_created', 'type', 'pointer']
- if raw_slicerec:
- #raw_slicerec.reg_slices_as_researcher
- raw_slicerec = raw_slicerec.__dict__
- slicerec = {}
- slicerec = \
- dict([(k, raw_slicerec[
- 'reg_slices_as_researcher'][0].__dict__[k])
- for k in slice_needed_fields])
- slicerec['reg_researchers'] = dict([(k, raw_slicerec[k])
- for k in user_needed_fields])
- #TODO Handle multiple slices for one user SA 10/12/12
- #for now only take the first slice record associated to the rec user
- ##slicerec = raw_slicerec['reg_slices_as_researcher'][0].__dict__
- #del raw_slicerec['reg_slices_as_researcher']
- #slicerec['reg_researchers'] = raw_slicerec
- ##del slicerec['_sa_instance_state']
-
- return slicerec
-
- else:
- return None
-
- def _get_slice_records(self, slice_filter=None,
- slice_filter_type=None):
- """
- Get the slice record depending on the slice filter and its type.
- :param slice_filter: Can be either the slice hrn or the user's record
- id.
- :type slice_filter: string
- :param slice_filter_type: describes the slice filter type used, can be
- slice_hrn or record_id_user
- :type: string
- :returns: the slice record
- :rtype:dict
- .. seealso::_sql_get_slice_info_from_user
- .. seealso:: _sql_get_slice_info
- """
-
- #Get list of slices based on the slice hrn
- if slice_filter_type == 'slice_hrn':
-
- #if get_authority(slice_filter) == self.root_auth:
- #login = slice_filter.split(".")[1].split("_")[0]
-
- slicerec = self._sql_get_slice_info(slice_filter)
- if slicerec is None:
- return None
- #return login, None
- #Get slice based on user id
- if slice_filter_type == 'record_id_user':
-
- slicerec = self._sql_get_slice_info_from_user(slice_filter)
-
- if slicerec:
- fixed_slicerec_dict = slicerec
- #At this point if there is no login it means
- #record_id_user filter has been used for filtering
- #if login is None :
- ##If theslice record is from iotlab
- #if fixed_slicerec_dict['peer_authority'] is None:
- #login = fixed_slicerec_dict['hrn'].split(".")[1].split("_")[0]
- #return login, fixed_slicerec_dict
- return fixed_slicerec_dict
- else:
- return None
-
-
- def GetSlices(self, slice_filter=None, slice_filter_type=None,
- login=None):
- """Get the slice records from the sfa db and add lease information
- if any.
-
- :param slice_filter: can be the slice hrn or slice record id in the db
- depending on the slice_filter_type.
- :param slice_filter_type: defines the type of the filtering used, Can be
- either 'slice_hrn' or 'record_id'.
- :type slice_filter: string
- :type slice_filter_type: string
- :returns: a slice dict if slice_filter and slice_filter_type
- are specified and a matching entry is found in the db. The result
- is put into a list.Or a list of slice dictionnaries if no filters
- arespecified.
-
- :rtype: list
-
- """
- #login = None
- authorized_filter_types_list = ['slice_hrn', 'record_id_user']
- return_slicerec_dictlist = []
-
- #First try to get information on the slice based on the filter provided
- if slice_filter_type in authorized_filter_types_list:
- fixed_slicerec_dict = self._get_slice_records(slice_filter,
- slice_filter_type)
- # if the slice was not found in the sfa db
- if fixed_slicerec_dict is None:
- return return_slicerec_dictlist
-
- slice_hrn = fixed_slicerec_dict['hrn']
-
- logger.debug(" CORTEXLAB_API \tGetSlices login %s \
- slice record %s slice_filter %s \
- slice_filter_type %s " % (login,
- fixed_slicerec_dict, slice_filter,
- slice_filter_type))
-
-
- #Now we have the slice record fixed_slicerec_dict, get the
- #jobs associated to this slice
- leases_list = []
-
- leases_list = self.GetLeases(login=login)
- #If no job is running or no job scheduled
- #return only the slice record
- if leases_list == [] and fixed_slicerec_dict:
- return_slicerec_dictlist.append(fixed_slicerec_dict)
-
- # if the jobs running don't belong to the user/slice we are looking
- # for
- leases_hrn = [lease['slice_hrn'] for lease in leases_list]
- if slice_hrn not in leases_hrn:
- return_slicerec_dictlist.append(fixed_slicerec_dict)
- #If several experiments for one slice , put the slice record into
- # each lease information dict
- for lease in leases_list:
- slicerec_dict = {}
- logger.debug("CORTEXLAB_API.PY \tGetSlices slice_filter %s \
- \t lease['slice_hrn'] %s"
- % (slice_filter, lease['slice_hrn']))
- if lease['slice_hrn'] == slice_hrn:
- slicerec_dict['experiment_id'] = lease['lease_id']
- #Update lease dict with the slice record
- if fixed_slicerec_dict:
- fixed_slicerec_dict['experiment_id'] = []
- fixed_slicerec_dict['experiment_id'].append(
- slicerec_dict['experiment_id'])
- slicerec_dict.update(fixed_slicerec_dict)
-
- slicerec_dict['slice_hrn'] = lease['slice_hrn']
- slicerec_dict['hrn'] = lease['slice_hrn']
- slicerec_dict['user'] = lease['user']
- slicerec_dict.update(
- {'list_node_ids':
- {'hostname': lease['reserved_nodes']}})
- slicerec_dict.update({'node_ids': lease['reserved_nodes']})
-
-
- return_slicerec_dictlist.append(slicerec_dict)
-
-
- logger.debug("CORTEXLAB_API.PY \tGetSlices \
- slicerec_dict %s return_slicerec_dictlist %s \
- lease['reserved_nodes'] \
- %s" % (slicerec_dict, return_slicerec_dictlist,
- lease['reserved_nodes']))
-
- logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \
- return_slicerec_dictlist %s"
- % (return_slicerec_dictlist))
-
- return return_slicerec_dictlist
-
-
- else:
- #Get all slices from the cortexlab sfa database , get the user info
- # as well at the same time put them in dict format
-
- query_slice_list = \
- dbsession.query(RegSlice).options(joinedload('reg_researchers')).all()
-
- for record in query_slice_list:
- tmp = record.__dict__
- tmp['reg_researchers'] = tmp['reg_researchers'][0].__dict__
- return_slicerec_dictlist.append(tmp)
-
-
- #Get all the experiments reserved nodes
- leases_list = self.GetReservedNodes()
-
- for fixed_slicerec_dict in return_slicerec_dictlist:
- slicerec_dict = {}
- #Check if the slice belongs to a cortexlab user
- if fixed_slicerec_dict['peer_authority'] is None:
- owner = fixed_slicerec_dict['hrn'].split(
- ".")[1].split("_")[0]
- else:
- owner = None
- for lease in leases_list:
- if owner == lease['user']:
- slicerec_dict['experiment_id'] = lease['lease_id']
-
- #for reserved_node in lease['reserved_nodes']:
- logger.debug("CORTEXLAB_API.PY \tGetSlices lease %s "
- % (lease))
- slicerec_dict.update(fixed_slicerec_dict)
- slicerec_dict.update({'node_ids':
- lease['reserved_nodes']})
- slicerec_dict.update({'list_node_ids':
- {'hostname':
- lease['reserved_nodes']}})
-
-
- fixed_slicerec_dict.update(slicerec_dict)
-
- logger.debug("CORTEXLAB_API.PY \tGetSlices RETURN \
- return_slicerec_dictlist %s \slice_filter %s " \
- %(return_slicerec_dictlist, slice_filter))
-
- return return_slicerec_dictlist
git://git.onelab.eu / sfa.git / commitdiff