X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=sfa%2Fplanetlab%2Fplaggregate.py;h=32d6d933e7cecd36972137428d5377d9456bfdbc;hb=6a5b6854b1b43e1936cbde6493151a522be337da;hp=733638b092fd322a7e0a29b553e3896bc71fec40;hpb=02993db8cc4867b78b4618d287aa97006ac9d185;p=sfa.git diff --git a/sfa/planetlab/plaggregate.py b/sfa/planetlab/plaggregate.py index 733638b0..32d6d933 100644 --- a/sfa/planetlab/plaggregate.py +++ b/sfa/planetlab/plaggregate.py @@ -1,26 +1,28 @@ #!/usr/bin/python from collections import defaultdict -from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn +from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn, get_authority, get_leaf from sfa.util.sfatime import utcparse, datetime_to_string from sfa.util.sfalogging import logger from sfa.util.faults import SliverDoesNotExist from sfa.rspecs.rspec import RSpec from sfa.rspecs.elements.hardware_type import HardwareType -from sfa.rspecs.elements.node import Node +from sfa.rspecs.elements.node import NodeElement from sfa.rspecs.elements.link import Link from sfa.rspecs.elements.sliver import Sliver from sfa.rspecs.elements.login import Login from sfa.rspecs.elements.location import Location from sfa.rspecs.elements.interface import Interface -from sfa.rspecs.elements.services import Services +from sfa.rspecs.elements.services import ServicesElement from sfa.rspecs.elements.pltag import PLTag from sfa.rspecs.elements.lease import Lease from sfa.rspecs.elements.granularity import Granularity from sfa.rspecs.version_manager import VersionManager -from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn +from sfa.planetlab.plxrn import PlXrn, hostname_to_urn from sfa.planetlab.vlink import get_tc_rate from sfa.planetlab.topology import Topology +from sfa.storage.model import SliverAllocation + import time @@ -29,7 +31,8 @@ class PlAggregate: def __init__(self, driver): self.driver = driver - def get_nodes(self, options={}): + def get_nodes(self, options=None): + if options is None: options={} filter = {'peer_id': None} geni_available = options.get('geni_available') if geni_available == True: @@ -38,13 +41,15 @@ class PlAggregate: return nodes - def get_sites(self, filter={}): + def get_sites(self, filter=None): + if filter is None: filter={} sites = {} for site in self.driver.shell.GetSites(filter): sites[site['site_id']] = site return sites - def get_interfaces(self, filter={}): + def get_interfaces(self, filter=None): + if filter is None: filter={} interfaces = {} for interface in self.driver.shell.GetInterfaces(filter): iface = Interface() @@ -96,57 +101,122 @@ class PlAggregate: return links - def get_node_tags(self, filter={}): + def get_node_tags(self, filter=None): + if filter is None: filter={} node_tags = {} - for node_tag in self.driver.shell.GetNodeTags(filter): + for node_tag in self.driver.shell.GetNodeTags(filter, ['tagname', 'value', 'node_id', 'node_tag_id'] ): node_tags[node_tag['node_tag_id']] = node_tag return node_tags - def get_pl_initscripts(self, filter={}): + def get_pl_initscripts(self, filter=None): + if filter is None: filter={} pl_initscripts = {} filter.update({'enabled': True}) for initscript in self.driver.shell.GetInitScripts(filter): pl_initscripts[initscript['initscript_id']] = initscript return pl_initscripts - def get_slivers(self, urns, options): + def get_slivers(self, urns, options=None): + if options is None: options={} names = set() slice_ids = set() node_ids = [] + slice_hrn = None for urn in urns: xrn = PlXrn(xrn=urn) if xrn.type == 'sliver': # id: slice_id-node_id - id_parts = xrn.leaf.split('-') - slice_ids.add(id_parts[0]) - node_ids.append(id_parts[1]) + try: + sliver_id_parts = xrn.get_sliver_id_parts() + slice_id = int(sliver_id_parts[0]) + node_id = int(sliver_id_parts[1]) + slice_ids.add(slice_id) + node_ids.append(node_id) + except ValueError: + pass else: - names.add(xrn.pl_slicename()) - if xrn.id: - ids.add(xrn.id) + slice_hrn = xrn.get_hrn() filter = {} - if names: - filter['name'] = list(names) + filter['peer_id'] = None if slice_ids: - filter['slice_id'] = list(slice_ids) - slices = self.driver.shell.GetSlices(filter) - slice = slices[0] + filter['slice_id'] = list(slice_ids) + # get all slices + fields = ['slice_id', 'name', 'hrn', 'person_ids', 'node_ids', 'slice_tag_ids', 'expires'] + all_slices = self.driver.shell.GetSlices(filter, fields) + if slice_hrn: + slices = [slice for slice in all_slices if slice['hrn'] == slice_hrn] + else: + slices = all_slices + + if not slices: + if slice_hrn: + logger.error("PlAggregate.get_slivers : no slice found with hrn {}".format(slice_hrn)) + else: + logger.error("PlAggregate.get_slivers : no sliver found with urns {}".format(urns)) + return [] + slice = slices[0] + slice['hrn'] = slice_hrn + + # get sliver users + persons = [] + person_ids = [] + for slice in slices: + person_ids.extend(slice['person_ids']) + if person_ids: + persons = self.driver.shell.GetPersons(person_ids) + + # get user keys + keys = {} + key_ids = [] + for person in persons: + key_ids.extend(person['key_ids']) + + if key_ids: + key_list = self.driver.shell.GetKeys(key_ids) + for key in key_list: + keys[key['key_id']] = key + + # construct user key info + users = [] + for person in persons: + person_urn = hrn_to_urn(self.driver.shell.GetPersonHrn(int(person['person_id'])), 'user') + user = { + 'login': slice['name'], + 'user_urn': person_urn, + 'keys': [keys[k_id]['key'] for k_id in person['key_ids'] if k_id in keys] + } + users.append(user) + if node_ids: + node_ids = [node_id for node_id in node_ids if node_id in slice['node_ids']] slice['node_ids'] = node_ids - tags_dict = self.get_slice_tags(slice) + pltags_dict = self.get_pltags_by_node_id(slice) nodes_dict = self.get_slice_nodes(slice, options) slivers = [] for node in nodes_dict.values(): - node.update(slices[0]) - node['tags'] = tags_dict[node['node_id']] + node.update(slice) + # slice-global tags + node['slice-tags'] = pltags_dict['slice-global'] + # xxx + # this is where we chould maybe add the nodegroup slice tags, + # but it's tedious... + # xxx + # sliver tags + node['slice-tags'] += pltags_dict[node['node_id']] sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'], node['node_id']) - node['urn'] = PlXrn(xrn=sliver_hrn, type='sliver').get_urn() + node['sliver_id'] = Xrn(sliver_hrn, type='sliver').urn + node['urn'] = node['sliver_id'] + node['services_user'] = users slivers.append(node) + if not slivers: + logger.warning("PlAggregate.get_slivers : slice(s) found but with no sliver {}".format(urns)) return slivers - def node_to_rspec_node(self, node, sites, interfaces, node_tags, pl_initscripts=[], grain=None, options={}): - rspec_node = Node() + def node_to_rspec_node(self, node, sites, interfaces, node_tags, pl_initscripts=None, grain=None, options=None): + if pl_initscripts is None: pl_initscripts=[] + if options is None: options={} + rspec_node = NodeElement() # xxx how to retrieve site['login_base'] site=sites[node['site_id']] rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], node['hostname']) @@ -155,7 +225,17 @@ class PlAggregate: rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa') # do not include boot state ( element) in the manifest rspec rspec_node['boot_state'] = node['boot_state'] - rspec_node['exclusive'] = 'false' + if node['boot_state'] == 'boot': + rspec_node['available'] = 'true' + else: + rspec_node['available'] = 'false' + + #distinguish between Shared and Reservable nodes + if node['node_type'] == 'reservable': + rspec_node['exclusive'] = 'true' + else: + rspec_node['exclusive'] = 'false' + rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}), HardwareType({'name': 'pc'})] # only doing this because protogeni rspec needs @@ -181,44 +261,73 @@ class PlAggregate: interface['client_id'] = "%s:%s" % (node['node_id'], if_id) rspec_node['interfaces'].append(interface) if_count+=1 - tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']] - rspec_node['tags'] = tags + # this is what describes a particular node + node_level_tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids'] if tag_id in node_tags] + rspec_node['tags'] = node_level_tags return rspec_node - def sliver_to_rspec_node(self, sliver, sites, interfaces, node_tags, pl_initscripts): + def sliver_to_rspec_node(self, sliver, sites, interfaces, node_tags, sliver_pltags, \ + pl_initscripts, sliver_allocations): # get the granularity in second for the reservation system grain = self.driver.shell.GetLeaseGranularity() rspec_node = self.node_to_rspec_node(sliver, sites, interfaces, node_tags, pl_initscripts, grain) + for pltag in sliver_pltags: + logger.debug("Need to expose {}".format(pltag)) # xxx how to retrieve site['login_base'] rspec_node['expires'] = datetime_to_string(utcparse(sliver['expires'])) # remove interfaces from manifest rspec_node['interfaces'] = [] # add sliver info rspec_sliver = Sliver({'sliver_id': sliver['urn'], - 'name': sliver['name'], - 'type': 'plab-vserver', - 'tags': []}) + 'name': sliver['name'], + 'type': 'plab-vserver', + 'tags': sliver_pltags, + }) rspec_node['sliver_id'] = rspec_sliver['sliver_id'] - rspec_node['client_id'] = sliver['hostname'] + if sliver['urn'] in sliver_allocations: + rspec_node['client_id'] = sliver_allocations[sliver['urn']].client_id + if sliver_allocations[sliver['urn']].component_id: + rspec_node['component_id'] = sliver_allocations[sliver['urn']].component_id rspec_node['slivers'] = [rspec_sliver] # slivers always provide the ssh service - login = Login({'authentication': 'ssh-keys', 'hostname': sliver['hostname'], 'port':'22', 'username': sliver['name']}) - service = Services({'login': login}) - rspec_node['services'] = [service] - return rspec_node + login = Login({'authentication': 'ssh-keys', + 'hostname': sliver['hostname'], + 'port':'22', + 'username': sliver['name'], + 'login': sliver['name'] + }) + service = ServicesElement({'login': login, + 'services_user': sliver['services_user']}) + rspec_node['services'] = [service] + return rspec_node - def get_slice_tags(self, slice): + def get_pltags_by_node_id(self, slice): slice_tag_ids = [] slice_tag_ids.extend(slice['slice_tag_ids']) - tags = self.driver.shell.GetSliceTags({'slice_tag_id': slice_tag_ids}) + tags = self.driver.shell.GetSliceTags({'slice_tag_id': slice_tag_ids}, + ['tagname', 'value', 'node_id', 'nodegroup_id']) # sorted by node_id - tags_dict = defaultdict(list) + pltags_dict = defaultdict(list) for tag in tags: - tags_dict[tag['node_id']] = tag - return tags_dict - - def get_slice_nodes(self, slice, options={}): + # specific to a node + if tag['node_id']: + tag['scope'] = 'sliver' + pltags_dict[tag['node_id']].append(PLTag(tag)) + # restricted to a nodegroup + # for now such tags are not exposed to describe + # xxx we should also expose the nodegroup name in this case to be complete.. + elif tag['nodegroup_id']: + tag['scope'] = 'nodegroup' + pltags_dict['nodegroup'].append(PLTag(tag)) + # this tag is global to the slice + else: + tag['scope'] = 'slice' + pltags_dict['slice-global'].append(PLTag(tag)) + return pltags_dict + + def get_slice_nodes(self, slice, options=None): + if options is None: options={} nodes_dict = {} filter = {'peer_id': None} tags_filter = {} @@ -236,24 +345,38 @@ class PlAggregate: nodes_dict[node['node_id']] = node return nodes_dict - def rspec_node_to_geni_sliver(self, rspec_node): - op_status = "geni_unknown" - state = rspec_node['boot_state'].lower() - if state == 'boot': - op_status = 'geni_ready' + def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations=None): + if sliver_allocations is None: sliver_allocations={} + if rspec_node['sliver_id'] in sliver_allocations: + # set sliver allocation and operational status + sliver_allocation = sliver_allocations[rspec_node['sliver_id']] + if sliver_allocation: + allocation_status = sliver_allocation.allocation_state + if allocation_status == 'geni_allocated': + op_status = 'geni_pending_allocation' + elif allocation_status == 'geni_provisioned': + if rspec_node['boot_state'] == 'boot': + op_status = 'geni_ready' + else: + op_status = 'geni_failed' + else: + op_status = 'geni_unknown' + else: + allocation_status = 'geni_unallocated' else: - op_status =' geni_failed' - + allocation_status = 'geni_unallocated' + op_status = 'geni_failed' # required fields geni_sliver = {'geni_sliver_urn': rspec_node['sliver_id'], 'geni_expires': rspec_node['expires'], - 'geni_allocation_status': 'geni_provisioned', + 'geni_allocation_status' : allocation_status, 'geni_operational_status': op_status, - 'geni_error': None, + 'geni_error': '', } return geni_sliver - def get_leases(self, slice=None, options={}): + def get_leases(self, slice=None, options=None): + if options is None: options={} now = int(time.time()) filter={} @@ -280,9 +403,8 @@ class PlAggregate: site_id=lease['site_id'] site=sites_dict[site_id] - rspec_lease['lease_id'] = lease['lease_id'] - rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], lease['hostname']) - slice_hrn = slicename_to_hrn(self.driver.hrn, lease['name']) + rspec_lease['component_id'] = hrn_to_urn(self.driver.shell.GetNodeHrn(lease['hostname']), 'node') + slice_hrn = self.driver.shell.GetSliceHrn(lease['slice_id']) slice_urn = hrn_to_urn(slice_hrn, 'slice') rspec_lease['slice_id'] = slice_urn rspec_lease['start_time'] = lease['t_from'] @@ -291,7 +413,8 @@ class PlAggregate: return rspec_leases - def list_resources(self, version = None, options={}): + def list_resources(self, version = None, options=None): + if options is None: options={} version_manager = VersionManager() version = version_manager.get_version(version) @@ -324,20 +447,38 @@ class PlAggregate: # add links links = self.get_links(sites, nodes_dict, interfaces) rspec.version.add_links(links) + + if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'resources': + leases = self.get_leases() + rspec.version.add_leases(leases) + return rspec.toxml() - def describe(self, urns, version=None, options={}): + def describe(self, urns, version=None, options=None): + if options is None: options={} version_manager = VersionManager() version = version_manager.get_version(version) rspec_version = version_manager._get_version(version.type, version.version, 'manifest') - rspec = RSpec(version=version, user_options=options) + rspec = RSpec(version=rspec_version, user_options=options) # get slivers geni_slivers = [] - slivers = self.get_slivers(urns, options) - if len(slivers) == 0: - raise SliverDoesNotExist("You have not allocated any slivers here") - rspec.xml.set('expires', datetime_to_string(utcparse(slivers[0]['expires']))) + slivers = self.get_slivers(urns, options) + if slivers: + rspec_expires = datetime_to_string(utcparse(slivers[0]['expires'])) + else: + rspec_expires = datetime_to_string(utcparse(time.time())) + rspec.xml.set('expires', rspec_expires) + + # lookup the sliver allocations + geni_urn = urns[0] + sliver_ids = [sliver['sliver_id'] for sliver in slivers] + constraint = SliverAllocation.sliver_id.in_(sliver_ids) + sliver_allocations = self.driver.api.dbsession().query(SliverAllocation).filter(constraint) + sliver_allocation_dict = {} + for sliver_allocation in sliver_allocations: + geni_urn = sliver_allocation.slice_urn + sliver_allocation_dict[sliver_allocation.sliver_id] = sliver_allocation if not options.get('list_leases') or options['list_leases'] != 'leases': # add slivers @@ -358,9 +499,14 @@ class PlAggregate: for sliver in slivers: if sliver['slice_ids_whitelist'] and sliver['slice_id'] not in sliver['slice_ids_whitelist']: continue - rspec_node = self.sliver_to_rspec_node(sliver, sites, interfaces, node_tags, pl_initscripts) - geni_sliver = self.rspec_node_to_geni_sliver(rspec_node) + sliver_pltags = sliver['slice-tags'] + rspec_node = self.sliver_to_rspec_node(sliver, sites, interfaces, node_tags, sliver_pltags, + pl_initscripts, sliver_allocation_dict) + logger.debug('rspec of type {}'.format(rspec_node.__class__.__name__)) + # manifest node element shouldn't contain available attribute + rspec_node.pop('available') rspec_nodes.append(rspec_node) + geni_sliver = self.rspec_node_to_geni_sliver(rspec_node, sliver_allocation_dict) geni_slivers.append(geni_sliver) rspec.version.add_nodes(rspec_nodes) @@ -376,10 +522,11 @@ class PlAggregate: rspec.version.add_links(links) if not options.get('list_leases') or options['list_leases'] != 'resources': - leases = self.get_leases(slivers[0]) - rspec.version.add_leases(leases) + if slivers: + leases = self.get_leases(slivers[0]) + rspec.version.add_leases(leases) - return {'geni_urn': urns[0], + return {'geni_urn': geni_urn, 'geni_rspec': rspec.toxml(), 'geni_slivers': geni_slivers}