X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=sfa%2Fplanetlab%2Fplaggregate.py;h=c6006fbbc5f8c936ca1d88e9e1daae501aa821f9;hb=1cc8e9613cab8b5b22478de369f259e591c54e6d;hp=5da16936760361c0928150aebb1c8c9d68206653;hpb=c3c2bf529e765e93c07a01242dc08a5f855e021a;p=sfa.git diff --git a/sfa/planetlab/plaggregate.py b/sfa/planetlab/plaggregate.py index 5da16936..c6006fbb 100644 --- a/sfa/planetlab/plaggregate.py +++ b/sfa/planetlab/plaggregate.py @@ -3,16 +3,16 @@ from collections import defaultdict from sfa.util.xrn import Xrn, hrn_to_urn, urn_to_hrn from sfa.util.sfatime import utcparse, datetime_to_string from sfa.util.sfalogging import logger - +from sfa.util.faults import SliverDoesNotExist from sfa.rspecs.rspec import RSpec from sfa.rspecs.elements.hardware_type import HardwareType -from sfa.rspecs.elements.node import Node +from sfa.rspecs.elements.node import NodeElement from sfa.rspecs.elements.link import Link from sfa.rspecs.elements.sliver import Sliver from sfa.rspecs.elements.login import Login from sfa.rspecs.elements.location import Location from sfa.rspecs.elements.interface import Interface -from sfa.rspecs.elements.services import Services +from sfa.rspecs.elements.services import ServicesElement from sfa.rspecs.elements.pltag import PLTag from sfa.rspecs.elements.lease import Lease from sfa.rspecs.elements.granularity import Granularity @@ -21,6 +21,9 @@ from sfa.rspecs.version_manager import VersionManager from sfa.planetlab.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, slicename_to_hrn from sfa.planetlab.vlink import get_tc_rate from sfa.planetlab.topology import Topology +from sfa.storage.alchemy import dbsession +from sfa.storage.model import SliverAllocation + import time @@ -109,45 +112,104 @@ class PlAggregate: pl_initscripts[initscript['initscript_id']] = initscript return pl_initscripts - def get_slivers(self, urns, options): + def get_slivers(self, urns, options={}): names = set() - ids = set() + slice_ids = set() + node_ids = [] for urn in urns: xrn = PlXrn(xrn=urn) - names.add(xrn.get_slice_name()) - if xrn.id: - ids.add(xrn.id) - - slices = self.driver.shell.GetSlices(names) - # filter on id - if ids: - slices = [slice for slice in slices if slice['slice_id'] in ids] - - tags_dict = self.get_slice_tags(slices) - nodes_dict = self.get_slice_nodes(slices, options) + if xrn.type == 'sliver': + # id: slice_id-node_id + try: + sliver_id_parts = xrn.get_sliver_id_parts() + slice_id = int(sliver_id_parts[0]) + node_id = int(sliver_id_parts[1]) + slice_ids.add(slice_id) + node_ids.append(node_id) + except ValueError: + pass + else: + names.add(xrn.pl_slicename()) + + filter = {} + if names: + filter['name'] = list(names) + if slice_ids: + filter['slice_id'] = list(slice_ids) + # get slices + slices = self.driver.shell.GetSlices(filter) + if not slices: + return [] + slice = slices[0] + slice['hrn'] = PlXrn(auth=self.driver.hrn, slicename=slice['name']).hrn + + # get sliver users + persons = [] + person_ids = [] + for slice in slices: + person_ids.extend(slice['person_ids']) + if person_ids: + persons = self.driver.shell.GetPersons(person_ids) + + # get user keys + keys = {} + key_ids = [] + for person in persons: + key_ids.extend(person['key_ids']) + + if key_ids: + key_list = self.driver.shell.GetKeys(key_ids) + for key in key_list: + keys[key['key_id']] = key + + # construct user key info + users = [] + for person in persons: + name = person['email'][0:person['email'].index('@')] + user = { + 'login': slice['name'], + 'user_urn': Xrn('%s.%s' % (self.driver.hrn, name), type='user').urn, + 'keys': [keys[k_id]['key'] for k_id in person['key_ids'] if k_id in keys] + } + users.append(user) + + if node_ids: + node_ids = [node_id for node_id in node_ids if node_id in slice['node_ids']] + slice['node_ids'] = node_ids + tags_dict = self.get_slice_tags(slice) + nodes_dict = self.get_slice_nodes(slice, options) slivers = [] for node in nodes_dict.values(): - sliver = node.update(slices[0]) - sliver['tags'] = tags_dict[node['node_id']] + node.update(slice) + node['tags'] = tags_dict[node['node_id']] + sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'], node['node_id']) + node['sliver_id'] = Xrn(sliver_hrn, type='sliver').urn + node['urn'] = node['sliver_id'] + node['services_user'] = users + slivers.append(node) return slivers - def node_to_rspec_node(self, sites, interfaces, tags, pl_initscripts=[], grain=None, options={}): - rspec_node = Node() + def node_to_rspec_node(self, node, sites, interfaces, node_tags, pl_initscripts=[], grain=None, options={}): + rspec_node = NodeElement() # xxx how to retrieve site['login_base'] - site=sites_dict[node['site_id']] - rspec_node['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], node['hostname']) + site=sites[node['site_id']] + rspec_node['component_id'] = PlXrn(self.driver.hrn, hostname=node['hostname']).get_urn() rspec_node['component_name'] = node['hostname'] rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn() rspec_node['authority_id'] = hrn_to_urn(PlXrn.site_hrn(self.driver.hrn, site['login_base']), 'authority+sa') # do not include boot state ( element) in the manifest rspec rspec_node['boot_state'] = node['boot_state'] + if node['boot_state'] == 'boot': + rspec_node['available'] = 'true' + else: + rspec_node['available'] = 'false' rspec_node['exclusive'] = 'false' rspec_node['hardware_types'] = [HardwareType({'name': 'plab-pc'}), HardwareType({'name': 'pc'})] # only doing this because protogeni rspec needs # to advertise available initscripts rspec_node['pl_initscripts'] = pl_initscripts.values() - # add site/interface info to nodes. + # add site/interface info to nodes. # assumes that sites, interfaces and tags have already been prepared. if site['longitude'] and site['latitude']: location = Location({'longitude': site['longitude'], 'latitude': site['latitude'], 'country': 'unknown'}) @@ -167,82 +229,101 @@ class PlAggregate: interface['client_id'] = "%s:%s" % (node['node_id'], if_id) rspec_node['interfaces'].append(interface) if_count+=1 - tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']] + tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids'] if tag_id in node_tags] rspec_node['tags'] = tags return rspec_node - def sliver_to_rspec_node(self, sliver, sites, interfaces, node_tags, pl_initscripts): + def sliver_to_rspec_node(self, sliver, sites, interfaces, node_tags, \ + pl_initscripts, sliver_allocations): # get the granularity in second for the reservation system grain = self.driver.shell.GetLeaseGranularity() - rspec_node = self.get_rspec_node(node, sites, interfaces, node_tags, pl_initscripts, grain) + rspec_node = self.node_to_rspec_node(sliver, sites, interfaces, node_tags, pl_initscripts, grain) # xxx how to retrieve site['login_base'] - rspec_node['expires'] = datetime_to_string(utcparse(slice[0]['expires'])) + rspec_node['expires'] = datetime_to_string(utcparse(sliver['expires'])) # remove interfaces from manifest rspec_node['interfaces'] = [] # add sliver info - id = ":".join(map(str, [slices[0]['slice_id'], node['node_id']])) - sliver_xrn = Xrn(slice_urn, id=id).get_urn() - sliver_xrn.set_authority(self.driver.hrn) - sliver = Sliver({'sliver_id': sliver_xrn.get_urn(), - 'name': slice[0]['name'], + rspec_sliver = Sliver({'sliver_id': sliver['urn'], + 'name': sliver['name'], 'type': 'plab-vserver', 'tags': []}) - rspec_node['sliver_id'] = sliver['sliver_id'] - rspec_node['client_id'] = node['hostname'] - rspec_node['slivers'] = [sliver] + rspec_node['sliver_id'] = rspec_sliver['sliver_id'] + rspec_node['client_id'] = sliver_allocations[sliver['urn']].client_id + if sliver_allocations[sliver['urn']].component_id: + rspec_node['component_id'] = sliver_allocations[sliver['urn']].component_id + rspec_node['slivers'] = [rspec_sliver] # slivers always provide the ssh service - login = Login({'authentication': 'ssh-keys', 'hostname': node['hostname'], 'port':'22', 'username': sliver['name']}) - service = Services({'login': login}) + login = Login({'authentication': 'ssh-keys', + 'hostname': sliver['hostname'], + 'port':'22', + 'username': sliver['name'], + 'login': sliver['name'] + }) + service = ServicesElement({'login': login, + 'services_user': sliver['services_user']}) rspec_node['services'] = [service] return rspec_node - def get_slice_tags(self, slices): + def get_slice_tags(self, slice): slice_tag_ids = [] - for slice in slices: - slice_tag_ids.extend(slice['slice_tag_ids']) + slice_tag_ids.extend(slice['slice_tag_ids']) tags = self.driver.shell.GetSliceTags({'slice_tag_id': slice_tag_ids}) # sorted by node_id - tags_dict = defaultdict([]) + tags_dict = defaultdict(list) for tag in tags: tags_dict[tag['node_id']] = tag return tags_dict - def get_slice_nodes(self, slices, options={}): + def get_slice_nodes(self, slice, options={}): + nodes_dict = {} filter = {'peer_id': None} tags_filter = {} - if slice and 'node_ids' in slice and slice['node_ids']: + if slice and slice.get('node_ids'): filter['node_id'] = slice['node_ids'] - tags_filter=filter.copy() - + else: + # there are no nodes to look up + return nodes_dict + tags_filter=filter.copy() geni_available = options.get('geni_available') if geni_available == True: filter['boot_state'] = 'boot' nodes = self.driver.shell.GetNodes(filter) - nodes_dict = {} for node in nodes: nodes_dict[node['node_id']] = node return nodes_dict - def rspec_node_to_geni_sliver(self, rspec_node): - op_status = "geni_unknown" - state = sliver['boot_stat'].lower() - if state == 'boot': - op_status = 'geni_ready' - else: - op_status =' geni_failed' - + def rspec_node_to_geni_sliver(self, rspec_node, sliver_allocations = {}): + if rspec_node['sliver_id'] in sliver_allocations: + # set sliver allocation and operational status + sliver_allocation = sliver_allocations[rspec_node['sliver_id']] + if sliver_allocation: + allocation_status = sliver_allocation.allocation_state + if allocation_status == 'geni_allocated': + op_status = 'geni_pending_allocation' + elif allocation_status == 'geni_provisioned': + if rspec_node['boot_state'] == 'boot': + op_status = 'geni_ready' + else: + op_status = 'geni_failed' + else: + op_status = 'geni_unknown' + else: + allocation_status = 'geni_unallocated' # required fields geni_sliver = {'geni_sliver_urn': rspec_node['sliver_id'], 'geni_expires': rspec_node['expires'], - 'geni_allocation_status': 'geni_provisioned', + 'geni_allocation_status' : allocation_status, 'geni_operational_status': op_status, - 'geni_error': None, + 'geni_error': '', } return geni_sliver - def get_leases(self, slice=None, options={}): + def get_leases(self, slice_xrn=None, slice=None, options={}): + if slice_xrn and not slice: + return [] + now = int(time.time()) filter={} filter.update({'clip':now}) @@ -268,10 +349,13 @@ class PlAggregate: site_id=lease['site_id'] site=sites_dict[site_id] - rspec_lease['lease_id'] = lease['lease_id'] rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn, site['login_base'], lease['hostname']) - slice_hrn = slicename_to_hrn(self.driver.hrn, lease['name']) - slice_urn = hrn_to_urn(slice_hrn, 'slice') + if slice_xrn: + slice_urn = slice_xrn + slice_hrn = urn_to_hrn(slice_urn) + else: + slice_hrn = slicename_to_hrn(self.driver.hrn, lease['name']) + slice_urn = hrn_to_urn(slice_hrn, 'slice') rspec_lease['slice_id'] = slice_urn rspec_lease['start_time'] = lease['t_from'] rspec_lease['duration'] = (lease['t_until'] - lease['t_from']) / grain @@ -293,14 +377,14 @@ class PlAggregate: interface_ids = [] tag_ids = [] nodes_dict = {} - for sliver in slivers: - site_ids.append(sliver['site_id']) - interface_ids.extend(sliver['interface_ids']) - tag_ids.extend(sliver['node_tag_ids']) - nodes_dict[sliver['node_id']] = sliver + for node in nodes: + site_ids.append(node['site_id']) + interface_ids.extend(node['interface_ids']) + tag_ids.extend(node['node_tag_ids']) + nodes_dict[node['node_id']] = node sites = self.get_sites({'site_id': site_ids}) interfaces = self.get_interfaces({'interface_id':interface_ids}) - node_tags = self.get_node_tags(tags_filter) + node_tags = self.get_node_tags({'node_tag_id': tag_ids}) pl_initscripts = self.get_pl_initscripts() # convert nodes to rspec nodes rspec_nodes = [] @@ -310,26 +394,39 @@ class PlAggregate: rspec.version.add_nodes(rspec_nodes) # add links - links = self.get_links(sites_dict, nodes_dict, interfaces) + links = self.get_links(sites, nodes_dict, interfaces) rspec.version.add_links(links) + + if not options.get('list_leases') or options.get('list_leases') and options['list_leases'] != 'resources': + leases = self.get_leases(slice_xrn, slice) + rspec.version.add_leases(leases) + return rspec.toxml() def describe(self, urns, version=None, options={}): - # update nova connection - tenant_name = OSXrn(xrn=urns[0], type='slice').get_tenant_name() - self.driver.shell.nova_manager.connect(tenant=tenant_name) - version_manager = VersionManager() version = version_manager.get_version(version) rspec_version = version_manager._get_version(version.type, version.version, 'manifest') - rspec = RSpec(version=version, user_options=options) + rspec = RSpec(version=rspec_version, user_options=options) # get slivers geni_slivers = [] - slivers = self.get_slivers(urns, options) - if len(slivers) == 0: - raise SliverDoesNotExist("You have not allocated any slivers here") - rspec.xml.set('expires', datetime_to_string(utcparse(slivers[0]['expires']))) + slivers = self.get_slivers(urns, options) + if slivers: + rspec_expires = datetime_to_string(utcparse(slivers[0]['expires'])) + else: + rspec_expires = datetime_to_string(utcparse(time.time())) + rspec.xml.set('expires', rspec_expires) + + # lookup the sliver allocations + geni_urn = urns[0] + sliver_ids = [sliver['sliver_id'] for sliver in slivers] + constraint = SliverAllocation.sliver_id.in_(sliver_ids) + sliver_allocations = dbsession.query(SliverAllocation).filter(constraint) + sliver_allocation_dict = {} + for sliver_allocation in sliver_allocations: + geni_urn = sliver_allocation.slice_urn + sliver_allocation_dict[sliver_allocation.sliver_id] = sliver_allocation if not options.get('list_leases') or options['list_leases'] != 'leases': # add slivers @@ -344,35 +441,38 @@ class PlAggregate: nodes_dict[sliver['node_id']] = sliver sites = self.get_sites({'site_id': site_ids}) interfaces = self.get_interfaces({'interface_id':interface_ids}) - node_tags = self.get_node_tags(tags_filter) + node_tags = self.get_node_tags({'node_tag_id': tag_ids}) pl_initscripts = self.get_pl_initscripts() rspec_nodes = [] for sliver in slivers: if sliver['slice_ids_whitelist'] and sliver['slice_id'] not in sliver['slice_ids_whitelist']: continue - rspec_node = self.sliver_to_rspec_node(sliver, sites, interfaces, node_tags, pl_initscripts) - geni_sliver = self.rspec_node_to_geni_sliver(rspec_node) + rspec_node = self.sliver_to_rspec_node(sliver, sites, interfaces, node_tags, + pl_initscripts, sliver_allocation_dict) + # manifest node element shouldn't contain available attribute + rspec_node.pop('available') rspec_nodes.append(rspec_node) + geni_sliver = self.rspec_node_to_geni_sliver(rspec_node, sliver_allocation_dict) geni_slivers.append(geni_sliver) rspec.version.add_nodes(rspec_nodes) # add sliver defaults - default_sliver = slivers.get(None, []) - if default_sliver: - default_sliver_attribs = default_sliver.get('tags', []) - for attrib in default_sliver_attribs: - rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value']) + #default_sliver = slivers.get(None, []) + #if default_sliver: + # default_sliver_attribs = default_sliver.get('tags', []) + # for attrib in default_sliver_attribs: + # rspec.version.add_default_sliver_attribute(attrib['tagname'], attrib['value']) # add links - links = self.get_links(sites_dict, nodes_dict, interfaces) + links = self.get_links(sites, nodes_dict, interfaces) rspec.version.add_links(links) if not options.get('list_leases') or options['list_leases'] != 'resources': - leases = self.get_leases(slivers[0]) - rspec.version.add_leases(leases) + if slivers: + leases = self.get_leases(slivers[0]) + rspec.version.add_leases(leases) - return {'geni_urn': urns[0], + return {'geni_urn': geni_urn, 'geni_rspec': rspec.toxml(), 'geni_slivers': geni_slivers} -