From: Thierry Parmentelat Date: Thu, 10 Nov 2011 14:40:22 +0000 (+0100) Subject: Merge branch 'upstreammaster' X-Git-Tag: sfa-2.1-24~35 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=7691d2227f76cac186cb98167ac425bd9b5c0d5e;hp=68f0f5bf538b2633fceee633be2748aaa3eaa709;p=sfa.git Merge branch 'upstreammaster' --- diff --git a/sfa/managers/aggregate_manager.py b/sfa/managers/aggregate_manager.py index fb49cb68..14ec7d0f 100644 --- a/sfa/managers/aggregate_manager.py +++ b/sfa/managers/aggregate_manager.py @@ -187,11 +187,10 @@ class AggregateManager: slices.verify_slice_attributes(slice, requested_attributes) # add/remove slice from nodes - requested_slivers = [str(host) for host in rspec.version.get_nodes_with_slivers()] + requested_slivers = [node['component_name'] for node in rspec.version.get_nodes_with_slivers()] slices.verify_slice_nodes(slice, requested_slivers, peer) - - aggregate.prepare_nodes({'hostname': requested_slivers}) - aggregate.prepare_interfaces({'node_id': aggregate.nodes.keys()}) + + # add/remove links links slices.verify_slice_links(slice, rspec.version.get_link_requests(), aggregate) # handle MyPLC peer association. diff --git a/sfa/managers/aggregate_manager_eucalyptus.py b/sfa/managers/aggregate_manager_eucalyptus.py index 4d73ab17..3f2d512c 100644 --- a/sfa/managers/aggregate_manager_eucalyptus.py +++ b/sfa/managers/aggregate_manager_eucalyptus.py @@ -15,20 +15,19 @@ from xmlbuilder import XMLBuilder from lxml import etree as ET from sqlobject import * -from sfa.util.faults import InvalidRSpec, +from sfa.util.faults import InvalidRSpec from sfa.util.xrn import urn_to_hrn, Xrn from sfa.util.plxrn import hrn_to_pl_slicename, slicename_to_hrn from sfa.util.callids import Callids #comes with its own logging #from sfa.util.sfalogging import logger from sfa.util.version import version_core - from sfa.trust.credential import Credential - from sfa.server.sfaapi import SfaApi - from sfa.plc.aggregate import Aggregate from sfa.plc.slices import Slice, Slices +from sfa.rspecs.version_manager import VersionManager +from sfa.rspecs.rspec import RSpec # not sure what this used to be nor where it is now defined #from sfa.rspecs.sfa_rspec import sfa_rspec_version @@ -679,15 +678,22 @@ class AggregateManagerEucalyptus: f.close() def GetVersion(api): + + version_manager = VersionManager() + ad_rspec_versions = [] + request_rspec_versions = [] + for rspec_version in version_manager.versions: + if rspec_version.content_type in ['*', 'ad']: + ad_rspec_versions.append(rspec_version.to_dict()) + if rspec_version.content_type in ['*', 'request']: + request_rspec_versions.append(rspec_version.to_dict()) + default_rspec_version = version_manager.get_version("sfa 1").to_dict() xrn=Xrn(api.hrn) - request_rspec_versions = [dict(sfa_rspec_version)] - ad_rspec_versions = [dict(sfa_rspec_version)] version_more = {'interface':'aggregate', 'testbed':'myplc', 'hrn':xrn.get_hrn(), 'request_rspec_versions': request_rspec_versions, 'ad_rspec_versions': ad_rspec_versions, - 'default_ad_rspec': dict(sfa_rspec_version) + 'default_ad_rspec': default_rspec_version } return version_core(version_more) - diff --git a/sfa/managers/aggregate_manager_max.py b/sfa/managers/aggregate_manager_max.py index ac3a8faf..09e56b77 100644 --- a/sfa/managers/aggregate_manager_max.py +++ b/sfa/managers/aggregate_manager_max.py @@ -214,20 +214,6 @@ class AggregateManagerMax (AggregateManager): ### GENI AM API Methods - def GetVersion(self, api): - xrn=Xrn(api.hrn) - request_rspec_versions = [dict(sfa_rspec_version)] - ad_rspec_versions = [dict(sfa_rspec_version)] - #TODO: MAX-AM specific - version_more = {'interface':'aggregate', - 'testbed':'myplc', - 'hrn':xrn.get_hrn(), - 'request_rspec_versions': request_rspec_versions, - 'ad_rspec_versions': ad_rspec_versions, - 'default_ad_rspec': dict(sfa_rspec_version) - } - return version_core(version_more) - def SliverStatus(self, api, slice_xrn, creds, call_id): if Callids().already_handled(call_id): return {} return self.slice_status(api, slice_xrn, creds) diff --git a/sfa/managers/slice_manager.py b/sfa/managers/slice_manager.py index 72a9c698..3d6c0a67 100644 --- a/sfa/managers/slice_manager.py +++ b/sfa/managers/slice_manager.py @@ -1,5 +1,6 @@ import sys import time +import traceback from StringIO import StringIO from copy import copy from lxml import etree @@ -89,15 +90,27 @@ class SliceManager: except Exception, e: logger.warn("drop_slicemgr_stats failed: %s " % (str(e))) - def add_slicemgr_stat(self, rspec, callname, aggname, elapsed, status): + def add_slicemgr_stat(self, rspec, callname, aggname, elapsed, status, exc_info=None): try: stats_tags = rspec.xml.xpath('//statistics[@call="%s"]' % callname) if stats_tags: stats_tag = stats_tags[0] else: stats_tag = etree.SubElement(rspec.xml.root, "statistics", call=callname) - - etree.SubElement(stats_tag, "aggregate", name=str(aggname), elapsed=str(elapsed), status=str(status)) + + stat_tag = etree.SubElement(stats_tag, "aggregate", name=str(aggname), elapsed=str(elapsed), status=str(status)) + + if exc_info: + exc_tag = etree.SubElement(stat_tag, "exc_info", name=str(exc_info[1])) + + # formats the traceback as one big text blob + #exc_tag.text = "\n".join(traceback.format_exception(exc_info[0], exc_info[1], exc_info[2])) + + # formats the traceback as a set of xml elements + tb = traceback.extract_tb(exc_info[2]) + for item in tb: + exc_frame = etree.SubElement(exc_tag, "tb_frame", filename=str(item[0]), line=str(item[1]), func=str(item[2]), code=str(item[3])) + except Exception, e: logger.warn("add_slicemgr_stat failed on %s: %s" %(aggname, str(e))) @@ -119,7 +132,7 @@ class SliceManager: return {"aggregate": aggregate, "rspec": rspec, "elapsed": time.time()-tStart, "status": "success"} except Exception, e: api.logger.log_exc("ListResources failed at %s" %(server.url)) - return {"aggregate": aggregate, "elapsed": time.time()-tStart, "status": "exception"} + return {"aggregate": aggregate, "elapsed": time.time()-tStart, "status": "exception", "exc_info": sys.exc_info()} if Callids().already_handled(call_id): return "" @@ -168,7 +181,7 @@ class SliceManager: result_version = version_manager._get_version(rspec_version.type, rspec_version.version, 'ad') rspec = RSpec(version=result_version) for result in results: - self.add_slicemgr_stat(rspec, "ListResources", result["aggregate"], result["elapsed"], result["status"]) + self.add_slicemgr_stat(rspec, "ListResources", result["aggregate"], result["elapsed"], result["status"], result.get("exc_info",None)) if result["status"]=="success": try: rspec.version.merge(result["rspec"]) @@ -205,9 +218,9 @@ class SliceManager: args.append(call_id) rspec = server.CreateSliver(*args) return {"aggregate": aggregate, "rspec": rspec, "elapsed": time.time()-tStart, "status": "success"} - except: + except: logger.log_exc('Something wrong in _CreateSliver with URL %s'%server.url) - return {"aggregate": aggregate, "elapsed": time.time()-tStart, "status": "exception"} + return {"aggregate": aggregate, "elapsed": time.time()-tStart, "status": "exception", "exc_info": sys.exc_info()} if Callids().already_handled(call_id): return "" # Validate the RSpec against PlanetLab's schema --disabled for now @@ -246,7 +259,7 @@ class SliceManager: manifest_version = version_manager._get_version(rspec.version.type, rspec.version.version, 'manifest') result_rspec = RSpec(version=manifest_version) for result in results: - self.add_slicemgr_stat(result_rspec, "CreateSliver", result["aggregate"], result["elapsed"], result["status"]) + self.add_slicemgr_stat(result_rspec, "CreateSliver", result["aggregate"], result["elapsed"], result["status"], result.get("exc_info",None)) if result["status"]=="success": try: result_rspec.version.merge(result["rspec"]) diff --git a/sfa/managers/vini/__init__.py b/sfa/managers/vini/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/sfa/managers/vini/request.xml b/sfa/managers/vini/request.xml deleted file mode 100644 index 111f7b0e..00000000 --- a/sfa/managers/vini/request.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/sfa/managers/vini/topology.py b/sfa/managers/vini/topology.py deleted file mode 100644 index 7ec3e95c..00000000 --- a/sfa/managers/vini/topology.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/python -# -# Links in the physical topology, gleaned from looking at the Internet2 -# topology map. Link (a, b) connects sites with IDs a and b. -# -PhysicalLinks = [(2, 12), # I2 Princeton - New York - (11, 13), # I2 Chicago - Wash DC - (11, 15), # I2 Chicago - Atlanta - (11, 16), # I2 Chicago - CESNET - (11, 17), # I2 Chicago - Kansas City - (12, 13), # I2 New York - Wash DC - (13, 15), # I2 Wash DC - Atlanta - (14, 15), # Ga Tech - I2 Atlanta - (15, 19), # I2 Atlanta - Houston - (17, 19), # I2 Kansas City - Houston - (17, 22), # I2 Kansas City - Salt Lake City - (17, 24), # I2 Kansas City - UMKC - (19, 20), # I2 Houston - Los Angeles - (20, 21), # I2 Los Angeles - Seattle - (20, 22), # I2 Los Angeles - Salt Lake City - (21, 22)] # I2 Seattle - Salt Lake City - - diff --git a/sfa/managers/vini/utils.py b/sfa/managers/vini/utils.py deleted file mode 100644 index a718c035..00000000 --- a/sfa/managers/vini/utils.py +++ /dev/null @@ -1,718 +0,0 @@ -from __future__ import with_statement -import re -import socket -from sfa.util.faults import * -from sfa.managers.vini.topology import PhysicalLinks -from xmlbuilder import XMLBuilder -from lxml import etree -import sys -from StringIO import StringIO - -VINI_RELAXNG_SCHEMA = "/var/www/html/schemas/vini.rng" - -# Taken from bwlimit.py -# -# See tc_util.c and http://physics.nist.gov/cuu/Units/binary.html. Be -# warned that older versions of tc interpret "kbps", "mbps", "mbit", -# and "kbit" to mean (in this system) "kibps", "mibps", "mibit", and -# "kibit" and that if an older version is installed, all rates will -# be off by a small fraction. -suffixes = { - "": 1, - "bit": 1, - "kibit": 1024, - "kbit": 1000, - "mibit": 1024*1024, - "mbit": 1000000, - "gibit": 1024*1024*1024, - "gbit": 1000000000, - "tibit": 1024*1024*1024*1024, - "tbit": 1000000000000, - "bps": 8, - "kibps": 8*1024, - "kbps": 8000, - "mibps": 8*1024*1024, - "mbps": 8000000, - "gibps": 8*1024*1024*1024, - "gbps": 8000000000, - "tibps": 8*1024*1024*1024*1024, - "tbps": 8000000000000 -} - - -def get_tc_rate(s): - """ - Parses an integer or a tc rate string (e.g., 1.5mbit) into bits/second - """ - - if type(s) == int: - return s - m = re.match(r"([0-9.]+)(\D*)", s) - if m is None: - return -1 - suffix = m.group(2).lower() - if suffixes.has_key(suffix): - return int(float(m.group(1)) * suffixes[suffix]) - else: - return -1 - -def format_tc_rate(rate): - """ - Formats a bits/second rate into a tc rate string - """ - - if rate >= 1000000000 and (rate % 1000000000) == 0: - return "%.0fgbit" % (rate / 1000000000.) - elif rate >= 1000000 and (rate % 1000000) == 0: - return "%.0fmbit" % (rate / 1000000.) - elif rate >= 1000: - return "%.0fkbit" % (rate / 1000.) - else: - return "%.0fbit" % rate - - -class Node: - def __init__(self, node, bps = 1000 * 1000000): - self.id = node['node_id'] - self.idtag = "n%s" % self.id - self.hostname = node['hostname'] - self.name = self.shortname = self.hostname.replace('.vini-veritas.net', '') - self.site_id = node['site_id'] - self.ipaddr = socket.gethostbyname(self.hostname) - self.bps = bps - self.links = set() - self.sliver = False - - def get_link_id(self, remote): - if self.id < remote.id: - link = (self.id<<7) + remote.id - else: - link = (remote.id<<7) + self.id - return link - - def get_iface_id(self, remote): - if self.id < remote.id: - iface = 1 - else: - iface = 2 - return iface - - def get_virt_ip(self, remote): - link = self.get_link_id(remote) - iface = self.get_iface_id(remote) - first = link >> 6 - second = ((link & 0x3f)<<2) + iface - return "192.168.%d.%d" % (first, second) - - def get_virt_net(self, remote): - link = self.get_link_id(remote) - first = link >> 6 - second = (link & 0x3f)<<2 - return "192.168.%d.%d/30" % (first, second) - - def get_site(self, sites): - return sites[self.site_id] - - def get_topo_rspec(self, link): - if link.end1 == self: - remote = link.end2 - elif link.end2 == self: - remote = link.end1 - else: - raise Error("Link does not connect to Node") - - my_ip = self.get_virt_ip(remote) - remote_ip = remote.get_virt_ip(self) - net = self.get_virt_net(remote) - bw = format_tc_rate(link.bps) - return (remote.id, remote.ipaddr, bw, my_ip, remote_ip, net) - - def add_link(self, link): - self.links.add(link) - - def add_tag(self, sites): - s = self.get_site(sites) - words = self.hostname.split(".") - index = words[0].replace("node", "") - if index.isdigit(): - self.tag = s.tag + index - else: - self.tag = None - - # Assumes there is at most one Link between two sites - def get_sitelink(self, node, sites): - site1 = sites[self.site_id] - site2 = sites[node.site_id] - sl = site1.links.intersection(site2.links) - if len(sl): - return sl.pop() - return None - - def add_sliver(self): - self.sliver = True - - def toxml(self, xml, hrn): - if not self.tag: - return - with xml.node(id = self.idtag): - with xml.hostname: - xml << self.hostname - with xml.kbps: - xml << str(int(self.bps/1000)) - if self.sliver: - with xml.sliver: - pass - - -class Link: - def __init__(self, end1, end2, bps = 1000 * 1000000, parent = None): - self.end1 = end1 - self.end2 = end2 - self.bps = bps - self.parent = parent - self.children = [] - - end1.add_link(self) - end2.add_link(self) - - if self.parent: - self.parent.children.append(self) - - def toxml(self, xml): - end_ids = "%s %s" % (self.end1.idtag, self.end2.idtag) - - if self.parent: - element = xml.vlink(endpoints=end_ids) - else: - element = xml.link(endpoints=end_ids) - - with element: - with xml.description: - xml << "%s -- %s" % (self.end1.name, self.end2.name) - with xml.kbps: - xml << str(int(self.bps/1000)) - for child in self.children: - child.toxml(xml) - - -class Site: - def __init__(self, site): - self.id = site['site_id'] - self.idtag = "s%s" % self.id - self.node_ids = site['node_ids'] - self.name = site['abbreviated_name'].replace(" ", "_") - self.tag = site['login_base'] - self.public = site['is_public'] - self.enabled = site['enabled'] - self.links = set() - - def get_sitenodes(self, nodes): - n = [] - for i in self.node_ids: - n.append(nodes[i]) - return n - - def add_link(self, link): - self.links.add(link) - - def toxml(self, xml, hrn, nodes): - if not (self.public and self.enabled and self.node_ids): - return - with xml.site(id = self.idtag): - with xml.name: - xml << self.name - - for node in self.get_sitenodes(nodes): - node.toxml(xml, hrn) - - -class Slice: - def __init__(self, slice): - self.id = slice['slice_id'] - self.name = slice['name'] - self.node_ids = set(slice['node_ids']) - self.slice_tag_ids = slice['slice_tag_ids'] - - def get_tag(self, tagname, slicetags, node = None): - for i in self.slice_tag_ids: - tag = slicetags[i] - if tag.tagname == tagname: - if (not node) or (node.id == tag.node_id): - return tag - else: - return None - - def get_nodes(self, nodes): - n = [] - for id in self.node_ids: - n.append(nodes[id]) - return n - - - # Add a new slice tag - def add_tag(self, tagname, value, slicetags, node = None): - record = {'slice_tag_id':None, 'slice_id':self.id, 'tagname':tagname, 'value':value} - if node: - record['node_id'] = node.id - else: - record['node_id'] = None - tag = Slicetag(record) - slicetags[tag.id] = tag - self.slice_tag_ids.append(tag.id) - tag.changed = True - tag.updated = True - return tag - - # Update a slice tag if it exists, else add it - def update_tag(self, tagname, value, slicetags, node = None): - tag = self.get_tag(tagname, slicetags, node) - if tag and tag.value == value: - value = "no change" - elif tag: - tag.value = value - tag.changed = True - else: - tag = self.add_tag(tagname, value, slicetags, node) - tag.updated = True - - def assign_egre_key(self, slicetags): - if not self.get_tag('egre_key', slicetags): - try: - key = free_egre_key(slicetags) - self.update_tag('egre_key', key, slicetags) - except: - # Should handle this case... - pass - return - - def turn_on_netns(self, slicetags): - tag = self.get_tag('netns', slicetags) - if (not tag) or (tag.value != '1'): - self.update_tag('netns', '1', slicetags) - return - - def turn_off_netns(self, slicetags): - tag = self.get_tag('netns', slicetags) - if tag and (tag.value != '0'): - tag.delete() - return - - def add_cap_net_admin(self, slicetags): - tag = self.get_tag('capabilities', slicetags) - if tag: - caps = tag.value.split(',') - for cap in caps: - if cap == "CAP_NET_ADMIN": - return - else: - newcaps = "CAP_NET_ADMIN," + tag.value - self.update_tag('capabilities', newcaps, slicetags) - else: - self.add_tag('capabilities', 'CAP_NET_ADMIN', slicetags) - return - - def remove_cap_net_admin(self, slicetags): - tag = self.get_tag('capabilities', slicetags) - if tag: - caps = tag.value.split(',') - newcaps = [] - for cap in caps: - if cap != "CAP_NET_ADMIN": - newcaps.append(cap) - if newcaps: - value = ','.join(newcaps) - self.update_tag('capabilities', value, slicetags) - else: - tag.delete() - return - - # Update the vsys/setup-link and vsys/setup-nat slice tags. - def add_vsys_tags(self, slicetags): - link = nat = False - for i in self.slice_tag_ids: - tag = slicetags[i] - if tag.tagname == 'vsys': - if tag.value == 'setup-link': - link = True - elif tag.value == 'setup-nat': - nat = True - if not link: - self.add_tag('vsys', 'setup-link', slicetags) - if not nat: - self.add_tag('vsys', 'setup-nat', slicetags) - return - - -class Slicetag: - newid = -1 - def __init__(self, tag): - self.id = tag['slice_tag_id'] - if not self.id: - # Make one up for the time being... - self.id = Slicetag.newid - Slicetag.newid -= 1 - self.slice_id = tag['slice_id'] - self.tagname = tag['tagname'] - self.value = tag['value'] - self.node_id = tag['node_id'] - self.updated = False - self.changed = False - self.deleted = False - - # Mark a tag as deleted - def delete(self): - self.deleted = True - self.updated = True - - def write(self, api): - if self.changed: - if int(self.id) > 0: - api.driver.UpdateSliceTag(self.id, self.value) - else: - api.driver.AddSliceTag(self.slice_id, self.tagname, self.value, self.node_id) - elif self.deleted and int(self.id) > 0: - api.driver.DeleteSliceTag(self.id) - - -""" -A topology is a compound object consisting of: -* a dictionary mapping site IDs to Site objects -* a dictionary mapping node IDs to Node objects -* the Site objects are connected via SiteLink objects representing - the physical topology and available bandwidth -* the Node objects are connected via Link objects representing - the requested or assigned virtual topology of a slice -""" -class Topology: - def __init__(self, api): - self.api = api - self.sites = get_sites(api) - self.nodes = get_nodes(api) - self.tags = get_slice_tags(api) - self.sitelinks = [] - self.nodelinks = [] - - for (s1, s2) in PhysicalLinks: - self.sitelinks.append(Link(self.sites[s1], self.sites[s2])) - - for id in self.nodes: - self.nodes[id].add_tag(self.sites) - - for t in self.tags: - tag = self.tags[t] - if tag.tagname == 'topo_rspec': - node1 = self.nodes[tag.node_id] - l = eval(tag.value) - for (id, realip, bw, lvip, rvip, vnet) in l: - allocbps = get_tc_rate(bw) - node1.bps -= allocbps - try: - node2 = self.nodes[id] - if node1.id < node2.id: - sl = node1.get_sitelink(node2, self.sites) - sl.bps -= allocbps - except: - pass - - - """ Lookup site based on id or idtag value """ - def lookupSite(self, id): - val = None - if isinstance(id, basestring): - id = int(id.lstrip('s')) - try: - val = self.sites[id] - except: - raise KeyError("site ID %s not found" % id) - return val - - def getSites(self): - sites = [] - for s in self.sites: - sites.append(self.sites[s]) - return sites - - """ Lookup node based on id or idtag value """ - def lookupNode(self, id): - val = None - if isinstance(id, basestring): - id = int(id.lstrip('n')) - try: - val = self.nodes[id] - except: - raise KeyError("node ID %s not found" % id) - return val - - def getNodes(self): - nodes = [] - for n in self.nodes: - nodes.append(self.nodes[n]) - return nodes - - def nodesInTopo(self): - nodes = [] - for n in self.nodes: - node = self.nodes[n] - if node.sliver: - nodes.append(node) - return nodes - - def lookupSliceTag(self, id): - val = None - try: - val = self.tags[id] - except: - raise KeyError("slicetag ID %s not found" % id) - return val - - def getSliceTags(self): - tags = [] - for t in self.tags: - tags.append(self.tags[t]) - return tags - - def lookupSiteLink(self, node1, node2): - site1 = self.sites[node1.site_id] - site2 = self.sites[node2.site_id] - for link in self.sitelinks: - if site1 == link.end1 and site2 == link.end2: - return link - if site2 == link.end1 and site1 == link.end2: - return link - return None - - - def __add_vlink(self, vlink, slicenodes, parent = None): - n1 = n2 = None - endpoints = vlink.get("endpoints") - if endpoints: - (end1, end2) = endpoints.split() - n1 = self.lookupNode(end1) - n2 = self.lookupNode(end2) - elif parent: - """ Try to infer the endpoints for the virtual link """ - site_endpoints = parent.get("endpoints") - (n1, n2) = self.__infer_endpoints(site_endpoints, slicenodes) - else: - raise Error("no endpoints given") - - #print "Added virtual link: %s -- %s" % (n1.tag, n2.tag) - bps = int(vlink.findtext("kbps")) * 1000 - sitelink = self.lookupSiteLink(n1, n2) - if not sitelink: - raise PermissionError("nodes %s and %s not adjacent" % - (n1.idtag, n2.idtag)) - self.nodelinks.append(Link(n1, n2, bps, sitelink)) - return - - """ - Infer the endpoints of the virtual link. If the slice exists on - only a single node at each end of the physical link, we'll assume that - the user wants the virtual link to terminate at these nodes. - """ - def __infer_endpoints(self, endpoints, slicenodes): - n = [] - ends = endpoints.split() - for end in ends: - found = 0 - site = self.lookupSite(end) - for id in site.node_ids: - if id in slicenodes: - n.append(slicenodes[id]) - found += 1 - if found != 1: - raise Error("could not infer endpoint for site %s" % site.id) - #print "Inferred endpoints: %s %s" % (n[0].idtag, n[1].idtag) - return n - - def nodeTopoFromRSpec(self, xml): - if self.nodelinks: - raise Error("virtual topology already present") - - nodedict = {} - for node in self.getNodes(): - nodedict[node.idtag] = node - - slicenodes = {} - - tree = etree.parse(StringIO(xml)) - - # Validate the incoming request against the RelaxNG schema - relaxng_doc = etree.parse(VINI_RELAXNG_SCHEMA) - relaxng = etree.RelaxNG(relaxng_doc) - - if not relaxng(tree): - error = relaxng.error_log.last_error - message = "%s (line %s)" % (error.message, error.line) - raise InvalidRSpec(message) - - rspec = tree.getroot() - - """ - Handle requests where the user has annotated a description of the - physical resources (nodes and links) with virtual ones (slivers - and vlinks). - """ - # Find slivers under node elements - for sliver in rspec.iterfind("./network/site/node/sliver"): - elem = sliver.getparent() - node = nodedict[elem.get("id")] - slicenodes[node.id] = node - node.add_sliver() - - # Find vlinks under link elements - for vlink in rspec.iterfind("./network/link/vlink"): - link = vlink.getparent() - self.__add_vlink(vlink, slicenodes, link) - - """ - Handle requests where the user has listed the virtual resources only - """ - # Find slivers that specify nodeid - for sliver in rspec.iterfind("./request/sliver[@nodeid]"): - node = nodedict[sliver.get("nodeid")] - slicenodes[node.id] = node - node.add_sliver() - - # Find vlinks that specify endpoints - for vlink in rspec.iterfind("./request/vlink[@endpoints]"): - self.__add_vlink(vlink, slicenodes) - - return - - def nodeTopoFromSliceTags(self, slice): - if self.nodelinks: - raise Error("virtual topology already present") - - for node in slice.get_nodes(self.nodes): - node.sliver = True - linktag = slice.get_tag('topo_rspec', self.tags, node) - if linktag: - l = eval(linktag.value) - for (id, realip, bw, lvip, rvip, vnet) in l: - if node.id < id: - bps = get_tc_rate(bw) - remote = self.lookupNode(id) - sitelink = self.lookupSiteLink(node, remote) - self.nodelinks.append(Link(node,remote,bps,sitelink)) - - def updateSliceTags(self, slice): - if not self.nodelinks: - return - - slice.update_tag('vini_topo', 'manual', self.tags) - slice.assign_egre_key(self.tags) - slice.turn_on_netns(self.tags) - slice.add_cap_net_admin(self.tags) - - for node in slice.get_nodes(self.nodes): - linkdesc = [] - for link in node.links: - linkdesc.append(node.get_topo_rspec(link)) - if linkdesc: - topo_str = "%s" % linkdesc - slice.update_tag('topo_rspec', topo_str, self.tags, node) - - # Update slice tags in database - for tag in self.getSliceTags(): - if tag.slice_id == slice.id: - if tag.tagname == 'topo_rspec' and not tag.updated: - tag.delete() - tag.write(self.api) - - """ - Check the requested topology against the available topology and capacity - """ - def verifyNodeTopo(self, hrn, topo): - for link in self.nodelinks: - if link.bps <= 0: - raise GeniInvalidArgument(bw, "BW") - - n1 = link.end1 - n2 = link.end2 - sitelink = self.lookupSiteLink(n1, n2) - if not sitelink: - raise PermissionError("%s: nodes %s and %s not adjacent" % (hrn, n1.tag, n2.tag)) - if sitelink.bps < link.bps: - raise PermissionError("%s: insufficient capacity between %s and %s" % (hrn, n1.tag, n2.tag)) - - """ - Produce XML directly from the topology specification. - """ - def toxml(self, hrn = None): - xml = XMLBuilder(format = True, tab_step = " ") - with xml.RSpec(type="VINI"): - if hrn: - element = xml.network(name="Public_VINI", slice=hrn) - else: - element = xml.network(name="Public_VINI") - - with element: - for site in self.getSites(): - site.toxml(xml, hrn, self.nodes) - for link in self.sitelinks: - link.toxml(xml) - - header = '\n' - return header + str(xml) - -""" -Create a dictionary of site objects keyed by site ID -""" -def get_sites(api): - tmp = [] - for site in api.driver.GetSites(): - t = site['site_id'], Site(site) - tmp.append(t) - return dict(tmp) - - -""" -Create a dictionary of node objects keyed by node ID -""" -def get_nodes(api): - tmp = [] - for node in api.driver.GetNodes(): - t = node['node_id'], Node(node) - tmp.append(t) - return dict(tmp) - -""" -Create a dictionary of slice objects keyed by slice ID -""" -def get_slice(api, slicename): - slice = api.driver.GetSlices([slicename]) - if slice: - return Slice(slice[0]) - else: - return None - -""" -Create a dictionary of slicetag objects keyed by slice tag ID -""" -def get_slice_tags(api): - tmp = [] - for tag in api.driver.GetSliceTags(): - t = tag['slice_tag_id'], Slicetag(tag) - tmp.append(t) - return dict(tmp) - -""" -Find a free EGRE key -""" -def free_egre_key(slicetags): - used = set() - for i in slicetags: - tag = slicetags[i] - if tag.tagname == 'egre_key': - used.add(int(tag.value)) - - for i in range(1, 256): - if i not in used: - key = i - break - else: - raise KeyError("No more EGRE keys available") - - return "%s" % key - diff --git a/sfa/managers/vini/vini.rnc b/sfa/managers/vini/vini.rnc deleted file mode 100644 index 09cf902a..00000000 --- a/sfa/managers/vini/vini.rnc +++ /dev/null @@ -1,72 +0,0 @@ -start = RSpec -RSpec = element RSpec { - attribute expires { xsd:NMTOKEN }, - attribute generated { xsd:NMTOKEN }, - attribute type { xsd:NMTOKEN }, - ( network | request ) -} -network = element network { - attribute name { xsd:NMTOKEN }, - attribute slice { xsd:NMTOKEN }?, - sliver_defaults?, - site+, - link* -} -sliver_defaults = element sliver_defaults { - sliver_elements -} -site = element site { - attribute id { xsd:ID }, - element name { text }, - node* -} -node = element node { - attribute id { xsd:ID }, - element hostname { text }, - element bw_unallocated { attribute units { xsd:NMTOKEN }, text }?, - element bw_limit { attribute units { xsd:NMTOKEN }, text }?, - sliver* -} -link = element link { - attribute endpoints { xsd:IDREFS }, - element description { text }?, - element bw_unallocated { attribute units { xsd:NMTOKEN }, text }, - vlink* -} -request = element request { - attribute name { xsd:NMTOKEN }, - sliver_defaults?, - sliver*, - vlink* -} -sliver = element sliver { - attribute nodeid { xsd:ID }?, - sliver_elements -} -sliver_elements = ( - element capabilities { text }? - & element codemux { text }* - & element cpu_pct { text }? - & element cpu_share { text }? - & element delegations { text }? - & element disk_max { text }? - & element initscript { text }? - & element ip_addresses {text }* - & element net_i2_max_kbyte { text }? - & element net_i2_max_rate { text }? - & element net_i2_min_rate { text }? - & element net_i2_share { text }? - & element net_i2_thresh_kbyte { text }? - & element net_max_kbyte { text }? - & element net_max_rate { text }? - & element net_min_rate { text }? - & element net_share { text }? - & element net_thresh_kbyte { text }? - & element vsys {text}* - & element vsys_vnet { text }? -) -vlink = element vlink { - attribute endpoints { xsd:IDREFS }?, - element description { text }?, - element kbps { text }? -} diff --git a/sfa/managers/vini/vini.rng b/sfa/managers/vini/vini.rng deleted file mode 100644 index 387c8313..00000000 --- a/sfa/managers/vini/vini.rng +++ /dev/null @@ -1,261 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/sfa/managers/vini/vini.xml b/sfa/managers/vini/vini.xml deleted file mode 100644 index eb0049b3..00000000 --- a/sfa/managers/vini/vini.xml +++ /dev/null @@ -1,373 +0,0 @@ - - - - - Princeton - - node1.princeton.vini-veritas.net - 999000 - 1000000 - - - - PSG - - node1.psg.vini-veritas.net - 1000000 - 1000000 - - - node2.psg.vini-veritas.net - 1000000 - 1000000 - - - - NLR Chicago - - node1.chic.nlr.vini-veritas.net - 1000000 - 1000000 - - - node2.chic.nlr.vini-veritas.net - 1000000 - 1000000 - - - - NLR Houston - - node1.hous.nlr.vini-veritas.net - 1000000 - 1000000 - - - node2.hous.nlr.vini-veritas.net - 1000000 - 1000000 - - - - NLR Atlanta - - node1.atla.nlr.vini-veritas.net - 1000000 - 1000000 - - - node2.atla.nlr.vini-veritas.net - 1000000 - 1000000 - - - - NLR Seattle - - node1.seat.nlr.vini-veritas.net - 1000000 - 1000000 - - - node2.seat.nlr.vini-veritas.net - 1000000 - 1000000 - - - - NLR Los Angeles - - node1.losa.nlr.vini-veritas.net - 1000000 - 1000000 - - - node2.losa.nlr.vini-veritas.net - 1000000 - 1000000 - - - - NLR New York - - node1.newy.nlr.vini-veritas.net - 1000000 - 1000000 - - - node2.newy.nlr.vini-veritas.net - 1000000 - 1000000 - - - - NLR Wash DC - - node1.wash.nlr.vini-veritas.net - 1000000 - 1000000 - - - node2.wash.nlr.vini-veritas.net - 1000000 - 1000000 - - - - I2 Chicago - - node1.chic.internet2.vini-veritas.net - 963000 - 1000000 - - - node2.chic.internet2.vini-veritas.net - 1000000 - 1000000 - - - - I2 New York - - node1.newy.internet2.vini-veritas.net - 988000 - 1000000 - - - node2.newy.internet2.vini-veritas.net - 1000000 - 1000000 - - - - I2 Wash DC - - node1.wash.internet2.vini-veritas.net - 964000 - 1000000 - - - node2.wash.internet2.vini-veritas.net - 1000000 - 1000000 - - - - Georgia Tech - - node1.gatech.vini-veritas.net - 999000 - 1000000 - - - - I2 Atlanta - - node1.atla.internet2.vini-veritas.net - 964000 - 1000000 - - - node2.atla.internet2.vini-veritas.net - 1000000 - 1000000 - - - - CESNET - - node1.cesnet.vini-veritas.net - 1000000 - 1000000 - - - node2.cesnet.vini-veritas.net - 1000000 - - - - I2 Kansas City - - node1.kans.internet2.vini-veritas.net - 961000 - 1000000 - - - node2.kans.internet2.vini-veritas.net - 1000000 - 1000000 - - - - I2 Houston - - node1.hous.internet2.vini-veritas.net - 964000 - 1000000 - - - node2.hous.internet2.vini-veritas.net - 1000000 - 1000000 - - - - I2 Los Angeles - - node1.losa.internet2.vini-veritas.net - 964000 - 1000000 - - - node2.losa.internet2.vini-veritas.net - 1000000 - 1000000 - - - - I2 Seattle - - node1.seat.internet2.vini-veritas.net - 975000 - 1000000 - - - node2.seat.internet2.vini-veritas.net - 1000000 - 1000000 - - - - I2 Salt Lake - - node1.salt.internet2.vini-veritas.net - 962000 - 1000000 - - - node2.salt.internet2.vini-veritas.net - 1000000 - 1000000 - - - - UMKC - - node1.umkc.vini-veritas.net - 999000 - - - node2.umkc.vini-veritas.net - 1000000 - - - - Princeton -- I2 New York - 1000000 - - - NLR Chicago -- NLR Houston - 1000000 - - - NLR Chicago -- NLR Atlanta - 1000000 - - - NLR Chicago -- NLR Seattle - 1000000 - - - NLR Chicago -- NLR New York - 1000000 - - - NLR Chicago -- NLR Wash DC - 1000000 - - - NLR Houston -- NLR Atlanta - 1000000 - - - NLR Houston -- NLR Los Angeles - 1000000 - - - NLR Atlanta -- NLR Wash DC - 1000000 - - - NLR Atlanta -- Georgia Tech - 1000000 - - - NLR Seattle -- NLR Los Angeles - 1000000 - - - NLR New York -- NLR Wash DC - 1000000 - - - I2 Chicago -- I2 Wash DC - 988000 - - - I2 Chicago -- I2 Atlanta - 988000 - - - I2 Chicago -- CESNET - 1000000 - - - I2 Chicago -- I2 Kansas City - 987000 - - - I2 New York -- I2 Wash DC - 988000 - - - I2 Wash DC -- I2 Atlanta - 988000 - - - Georgia Tech -- I2 Atlanta - 1000000 - - - I2 Atlanta -- I2 Houston - 988000 - - - I2 Kansas City -- I2 Houston - 988000 - - - I2 Kansas City -- I2 Salt Lake - 987000 - - - I2 Kansas City -- UMKC - 999000 - - - I2 Houston -- I2 Los Angeles - 988000 - - - I2 Los Angeles -- I2 Seattle - 988000 - - - I2 Los Angeles -- I2 Salt Lake - 988000 - - - I2 Seattle -- I2 Salt Lake - 987000 - - - diff --git a/sfa/managers/vini/vini_network.py b/sfa/managers/vini/vini_network.py deleted file mode 100644 index ba1ce2a4..00000000 --- a/sfa/managers/vini/vini_network.py +++ /dev/null @@ -1,469 +0,0 @@ -from __future__ import with_statement -from sfa.util.faults import * -from xmlbuilder import XMLBuilder -from lxml import etree -import sys -from sfa.plc.network import * -from sfa.managers.vini.topology import PhysicalLinks - -# Taken from bwlimit.py -# -# See tc_util.c and http://physics.nist.gov/cuu/Units/binary.html. Be -# warned that older versions of tc interpret "kbps", "mbps", "mbit", -# and "kbit" to mean (in this system) "kibps", "mibps", "mibit", and -# "kibit" and that if an older version is installed, all rates will -# be off by a small fraction. -suffixes = { - "": 1, - "bit": 1, - "kibit": 1024, - "kbit": 1000, - "mibit": 1024*1024, - "mbit": 1000000, - "gibit": 1024*1024*1024, - "gbit": 1000000000, - "tibit": 1024*1024*1024*1024, - "tbit": 1000000000000, - "bps": 8, - "kibps": 8*1024, - "kbps": 8000, - "mibps": 8*1024*1024, - "mbps": 8000000, - "gibps": 8*1024*1024*1024, - "gbps": 8000000000, - "tibps": 8*1024*1024*1024*1024, - "tbps": 8000000000000 -} - - -def get_tc_rate(s): - """ - Parses an integer or a tc rate string (e.g., 1.5mbit) into bits/second - """ - - if type(s) == int: - return s - m = re.match(r"([0-9.]+)(\D*)", s) - if m is None: - return -1 - suffix = m.group(2).lower() - if suffixes.has_key(suffix): - return int(float(m.group(1)) * suffixes[suffix]) - else: - return -1 - -def format_tc_rate(rate): - """ - Formats a bits/second rate into a tc rate string - """ - - if rate >= 1000000000 and (rate % 1000000000) == 0: - return "%.0fgbit" % (rate / 1000000000.) - elif rate >= 1000000 and (rate % 1000000) == 0: - return "%.0fmbit" % (rate / 1000000.) - elif rate >= 1000: - return "%.0fkbit" % (rate / 1000.) - else: - return "%.0fbit" % rate - - -class ViniNode(Node): - def __init__(self, network, node, bps = 1000 * 1000000): - Node.__init__(self, network, node) - self.bps = bps - self.links = set() - self.name = self.hostname.replace('.vini-veritas.net', '') - - def get_link_id(self, remote): - if self.id < remote.id: - link = (self.id<<7) + remote.id - else: - link = (remote.id<<7) + self.id - return link - - def get_iface_id(self, remote): - if self.id < remote.id: - iface = 1 - else: - iface = 2 - return iface - - def get_virt_ip(self, remote): - link = self.get_link_id(remote) - iface = self.get_iface_id(remote) - first = link >> 6 - second = ((link & 0x3f)<<2) + iface - return "192.168.%d.%d" % (first, second) - - def get_virt_net(self, remote): - link = self.get_link_id(remote) - first = link >> 6 - second = (link & 0x3f)<<2 - return "192.168.%d.%d/30" % (first, second) - - def get_topo_rspec(self, link): - if link.end1 == self: - remote = link.end2 - elif link.end2 == self: - remote = link.end1 - else: - raise Error("Link does not connect to Node") - - my_ip = self.get_virt_ip(remote) - remote_ip = remote.get_virt_ip(self) - net = self.get_virt_net(remote) - bw = format_tc_rate(link.bps) - ipaddr = remote.get_primary_iface().ipv4 - return (remote.id, ipaddr, bw, my_ip, remote_ip, net) - - def add_link(self, link): - self.links.add(link) - - # Assumes there is at most one Link between two sites - def get_sitelink(self, node): - site1 = self.network.sites[self.site_id] - site2 = self.network.sites[node.site_id] - sl = site1.links.intersection(site2.links) - if len(sl): - return sl.pop() - return None - - def toxml(self, xml): - slice = self.network.slice - if self.whitelist and not self.sliver: - if not slice or slice.id not in self.whitelist: - return - - with xml.node(id = self.idtag): - with xml.hostname: - xml << self.hostname - with xml.bw_unallocated(units="kbps"): - xml << str(int(self.bps/1000)) - self.get_primary_iface().toxml(xml) - if self.sliver: - self.sliver.toxml(xml) - - -class ViniSite(Site): - def __init__(self, network, site): - Site.__init__(self, network, site) - self.links = set() - - def add_link(self, link): - self.links.add(link) - -class ViniSlice(Slice): - def assign_egre_key(self): - tag = self.get_tag('egre_key') - if not tag: - try: - key = self.network.free_egre_key() - except: - raise InvalidRSpec("ran out of EGRE keys!") - tag = self.update_tag('egre_key', key, None, 'admin') - return - - def turn_on_netns(self): - tag = self.get_tag('netns') - if (not tag) or (tag.value != '1'): - tag = self.update_tag('netns', '1', None, 'admin') - return - - def turn_off_netns(self): - tag = self.get_tag('netns') - if tag and (tag.value != '0'): - tag.delete() - return - - def add_cap_net_admin(self): - tag = self.get_tag('capabilities') - if tag: - caps = tag.value.split(',') - for cap in caps: - if cap == "CAP_NET_ADMIN": - newcaps = tag.value - break - else: - newcaps = "CAP_NET_ADMIN," + tag.value - self.update_tag('capabilities', newcaps, None, 'admin') - else: - tag = self.add_tag('capabilities', 'CAP_NET_ADMIN', None, 'admin') - return - - def remove_cap_net_admin(self): - tag = self.get_tag('capabilities') - if tag: - caps = tag.value.split(',') - newcaps = [] - for cap in caps: - if cap != "CAP_NET_ADMIN": - newcaps.append(cap) - if newcaps: - value = ','.join(newcaps) - self.update_tag('capabilities', value, None, 'admin') - else: - tag.delete() - return - -class Link: - def __init__(self, end1, end2, bps = 1000 * 1000000, parent = None): - self.end1 = end1 - self.end2 = end2 - self.bps = bps - self.parent = parent - self.children = [] - - end1.add_link(self) - end2.add_link(self) - - if self.parent: - self.parent.children.append(self) - - def toxml(self, xml): - end_ids = "%s %s" % (self.end1.idtag, self.end2.idtag) - - if self.parent: - with xml.vlink(endpoints=end_ids): - with xml.description: - xml << "%s -- %s" % (self.end1.name, self.end2.name) - with xml.kbps: - xml << str(int(self.bps/1000)) - else: - with xml.link(endpoints=end_ids): - with xml.description: - xml << "%s -- %s" % (self.end1.name, self.end2.name) - with xml.bw_unallocated(units="kbps"): - xml << str(int(self.bps/1000)) - for child in self.children: - child.toxml(xml) - - - -class ViniNetwork(Network): - def __init__(self, api, type = "SFA"): - Network.__init__(self, api, type) - self.sitelinks = [] - self.nodelinks = [] - - for (s1, s2) in PhysicalLinks: - self.sitelinks.append(Link(self.sites[s1], self.sites[s2])) - - for t in self.tags: - tag = self.tags[t] - if tag.tagname == 'topo_rspec': - node1 = self.nodes[tag.node_id] - l = eval(tag.value) - for (id, realip, bw, lvip, rvip, vnet) in l: - allocbps = get_tc_rate(bw) - node1.bps -= allocbps - try: - node2 = self.nodes[id] - if node1.id < node2.id: - sl = node1.get_sitelink(node2) - sl.bps -= allocbps - except: - pass - - def lookupSiteLink(self, node1, node2): - site1 = self.sites[node1.site_id] - site2 = self.sites[node2.site_id] - for link in self.sitelinks: - if site1 == link.end1 and site2 == link.end2: - return link - if site2 == link.end1 and site1 == link.end2: - return link - return None - - - """ - Check the requested topology against the available topology and capacity - """ - def verifyTopology(self): - for link in self.nodelinks: - if link.bps <= 0: - raise InvalidRSpec("must request positive bandwidth") - - n1 = link.end1 - n2 = link.end2 - sitelink = self.lookupSiteLink(n1, n2) - if not sitelink: - raise InvalidRSpec("nodes %s and %s are not adjacent" % - (n1.idtag, n2.idtag)) - if sitelink.bps < link.bps: - raise InvalidRSpec("not enough capacity between %s and %s" % - (n1.idtag, n2.idtag)) - - def __add_vlink(self, vlink, parent = None): - n1 = n2 = None - endpoints = vlink.get("endpoints") - if endpoints: - (end1, end2) = endpoints.split() - n1 = self.lookupNode(end1) - n2 = self.lookupNode(end2) - elif parent: - """ Try to infer the endpoints for the virtual link """ - site_endpoints = parent.get("endpoints") - (n1, n2) = self.__infer_endpoints(site_endpoints) - else: - raise InvalidRSpec("no endpoints given") - - #print "Added virtual link: %s -- %s" % (n1.tag, n2.tag) - bps = int(vlink.findtext("kbps")) * 1000 - sitelink = self.lookupSiteLink(n1, n2) - if not sitelink: - raise InvalidRSpec("nodes %s and %s are not adjacent" % - (n1.idtag, n2.idtag)) - self.nodelinks.append(Link(n1, n2, bps, sitelink)) - return - - """ - Infer the endpoints of the virtual link. If the slice exists on - only a single node at each end of the physical link, we'll assume that - the user wants the virtual link to terminate at these nodes. - """ - def __infer_endpoints(self, endpoints): - n = [] - ends = endpoints.split() - for end in ends: - found = 0 - site = self.lookupSite(end) - for id in site.node_ids: - if id in self.nodedict: - n.append(self.nodedict[id]) - found += 1 - if found != 1: - raise InvalidRSpec("could not infer endpoint for site %s" % - site.idtag) - #print "Inferred endpoints: %s %s" % (n[0].idtag, n[1].idtag) - return n - - def addRSpec(self, xml, schema = None): - Network.addRSpec(self, xml, schema) - self.nodedict = {} - for node in self.nodesWithSlivers(): - self.nodedict[node.id] = node - - # Find vlinks under link elements - for vlink in self.rspec.iterfind("./network/link/vlink"): - link = vlink.getparent() - self.__add_vlink(vlink, link) - - # Find vlinks that specify endpoints - for vlink in self.rspec.iterfind("./request/vlink[@endpoints]"): - self.__add_vlink(vlink) - - - def addSlice(self): - Network.addSlice(self) - - for node in self.slice.get_nodes(): - linktag = self.slice.get_tag('topo_rspec', node) - if linktag: - l = eval(linktag.value) - for (id, realip, bw, lvip, rvip, vnet) in l: - if node.id < id: - bps = get_tc_rate(bw) - remote = self.lookupNode(id) - sitelink = self.lookupSiteLink(node, remote) - self.nodelinks.append(Link(node,remote,bps,sitelink)) - - - def updateSliceTags(self): - slice = self.slice - - tag = slice.update_tag('vini_topo', 'manual', None, 'admin') - slice.assign_egre_key() - slice.turn_on_netns() - slice.add_cap_net_admin() - - for node in self.nodesWithSlivers(): - linkdesc = [] - for link in node.links: - linkdesc.append(node.get_topo_rspec(link)) - if linkdesc: - topo_str = "%s" % linkdesc - tag = slice.update_tag('topo_rspec', topo_str, node, 'admin') - - # Expire the un-updated topo_rspec tags - for tag in self.getSliceTags(): - if tag.tagname in ['topo_rspec']: - if not tag.was_updated(): - tag.delete() - - Network.updateSliceTags(self) - - """ - Find a free EGRE key - """ - def free_egre_key(self): - used = set() - for tag in self.getSliceTags(): - if tag.tagname == 'egre_key': - used.add(int(tag.value)) - - for i in range(1, 256): - if i not in used: - key = i - break - else: - raise KeyError("No more EGRE keys available") - - return str(key) - - """ - Produce XML directly from the topology specification. - """ - def toxml(self): - xml = XMLBuilder(format = True, tab_step = " ") - with xml.RSpec(type=self.type): - if self.slice: - element = xml.network(name=self.api.hrn, slice=self.slice.hrn) - else: - element = xml.network(name=self.api.hrn) - - with element: - if self.slice: - self.slice.toxml(xml) - for site in self.getSites(): - site.toxml(xml) - for link in self.sitelinks: - link.toxml(xml) - - header = '\n' - return header + str(xml) - - """ - Create a dictionary of ViniSite objects keyed by site ID - """ - def get_sites(self, api): - tmp = [] - for site in api.driver.GetSites({'peer_id': None}): - t = site['site_id'], ViniSite(self, site) - tmp.append(t) - return dict(tmp) - - - """ - Create a dictionary of ViniNode objects keyed by node ID - """ - def get_nodes(self, api): - tmp = [] - for node in api.driver.GetNodes({'peer_id': None}): - t = node['node_id'], ViniNode(self, node) - tmp.append(t) - return dict(tmp) - - """ - Return a ViniSlice object for a single slice - """ - def get_slice(self, api, hrn): - slicename = hrn_to_pl_slicename(hrn) - slice = api.driver.GetSlices([slicename]) - if slice: - self.slice = ViniSlice(self, slicename, slice[0]) - return self.slice - else: - return None - - - diff --git a/sfa/plc/aggregate.py b/sfa/plc/aggregate.py index abd118f8..bda3bbcc 100644 --- a/sfa/plc/aggregate.py +++ b/sfa/plc/aggregate.py @@ -1,11 +1,13 @@ #!/usr/bin/python -from sfa.util.xrn import hrn_to_urn, urn_to_hrn -from sfa.util.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename, urn_to_sliver_id +from sfa.util.xrn import hrn_to_urn, urn_to_hrn, urn_to_sliver_id +from sfa.util.plxrn import PlXrn, hostname_to_urn, hrn_to_pl_slicename from sfa.rspecs.rspec import RSpec from sfa.rspecs.elements.hardware_type import HardwareType +from sfa.rspecs.elements.node import Node from sfa.rspecs.elements.link import Link from sfa.rspecs.elements.login import Login +from sfa.rspecs.elements.location import Location from sfa.rspecs.elements.interface import Interface from sfa.rspecs.elements.services import Services from sfa.rspecs.elements.pltag import PLTag @@ -43,7 +45,7 @@ class Aggregate: def get_links(self, filter={}): if not self.api.config.SFA_AGGREGATE_TYPE.lower() == 'vini': - return + return [] topology = Topology() links = {} @@ -131,10 +133,12 @@ class Aggregate: return (slice, slivers) - def get_nodes(self, slice=None): + def get_nodes (self, slice=None,slivers=[]): filter = {} + tags_filter = {} if slice and 'node_ids' in slice and slice['node_ids']: filter['node_id'] = slice['node_ids'] + tags_filter=filter.copy() filter.update({'peer_id': None}) nodes = self.api.driver.GetNodes(filter) @@ -152,9 +156,14 @@ class Aggregate: # get interfaces interfaces = self.get_interfaces({'interface_id':interface_ids}) # get slivers - slivers = self.get_slivers(slice) + # + # thierry: no get_slivers, we have slivers as a result of + # get_slice_and_slivers passed as an argument + # +# slivers = self.get_slivers(slice) + # get tags - node_tags = self.get_node_tags({'node_id': node_ids}) + node_tags = self.get_node_tags(tags_filter) # get initscripts pl_initscripts = self.get_pl_initscripts() @@ -165,6 +174,9 @@ class Aggregate: if not slice or slice['slice_id'] not in node['slice_ids_whitelist']: continue rspec_node = Node() + # xxx how to retrieve site['login_base'] + site_id=node['site_id'] + site=sites_dict[site_id] rspec_node['component_id'] = hostname_to_urn(self.api.hrn, site['login_base'], node['hostname']) rspec_node['component_name'] = node['hostname'] rspec_node['component_manager_id'] = self.api.hrn @@ -183,7 +195,7 @@ class Aggregate: rspec_node['interfaces'] = [] for if_id in node['interface_ids']: interface = Interface(interfaces[if_id]) - interface['ipv4'] = interface['ip'] + interface['ipv4'] = interface['ipv4'] rspec_node['interfaces'].append(interface) tags = [PLTag(node_tags[tag_id]) for tag_id in node['node_tag_ids']] rspec_node['tags'] = tags @@ -213,7 +225,7 @@ class Aggregate: slice, slivers = self.get_slice_and_slivers(slice_xrn) rspec = RSpec(version=rspec_version, user_options=self.user_options) - rspec.version.add_nodes(self.get_nodes(slice, slivers)) + rspec.version.add_nodes(self.get_nodes(slice), slivers) rspec.version.add_links(self.get_links(slice)) # add sliver defaults diff --git a/sfa/plc/slices.py b/sfa/plc/slices.py index fcef25e6..9e188d79 100644 --- a/sfa/plc/slices.py +++ b/sfa/plc/slices.py @@ -184,10 +184,36 @@ class Slices: except: self.api.logger.log_exc('Failed to add/remove slice from nodes') + def free_egre_key(self): + used = set() + for tag in self.api.driver.GetSliceTags({'tagname': 'egre_key'}): + used.add(int(tag['value'])) + + for i in range(1, 256): + if i not in used: + key = i + break + else: + raise KeyError("No more EGRE keys available") + + return str(key) + def verify_slice_links(self, slice, links, aggregate): # nodes is undefined here if not links: - return + return + + slice_tags = [] + + # set egre key + slice_tags.append({'name': 'egre_key', 'value': self.free_egre_key()}) + + # set netns + slice_tags.append({'name': 'netns', 'value': '1'}) + + # set cap_net_admin + # need to update the attribute string? + slice_tags.append({'name': 'capabilities', 'value': 'CAP_NET_ADMIN'}) for link in links: # get the ip address of the first node in the link @@ -198,7 +224,13 @@ class Slices: if1 = aggregate.interfaces[node['interface_ids'][0]] ipaddr = if1['ip'] topo_rspec = VLink.get_topo_rspec(link, ipaddr) - self.api.driver.AddSliceTag(slice['name'], 'topo_rspec', str([topo_rspec]), node_id) + # set topo_rspec tag + slice_tags.append({'name': 'topo_rspec', 'value': str([topo_rspec]), 'node_id': node_id}) + # set vini_topo tag + slice_tags.append({'name': 'vini_topo', 'value': 'manual', 'node_id': node_id}) + #self.api.driver.AddSliceTag(slice['name'], 'topo_rspec', str([topo_rspec]), node_id) + + self.verify_slice_attributes(slice, slice_tags, append=True, admin=True) @@ -487,9 +519,12 @@ class Slices: except: pass - def verify_slice_attributes(self, slice, requested_slice_attributes): + def verify_slice_attributes(self, slice, requested_slice_attributes, append=False, admin=False): # get list of attributes users ar able to manage - slice_attributes = self.api.driver.GetTagTypes({'category': '*slice*', '|roles': ['user']}) + filter = {'category': '*slice*'} + if not admin: + filter['|roles'] = ['user'] + slice_attributes = self.api.driver.GetTagTypes(filter) valid_slice_attribute_names = [attribute['tagname'] for attribute in slice_attributes] # get sliver attributes @@ -514,7 +549,7 @@ class Slices: attribute_found=True break - if not attribute_found: + if not attribute_found and not append: removed_slice_attributes.append(slice_tag) # get attributes that should be added: diff --git a/sfa/rspecs/elements/pl_tag.py b/sfa/rspecs/elements/pltag.py similarity index 100% rename from sfa/rspecs/elements/pl_tag.py rename to sfa/rspecs/elements/pltag.py diff --git a/sfa/rspecs/elements/versions/pgv2Node.py b/sfa/rspecs/elements/versions/pgv2Node.py index 4ddafc1f..b293b049 100644 --- a/sfa/rspecs/elements/versions/pgv2Node.py +++ b/sfa/rspecs/elements/versions/pgv2Node.py @@ -10,7 +10,7 @@ from sfa.rspecs.elements.hardware_type import HardwareType from sfa.rspecs.elements.disk_image import DiskImage from sfa.rspecs.elements.interface import Interface from sfa.rspecs.elements.bwlimit import BWlimit -from sfa.rspecs.elements.pl_tag import PLTag +from sfa.rspecs.elements.pltag import PLTag from sfa.rspecs.rspec_elements import RSpecElement, RSpecElements from sfa.rspecs.elements.versions.pgv2Service import PGv2Service @@ -100,6 +100,7 @@ class PGv2Node: for services_elem in services_elems: # services element has no useful info, but the child elements do for child in services_elem.iterchildren(): + pass # set interfaces interface_elems = node_elem.xpath(PGv2Node.elements['interface'].path, xml.namespaces) diff --git a/sfa/rspecs/elements/versions/pgv2Services.py b/sfa/rspecs/elements/versions/pgv2Services.py index 470a2e65..fe946b3a 100644 --- a/sfa/rspecs/elements/versions/pgv2Services.py +++ b/sfa/rspecs/elements/versions/pgv2Services.py @@ -9,9 +9,9 @@ from sfa.rspecs.rspec_elements import RSpecElement, RSpecElements class PGv2Services: elements = { 'services': RSpecElement(RSpecElements.SERVICES, '//default:services | //services'), - 'install': RSpecElement(RspecElements.INSTALL, './default:install | ./install'), - 'execute': RSpecElement(RspecElements.EXECUTE, './default:execute | ./execute'), - 'login': RSpecElement(RspecElements.LOGIN, './default:login | ./login'), + 'install': RSpecElement(RSpecElements.INSTALL, './default:install | ./install'), + 'execute': RSpecElement(RSpecElements.EXECUTE, './default:execute | ./execute'), + 'login': RSpecElement(RSpecElements.LOGIN, './default:login | ./login'), } @staticmethod diff --git a/sfa/rspecs/elements/versions/sfav1Node.py b/sfa/rspecs/elements/versions/sfav1Node.py index 6cdf9954..0b67c91d 100644 --- a/sfa/rspecs/elements/versions/sfav1Node.py +++ b/sfa/rspecs/elements/versions/sfav1Node.py @@ -10,7 +10,7 @@ from sfa.rspecs.elements.hardware_type import HardwareType from sfa.rspecs.elements.disk_image import DiskImage from sfa.rspecs.elements.interface import Interface from sfa.rspecs.elements.bwlimit import BWlimit -from sfa.rspecs.elements.pl_tag import PLTag +from sfa.rspecs.elements.pltag import PLTag from sfa.rspecs.rspec_elements import RSpecElement, RSpecElements from sfa.rspecs.elements.versions.sfav1Network import SFAv1Network from sfa.rspecs.elements.versions.pgv2Services import PGv2Services diff --git a/sfa/util/xml.py b/sfa/util/xml.py index ddb06e4f..b2aea13b 100755 --- a/sfa/util/xml.py +++ b/sfa/util/xml.py @@ -2,11 +2,22 @@ from types import StringTypes from lxml import etree from StringIO import StringIO - from sfa.util.faults import InvalidXML class XpathFilter: @staticmethod + + def filter_value(key, value): + xpath = "" + if isinstance(value, str): + if '*' in value: + value = value.replace('*', '') + xpath = 'contains(%s, "%s")' % (key, value) + else: + xpath = '%s="%s"' % (key, value) + return xpath + + @staticmethod def xpath(filter={}): xpath = "" if filter: @@ -17,27 +28,72 @@ class XpathFilter: else: key = '@'+key if isinstance(value, str): - filter_list.append('%s="%s"' % (key, value)) + filter_list.append(XpathFilter.filter_value(key, value)) elif isinstance(value, list): - filter_list.append('contains("%s", %s)' % (' '.join(map(str, value)), key)) + stmt = ' or '.join([XpathFilter.filter_value(key, str(val)) for val in value]) + filter_list.append(stmt) if filter_list: xpath = ' and '.join(filter_list) xpath = '[' + xpath + ']' return xpath +class XmlNode: + def __init__(self, node, namespaces): + self.node = node + self.namespaces = namespaces + self.attrib = node.attrib + + def xpath(self, xpath, namespaces=None): + if not namespaces: + namespaces = self.namespaces + return self.node.xpath(xpath, namespaces=namespaces) + + def add_element(name, *args, **kwds): + element = etree.SubElement(name, args, kwds) + return XmlNode(element, self.namespaces) + + def remove_elements(name): + """ + Removes all occurences of an element from the tree. Start at + specified root_node if specified, otherwise start at tree's root. + """ + + if not element_name.startswith('//'): + element_name = '//' + element_name + elements = self.node.xpath('%s ' % name, namespaces=self.namespaces) + for element in elements: + parent = element.getparent() + parent.remove(element) + + def set(self, key, value): + self.node.set(key, value) + + def set_text(self, text): + self.node.text = text + + def unset(self, key): + del self.node.attrib[key] + + def toxml(self): + return etree.tostring(self.node, encoding='UTF-8', pretty_print=True) + + def __str__(self): + return self.toxml() + class XML: - def __init__(self, xml=None): + def __init__(self, xml=None, namespaces=None): self.root = None - self.namespaces = None + self.namespaces = namespaces self.default_namespace = None self.schema = None if isinstance(xml, basestring): self.parse_xml(xml) - elif isinstance(xml, etree._ElementTree): - self.root = xml.getroot() - elif isinstance(xml, etree._Element): - self.root = xml + if isinstance(xml, XmlNode): + self.root = xml + self.namespces = xml.namespaces + elif isinstance(xml, etree._ElementTree) or isinstance(xml, etree._Element): + self.parse_xml(etree.tostring(xml)) def parse_xml(self, xml): """ @@ -52,9 +108,9 @@ class XML: tree = etree.parse(StringIO(xml), parser) except Exception, e: raise InvalidXML(str(e)) - self.root = tree.getroot() + root = tree.getroot() + self.namespaces = dict(root.nsmap) # set namespaces map - self.namespaces = dict(self.root.nsmap) if 'default' not in self.namespaces and None in self.namespaces: # If the 'None' exist, then it's pointing to the default namespace. This makes # it hard for us to write xpath queries for the default naemspace because lxml @@ -64,6 +120,8 @@ class XML: else: self.namespaces['default'] = 'default' + + self.root = XmlNode(root, self.namespaces) # set schema for key in self.root.attrib.keys(): if key.endswith('schemaLocation'): @@ -126,15 +184,16 @@ class XML: namespaces = self.namespaces return self.root.xpath(xpath, namespaces=namespaces) - def set(self, key, value): - return self.root.set(key, value) + def set(self, key, value, node=None): + if not node: + node = self.root + return node.set(key, value) - def add_attribute(self, elem, name, value): - """ - Add attribute to specified etree element - """ - opt = etree.SubElement(elem, name) - opt.text = value + def remove_attribute(self, name, node=None): + if not node: + node = self.root + node.remove_attribute(name) + def add_element(self, name, attrs={}, parent=None, text=""): """ @@ -150,34 +209,17 @@ class XML: if isinstance(attrs, dict): for attr in attrs: element.set(attr, attrs[attr]) - return element + return XmlNode(element, self.namespaces) - def remove_attribute(self, elem, name, value): - """ - Removes an attribute from an element - """ - if elem is not None: - opts = elem.iterfind(name) - if opts is not None: - for opt in opts: - if opt.text == value: - elem.remove(opt) - - def remove_element(self, element_name, root_node = None): + def remove_elements(self, name, node = None): """ Removes all occurences of an element from the tree. Start at specified root_node if specified, otherwise start at tree's root. """ - if not root_node: - root_node = self.root - - if not element_name.startswith('//'): - element_name = '//' + element_name + if not node: + node = self.root - elements = root_node.xpath('%s ' % element_name, namespaces=self.namespaces) - for element in elements: - parent = element.getparent() - parent.remove(element) + node.remove_elements(name) def attributes_list(self, elem): # convert a list of attribute tags into list of tuples