From: Claudio-Daniel Freire Date: Wed, 20 Apr 2011 16:54:40 +0000 (+0200) Subject: Implementing PlanetLab testbed X-Git-Tag: nepi_v2~142 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=603b41dc4b985b19e59f5bb9ac75ff361fa99b22;p=nepi.git Implementing PlanetLab testbed --- diff --git a/src/nepi/core/testbed_impl.py b/src/nepi/core/testbed_impl.py index fc8cf060..9018257a 100644 --- a/src/nepi/core/testbed_impl.py +++ b/src/nepi/core/testbed_impl.py @@ -44,6 +44,10 @@ class TestbedController(execute.TestbedController): @property def elements(self): return self._elements + + def _get_factory_id(self, guid): + """ Returns the factory ID of the (perhaps not yet) created object """ + return self._create.get(guid, None) def defer_configure(self, name, value): if not self._attributes.has_attribute(name): diff --git a/src/nepi/testbeds/planetlab/execute.py b/src/nepi/testbeds/planetlab/execute.py index 5e6a63a6..b2c36b91 100644 --- a/src/nepi/testbeds/planetlab/execute.py +++ b/src/nepi/testbeds/planetlab/execute.py @@ -9,7 +9,12 @@ class TestbedController(testbed_impl.TestbedController): def __init__(self, testbed_version): super(TestbedController, self).__init__(TESTBED_ID, testbed_version) self._home_directory = None + self.slicename = None self._traces = dict() + + import node, interfaces + self._node = node + self._interfaces = interfaces @property def home_directory(self): @@ -18,6 +23,22 @@ class TestbedController(testbed_impl.TestbedController): def do_setup(self): self._home_directory = self._attributes.\ get_attribute_value("homeDirectory") + self.slicename = self._attributes.\ + get_attribute_value("slice") + + def do_create(self): + # Create node elements per XML data + super(TestbedController, self).do_create() + + # Perform resource discovery if we don't have + # specific resources assigned yet + self.do_resource_discovery() + + # Create PlanetLab slivers + self.do_provisioning() + + # Wait for all nodes to be ready + self.wait_nodes() def set(self, time, guid, name, value): super(TestbedController, self).set(time, guid, name, value) @@ -71,4 +92,36 @@ class TestbedController(testbed_impl.TestbedController): def follow_trace(self, trace_id, trace): self._traces[trace_id] = trace - + def _make_node(self, parameters): + node = self._node.Node() + + # Note: there is 1-to-1 correspondence between attribute names + # If that changes, this has to change as well + for attr in parameters.get_attribute_names(): + setattr(node, attr, parameters.get_attribute_value(attr)) + + return node + + def _make_node_iface(self, parameters): + iface = self._interfaces.NodeIface() + + # Note: there is 1-to-1 correspondence between attribute names + # If that changes, this has to change as well + for attr in parameters.get_attribute_names(): + setattr(iface, attr, parameters.get_attribute_value(attr)) + + return iface + + def _make_tun_iface(self, parameters): + iface = self._interfaces.TunIface() + + # Note: there is 1-to-1 correspondence between attribute names + # If that changes, this has to change as well + for attr in parameters.get_attribute_names(): + setattr(iface, attr, parameters.get_attribute_value(attr)) + + return iface + + def _make_internet(self, parameters): + return self._node.Internet() + diff --git a/src/nepi/testbeds/planetlab/interfaces.py b/src/nepi/testbeds/planetlab/interfaces.py new file mode 100644 index 00000000..96e4a422 --- /dev/null +++ b/src/nepi/testbeds/planetlab/interfaces.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from constants import TESTBED_ID +import nepi.util.ipaddr +import plcapi + +class NodeIface(object): + def __init__(self, api=None): + if not api: + api = plcapi.PLCAPI() + self._api = api + + # Attributes + self.primary = True + + # These get initialized at configuration time + self.address = None + self.lladdr = None + self.netprefix = None + self.broadcast = True + self._interface_id = None + + # These get initialized when the iface is connected to its node + self.node = None + + # These get initialized when the iface is connected to the internet + self.has_internet = False + + def add_address(self, address, netprefix, broadcast): + raise RuntimeError, "Cannot add explicit addresses to public interface" + + def pick_iface(self, siblings): + """ + Picks an interface using the PLCAPI to query information about the node. + + Needs an assigned node. + + Params: + siblings: other NodeIface elements attached to the same node + """ + + if (self.node or self.node._node_id) is None: + raise RuntimeError, "Cannot pick interface without an assigned node" + + avail = self._api.GetInterfaces( + node_id=self.node._node_id, + is_primary=self.primary, + fields=('interface_id','mac','netmask','ip') )) + + used = set([sibling._interface_id for sibling in siblings + if sibling._interface_id is not None]) + + for candidate in avail: + candidate_id = candidate['interface_id'] + if candidate_id not in used: + # pick it! + self._interface_id = candidate_id + self.address = candidate['ip'] + self.lladdr = candidate['mac'] + self.netprefix = candidate['netmask'] + return + else: + raise RuntimeError, "Cannot configure interface: cannot find suitable interface in PlanetLab node" + + def validate(self): + if not element.has_internet: + raise RuntimeError, "All external interface devices must be connected to the Internet" + + +class TunIface(object): + def __init__(self, api=None): + if not api: + api = plcapi.PLCAPI() + self._api = api + + # Attributes + self.address = None + self.netprefix = None + self.netmask = None + + self.up = None + self.device_name = None + self.mtu = None + self.snat = False + + # These get initialized when the iface is connected to its node + self.node = None + + def add_address(self, address, netprefix, broadcast): + if (self.address or self.netprefix or self.netmask) is not None: + raise RuntimeError, "Cannot add more than one address to TUN interfaces" + if broadcast: + raise ValueError, "TUN interfaces cannot broadcast in PlanetLab" + + self.address = address + self.netprefix = netprefix + self.netmask = nepi.util.ipaddr.ipv4_dot2mask(netprefix) + + def validate(self): + pass + + +# Yep, it does nothing - yet +class Internet(object): + def __init__(self, api=None): + if not api: + api = plcapi.PLCAPI() + self._api = api + + diff --git a/src/nepi/testbeds/planetlab/metadata_v01.py b/src/nepi/testbeds/planetlab/metadata_v01.py index 74b22bc9..d3541f50 100644 --- a/src/nepi/testbeds/planetlab/metadata_v01.py +++ b/src/nepi/testbeds/planetlab/metadata_v01.py @@ -10,26 +10,60 @@ from nepi.util.constants import STATUS_NOT_STARTED, STATUS_RUNNING, \ NODE = "Node" NODEIFACE = "NodeInterface" +TUNIFACE = "TunInterface" APPLICATION = "Application" +INTERNET = "Internet" PL_TESTBED_ID = "planetlab" ### Connection functions #### +def connect_node_iface_node(testbed_instance, node, iface): + iface.node = node + +def connect_node_iface_inet(testbed_instance, node, internet): + iface.has_internet = True + +def connect_tun_iface_node(testbed_instance, node, iface): + iface.node = node + ### Creation functions ### def create_node(testbed_instance, guid): parameters = testbed_instance._get_parameters(guid) - element = testbed_instance.pl.Node() + + # create element with basic attributes + element = testbed_instance._make_node(parameters) + + # add constraint on number of (real) interfaces + # by counting connected devices + dev_guids = testbed_instance.get_connected(guid, "node", "devs") + num_open_ifaces = sum( # count True values + TUNEIFACE == testbed_instance._get_factory_id(guid) + for guid in dev_guids ) + element.min_num_external_ifaces = num_open_ifaces + testbed_instance.elements[guid] = element def create_nodeiface(testbed_instance, guid): parameters = testbed_instance._get_parameters(guid) - element = testbed_instance.pl.Iface() + element = testbed_instance.create_node_iface(parameters) + testbed_instance.elements[guid] = element + +def create_tuniface(testbed_instance, guid): + parameters = testbed_instance._get_parameters(guid) + element = testbed_instance._make_tun_iface(parameters) testbed_instance.elements[guid] = element def create_application(testbed_instance, guid): - testbed_instance.elements[guid] = None # Delayed construction + parameters = testbed_instance._get_parameters(guid) + element = testbed_instance._make_internet(parameters) + testbed_instance.elements[guid] = element + +def create_internet(testbed_instance, guid): + parameters = testbed_instance._get_parameters(guid) + element = None #TODO + testbed_instance.elements[guid] = element ### Start/Stop functions ### @@ -40,22 +74,19 @@ def start_application(testbed_instance, guid): command = parameters["command"] stdout = stderr = None if "stdout" in traces: - filename = testbed_instance.trace_filename(guid, "stdout") - stdout = open(filename, "wb") - testbed_instance.follow_trace("stdout", stdout) + # TODO + pass if "stderr" in traces: - filename = testbed_instance.trace_filename(guid, "stderr") - stderr = open(filename, "wb") - testbed_instance.follow_trace("stderr", stderr) - - node_guid = testbed_instance.get_connected(guid, "node", "apps") - if len(node_guid) == 0: - raise RuntimeError("Can't instantiate interface %d outside netns \ - node" % guid) - node = testbed_instance.elements[node_guid[0]] - element = node.Popen(command, shell = True, stdout = stdout, - stderr = stderr, user = user) - testbed_instance.elements[guid] = element + # TODO + pass + + node_guids = testbed_instance.get_connected(guid, "node", "apps") + if not node_guid: + raise RuntimeError, "Can't instantiate interface %d outside planetlab node" % (guid,) + + node = testbed_instance.elements[node_guids[0]] + # TODO + pass ### Status functions ### @@ -63,31 +94,49 @@ def status_application(testbed_instance, guid): if guid not in testbed_instance.elements.keys(): return STATUS_NOT_STARTED app = testbed_instance.elements[guid] - if app.poll() == None: - return STATUS_RUNNING - return STATUS_FINISHED + # TODO + return STATUS_NOT_STARTED ### Configure functions ### -def configure_device(testbed_instance, guid): +def configure_nodeiface(testbed_instance, guid): + element = testbed_instance._elements[guid] + if not guid in testbed_instance._add_address: + return + + # Cannot explicitly configure addresses + del testbed_instance._add_address[guid] + + # Get siblings + node_guid = testbed_instance.get_connected(guid, "node", "devs")[0] + dev_guids = testbed_instance.get_connected(node_guid, "node", "devs") + siblings = [ self._element[dev_guid] + for dev_guid in dev_guids + if dev_guid != guid ] + + # Fetch address from PLC api + element.pick_iface(siblings) + + # Do some validations + element.validate() + +def configure_tuniface(testbed_instance, guid): element = testbed_instance._elements[guid] if not guid in testbed_instance._add_address: return addresses = testbed_instance._add_address[guid] for address in addresses: (address, netprefix, broadcast) = address - # TODO: Decide if we should add a ipv4 or ipv6 address - element.add_v4_address(address, netprefix) + # TODO + + # Do some validations + element.validate() def configure_node(testbed_instance, guid): element = testbed_instance._elements[guid] - if not guid in testbed_instance._add_route: - return - routes = testbed_instance._add_route[guid] - for route in routes: - (destination, netprefix, nexthop) = route - element.add_route(prefix = destination, prefix_len = netprefix, - nexthop = nexthop) + + # Do some validations + element.validate() ### Factory information ### @@ -104,67 +153,37 @@ connector_types = dict({ "max": -1, "min": 0 }), + "inet": dict({ + "help": "Connector from network interfaces to the internet", + "name": "inet", + "max": 1, + "min": 1 + }), "node": dict({ "help": "Connector to a Node", "name": "node", "max": 1, "min": 1 }), - "p2p": dict({ - "help": "Connector to a P2PInterface", - "name": "p2p", - "max": 1, - "min": 0 - }), - "fd": dict({ - "help": "Connector to a network interface that can receive a file descriptor", - "name": "fd", - "max": 1, - "min": 0 - }), - "switch": dict({ - "help": "Connector to a switch", - "name": "switch", - "max": 1, - "min": 0 - }) }) connections = [ - dict({ - "from": (TESTBED_ID, NODE, "devs"), - "to": (TESTBED_ID, P2PIFACE, "node"), - "code": None, - "can_cross": False - }), - dict({ - "from": (TESTBED_ID, NODE, "devs"), - "to": (TESTBED_ID, TAPIFACE, "node"), - "code": None, - "can_cross": False - }), dict({ "from": (TESTBED_ID, NODE, "devs"), "to": (TESTBED_ID, NODEIFACE, "node"), - "code": None, + "code": connect_node_iface_node, "can_cross": False }), dict({ - "from": (TESTBED_ID, P2PIFACE, "p2p"), - "to": (TESTBED_ID, P2PIFACE, "p2p"), - "code": None, + "from": (TESTBED_ID, NODE, "devs"), + "to": (TESTBED_ID, TUNIFACE, "node"), + "code": connect_tun_iface_node, "can_cross": False }), dict({ - "from": (TESTBED_ID, TAPIFACE, "fd"), - "to": (NS3_TESTBED_ID, FDNETDEV, "fd"), - "code": connect_fd_local, - "can_cross": True - }), - dict({ - "from": (TESTBED_ID, SWITCH, "devs"), - "to": (TESTBED_ID, NODEIFACE, "switch"), - "code": connect_switch, + "from": (TESTBED_ID, NODEIFACE, "inet"), + "to": (TESTBED_ID, INTERNET, "devs"), + "code": connect_node_iface_inet, "can_cross": False }), dict({ @@ -182,15 +201,87 @@ attributes = dict({ "type": Attribute.BOOL, "value": False, "flags": Attribute.DesignOnly, - "validation_function": validation.is_bool + "validation_function": validation.is_bool, }), - "lladdr": dict({ - "name": "lladdr", - "help": "Mac address", - "type": Attribute.STRING, + "hostname": dict({ + "name": "hosname", + "help": "Constrain hostname during resource discovery. May use wildcards.", + "type": Attribute.STRING, + "flags": Attribute.DesignOnly, + "validation_function": validation.is_string, + }), + "architecture": dict({ + "name": "architecture", + "help": "Constrain architexture during resource discovery.", + "type": Attribute.ENUM, + "flags": Attribute.DesignOnly, + "allowed": ["x86_64", + "i386"], + "validation_function": validation.is_enum, + }), + "operating_system": dict({ + "name": "operatingSystem", + "help": "Constrain operating system during resource discovery.", + "type": Attribute.ENUM, + "flags": Attribute.DesignOnly, + "allowed": ["f8", + "f12", + "f14", + "centos", + "other"], + "validation_function": validation.is_enum, + }), + "site": dict({ + "name": "site", + "help": "Constrain the PlanetLab site this node should reside on.", + "type": Attribute.ENUM, + "flags": Attribute.DesignOnly, + "allowed": ["PLE", + "PLC", + "PLJ"], + "validation_function": validation.is_enum, + }), + "emulation": dict({ + "name": "emulation", + "help": "Enable emulation on this node. Enables NetfilterRoutes, bridges, and a host of other functionality.", + "type": Attribute.BOOL, + "value": False, "flags": Attribute.DesignOnly, - "validation_function": validation.is_mac_address + "validation_function": validation.is_bool, }), + "min_reliability": dict({ + "name": "minReliability", + "help": "Constrain reliability while picking PlanetLab nodes. Specifies a lower acceptable bound.", + "type": Attribute.DOUBLE, + "range": (0,100), + "flags": Attribute.DesignOnly, + "validation_function": validation.is_double, + }), + "max_reliability": dict({ + "name": "maxReliability", + "help": "Constrain reliability while picking PlanetLab nodes. Specifies an upper acceptable bound.", + "type": Attribute.DOUBLE, + "range": (0,100), + "flags": Attribute.DesignOnly, + "validation_function": validation.is_double, + }), + "min_bandwidth": dict({ + "name": "minBandwidth", + "help": "Constrain available bandwidth while picking PlanetLab nodes. Specifies a lower acceptable bound.", + "type": Attribute.DOUBLE, + "range": (0,2**31), + "flags": Attribute.DesignOnly, + "validation_function": validation.is_double, + }), + "max_bandwidth": dict({ + "name": "maxBandwidth", + "help": "Constrain available bandwidth while picking PlanetLab nodes. Specifies an upper acceptable bound.", + "type": Attribute.DOUBLE, + "range": (0,2**31), + "flags": Attribute.DesignOnly, + "validation_function": validation.is_double, + }), + "up": dict({ "name": "up", "help": "Link up", @@ -198,6 +289,13 @@ attributes = dict({ "value": False, "validation_function": validation.is_bool }), + "primary": dict({ + "name": "primary", + "help": "This is the primary interface for the attached node", + "type": Attribute.BOOL, + "value": True, + "validation_function": validation.is_bool + }), "device_name": dict({ "name": "name", "help": "Device name", @@ -209,28 +307,23 @@ attributes = dict({ "name": "mtu", "help": "Maximum transmition unit for device", "type": Attribute.INTEGER, - "validation_function": validation.is_integer - }), - "broadcast": dict({ - "name": "broadcast", - "help": "Broadcast address", - "type": Attribute.STRING, - "validation_function": validation.is_string # TODO: should be is address! + "range": (0,1500), + "validation_function": validation.is_integer_range(0,1500) }), - "multicast": dict({ - "name": "multicast", - "help": "Multicast enabled", - "type": Attribute.BOOL, - "value": False, - "validation_function": validation.is_bool + "mask": dict({ + "name": "mask", + "help": "Network mask for the device (eg: 24 for /24 network)", + "type": Attribute.INTEGER, + "validation_function": validation.is_integer_range(8,24) }), - "arp": dict({ - "name": "arp", - "help": "ARP enabled", + "snat": dict({ + "name": "snat", + "help": "Enable SNAT (source NAT to the internet) no this device", "type": Attribute.BOOL, "value": False, "validation_function": validation.is_bool }), + "command": dict({ "name": "command", "help": "Command line string", @@ -265,64 +358,50 @@ traces = dict({ }) }) -create_order = [ NODE, P2PIFACE, NODEIFACE, TAPIFACE, SWITCH, - APPLICATION ] +create_order = [ NODE, NODEIFACE, TUNIFACE, APPLICATION ] -configure_order = [ P2PIFACE, NODEIFACE, TAPIFACE, SWITCH, NODE, - APPLICATION ] +configure_order = [ NODE, NODEIFACE, TUNIFACE, APPLICATION ] factories_info = dict({ NODE: dict({ - "allow_routes": True, - "help": "Emulated Node with virtualized network stack", + "allow_routes": False, + "help": "Virtualized Node (V-Server style)", "category": "topology", "create_function": create_node, "configure_function": configure_node, - "box_attributes": ["forward_X11"], + "box_attributes": [ + "forward_X11", + "hostname", + "architecture", + "operating_system", + "site", + "emulation", + "min_reliability", + "max_reliability", + "min_bandwidth", + "max_bandwidth", + ], "connector_types": ["devs", "apps"] }), - P2PIFACE: dict({ - "allow_addresses": True, - "help": "Point to point network interface", - "category": "devices", - "create_function": create_p2piface, - "configure_function": configure_device, - "box_attributes": ["lladdr", "up", "device_name", "mtu", - "multicast", "broadcast", "arp"], - "connector_types": ["node", "p2p"] - }), - TAPIFACE: dict({ - "allow_addresses": True, - "help": "Tap device network interface", - "category": "devices", - "create_function": create_tapiface, - "configure_function": configure_device, - "box_attributes": ["lladdr", "up", "device_name", "mtu", - "multicast", "broadcast", "arp"], - "connector_types": ["node", "fd"] - }), NODEIFACE: dict({ "allow_addresses": True, - "help": "Node network interface", + "help": "External network interface - they cannot be brought up or down, and they MUST be connected to the internet.", "category": "devices", "create_function": create_nodeiface, - "configure_function": configure_device, - "box_attributes": ["lladdr", "up", "device_name", "mtu", - "multicast", "broadcast", "arp"], - "connector_types": ["node", "switch"] + "configure_function": configure_nodeiface, + "box_attributes": [ ], + "connector_types": ["node", "inet"] }), - SWITCH: dict({ - "display_name": "Switch", - "help": "Switch interface", + TUNIFACE: dict({ + "allow_addresses": True, + "help": "Virtual TUN network interface", "category": "devices", - "create_function": create_switch, - "box_attributes": ["up", "device_name", "mtu", "multicast"], - #TODO: Add attribute ("Stp", help, type, value, range, allowed, readonly, validation_function), - #TODO: Add attribute ("ForwarddDelay", help, type, value, range, allowed, readonly, validation_function), - #TODO: Add attribute ("HelloTime", help, type, value, range, allowed, readonly, validation_function), - #TODO: Add attribute ("AgeingTime", help, type, value, range, allowed, readonly, validation_function), - #TODO: Add attribute ("MaxAge", help, type, value, range, allowed, readonly, validation_function) - "connector_types": ["devs"] + "create_function": create_tuniface, + "configure_function": configure_tuniface, + "box_attributes": [ + "up", "device_name", "mtu", "snat", + ], + "connector_types": ["node"] }), APPLICATION: dict({ "help": "Generic executable command line application", @@ -334,16 +413,22 @@ factories_info = dict({ "connector_types": ["node"], "traces": ["stdout", "stderr"] }), + INTERNET: dict({ + "help": "Internet routing", + "category": "topology", + "create_function": create_internet, + "connector_types": ["devs"], + }), }) testbed_attributes = dict({ - "enable_debug": dict({ - "name": "enableDebug", - "help": "Enable netns debug output", - "type": Attribute.BOOL, - "value": False, - "validation_function": validation.is_bool - }), + "slice": dict({ + "name": "slice", + "help": "The name of the PlanetLab slice to use", + "type": Attribute.STRING, + "flags": Attribute.DesignOnly | Attribute.HasNoDefaultValue, + "validation_function": validation.is_string + }), }) class VersionedMetadataInfo(metadata.VersionedMetadataInfo): diff --git a/src/nepi/testbeds/planetlab/node.py b/src/nepi/testbeds/planetlab/node.py new file mode 100644 index 00000000..e9a95579 --- /dev/null +++ b/src/nepi/testbeds/planetlab/node.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from constants import TESTBED_ID +import plcapi +import operator + +class Node(object): + BASEFILTERS = { + # Map Node attribute to plcapi filter name + 'hostname' : 'hostname', + } + + TAGFILTERS = { + # Map Node attribute to (, ) + # There are replacements that are applied with string formatting, + # so '%' has to be escaped as '%%'. + 'architecture' : ('arch','value'), + 'operating_system' : ('fcdistro','value'), + 'pl_distro' : ('pldistro','value'), + 'min_reliability' : ('reliability%(timeframe)s', ']value'), + 'max_reliability' : ('reliability%(timeframe)s', '[value'), + 'min_bandwidth' : ('bw%(timeframe)s', ']value'), + 'max_bandwidth' : ('bw%(timeframe)s', '[value'), + } + + def __init__(self, api=None): + if not api: + api = plcapi.PLCAPI() + self._api = api + + # Attributes + self.hostname = None + self.architecture = None + self.operating_system = None + self.pl_distro = None + self.site = None + self.emulation = None + self.min_reliability = None + self.max_reliability = None + self.min_bandwidth = None + self.max_bandwidth = None + self.min_num_external_ifaces = None + self.max_num_external_ifaces = None + self.timeframe = 'm' + + # Those are filled when an actual node is allocated + self._node_id = None + + def build_filters(self, target_filters, filter_map): + for attr, tag in filter_map.iteritems(): + value = getattr(self, attr, None) + if value is not None: + target_filters[tag] = value + return target_filters + + @property + def applicable_filters(self): + has = lambda att : getattr(self,att,None) is not None + return ( + filter(has, self.BASEFILTERS.iterkeys()) + + filter(has, self.TAGFILTERS.iterkeys()) + ) + + def find_candidates(self): + fields = ('node_id',) + replacements = {'timeframe':self.timeframe} + + # get initial candidates (no tag filters) + basefilters = self.build_filters({}, self.BASEFILTERS) + + # keyword-only "pseudofilters" + extra = {} + if self.site: + extra['peer'] = self.site + + candidates = set(map(operator.itemgetter('node_id'), + self._api.GetNodes(filters=basefilters, fields=fields, **extra))) + + # filter by tag, one tag at a time + applicable = self.applicable_filters + for tagfilter in self.TAGFILTERS.iteritems(): + attr, (tagname, expr) = tagfilter + + # don't bother if there's no filter defined + if attr in applicable: + tagfilter = basefilters.copy() + tagfilter['tagname'] = tagname % replacements + tagfilter[expr % replacements] = getattr(self,attr) + tagfilter['node_id'] = list(candidates) + + candidates &= set(map(operator.itemgetter('node_id'), + self._api.GetNodeTags(filters=tagfilter, fields=fields))) + + # filter by iface count + if self.min_num_external_ifaces is not None or self.max_num_external_ifaces is not None: + # fetch interfaces for all, in one go + filters = basefilters.copy() + filters['node_id'] = list(candidates) + ifaces = dict(map(operator.itemgetter('node_id','interface_ids'), + self._api.GetNodes(filters=basefilters, fields=('node_id','interface_ids')) )) + + # filter candidates by interface count + if self.min_num_external_ifaces is not None and self.max_num_external_ifaces is not None: + predicate = ( lambda node_id : + self.min_num_external_ifaces <= len(ifaces.get(node_id,())) <= self.max_num_external_ifaces ) + elif self.min_num_external_ifaces is not None: + predicate = ( lambda node_id : + self.min_num_external_ifaces <= len(ifaces.get(node_id,())) ) + else: + predicate = ( lambda node_id : + len(ifaces.get(node_id,())) <= self.max_num_external_ifaces ) + + candidates = set(filter(predicate, candidates)) + + return candidates + + def validate(self): + pass + diff --git a/src/nepi/testbeds/planetlab/plcapi.py b/src/nepi/testbeds/planetlab/plcapi.py index 4521e8cf..ab8fd2ec 100644 --- a/src/nepi/testbeds/planetlab/plcapi.py +++ b/src/nepi/testbeds/planetlab/plcapi.py @@ -205,6 +205,29 @@ class PLCAPI(object): filters.update(kw) return self.api.GetNodes(self.auth, filters, *fieldstuple) + def GetNodeTags(self, nodeTagId=None, fields=None, **kw): + if fields is not None: + fieldstuple = (fields,) + else: + fieldstuple = () + if nodeTagId is not None: + return self.api.GetNodeTags(self.auth, nodeTagId, *fieldstuple) + else: + filters = kw.pop('filters',{}) + filters.update(kw) + return self.api.GetNodeTags(self.auth, filters, *fieldstuple) + - - + def GetInterfaces(self, interfaceIdOrIp=None, fields=None, **kw): + if fields is not None: + fieldstuple = (fields,) + else: + fieldstuple = () + if interfaceIdOrIp is not None: + return self.api.GetNodeTags(self.auth, interfaceIdOrIp, *fieldstuple) + else: + filters = kw.pop('filters',{}) + filters.update(kw) + return self.api.GetNodeTags(self.auth, filters, *fieldstuple) + + diff --git a/src/nepi/util/ipaddr.py b/src/nepi/util/ipaddr.py new file mode 100644 index 00000000..a88d1b75 --- /dev/null +++ b/src/nepi/util/ipaddr.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +def ipv4_dot2mask(mask): + mask = mask.split('.',4) # a.b.c.d -> [a,b,c,d] + mask = map(int,mask) # to ints + + n = 0 + while mask and mask[0] == 0xff: + n += 8 + del mask[0] + + if mask: + mask = mask[0] + while mask: + n += 1 + mask = (mask << 1) & 0xff + + return n + diff --git a/src/nepi/util/settools/__init__.py b/src/nepi/util/settools/__init__.py new file mode 100644 index 00000000..571fd591 --- /dev/null +++ b/src/nepi/util/settools/__init__.py @@ -0,0 +1,2 @@ +from setclusters import * + diff --git a/src/nepi/util/settools/setclusters.py b/src/nepi/util/settools/setclusters.py new file mode 100644 index 00000000..8a74d695 --- /dev/null +++ b/src/nepi/util/settools/setclusters.py @@ -0,0 +1,82 @@ +import itertools +import collections + +def disjoint_sets(*sets): + """ + Given a series of sets S1..SN, computes disjoint clusters C1..CM + such that C1=U Sc1..Sc1', C2=U Sc2..Sc2', ... CM=ScM..ScM' + and any component of Ci is disjoint against any component of Cj + for i!=j + + The result is given in terms of the component sets, so C1 is given + as the sequence Sc1..Sc1', etc. + + Example: + + >>> disjoint_sets( set([1,2,4]), set([2,3,4,5]), set([4,5]), set([6,7]), set([7,8]) ) + [[set([1, 2, 4]), set([4, 5]), set([2, 3, 4, 5])], [set([6, 7]), set([8, 7])]] + + >>> disjoint_sets( set([1]), set([2]), set([3]) ) + [[set([1])], [set([2])], [set([3])]] + + """ + + # Pseudo: + # + # While progress is made: + # - Join intersecting clusters + # - Track their components + # - Replace sets with the new clusters, restart + cluster_components = [ [s] for s in sets ] + clusters = [s.copy() for s in sets] + + changed = True + while changed: + changed = False + + for i,s in enumerate(clusters): + for j in xrange(len(clusters)-1,i,-1): + cluster = clusters[j] + if cluster & s: + changed = True + cluster.update(s) + cluster_components[i].extend(cluster_components[j]) + del cluster_components[j] + del clusters[j] + + return cluster_components + +def disjoint_partition(*sets): + """ + Given a series of sets S1..SN, computes a disjoint partition of + the population maintaining set boundaries. + + That is, it computes a disjoint partition P1..PM where + Pn is the equivalence relation given by + + R <==> a in Sn <--> b in Sn for all n + + NOTE: Given the current implementation, the contents of the + sets must be hashable. + + Examples: + + >>> disjoint_partition( set([1,2,4]), set([2,3,4,5]), set([4,5]), set([6,7]), set([7,8]) ) + [set([2]), set([5]), set([1]), set([3]), set([4]), set([6]), set([8]), set([7])] + + >>> disjoint_partition( set([1,2,4]), set([2,3,4,5,10]), set([4,5]), set([6,7]), set([7,8]) ) + [set([2]), set([5]), set([1]), set([10, 3]), set([4]), set([6]), set([8]), set([7])] + + """ + reverse_items = collections.defaultdict(list) + + for i,s in enumerate(sets): + for item in s: + reverse_items[item].append(i) + + partitions = collections.defaultdict(set) + for item, cats in reverse_items.iteritems(): + partitions[tuple(cats)].add(item) + + return partitions.values() + diff --git a/src/nepi/util/validation.py b/src/nepi/util/validation.py index 13a067a9..4d4a4d94 100644 --- a/src/nepi/util/validation.py +++ b/src/nepi/util/validation.py @@ -13,8 +13,26 @@ def is_bool(attribute, value): def is_double(attribute, value): return isinstance(value, float) -def is_integer(attribute, value): - return isinstance(value, int) +def is_integer(attribute, value, min=None, max=None): + if not isinstance(value, int): + return False + if min is not None and value < min: + return False + if max is not None and value > max: + return False + return True + +def is_integer_range(min=None, max=None): + def is_integer_range(attribute, value): + if not isinstance(value, int): + return False + if min is not None and value < min: + return False + if max is not None and value > max: + return False + return True + return is_integer_range + def is_string(attribute, value): return isinstance(value, str)