X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=src%2Fnepi%2Ftestbeds%2Fplanetlab%2Fmetadata_v01.py;h=6fcc07d2a1e8a519d7e906d4f56a806d4c5e6bd5;hb=3f1dc42591981522c72c19ad5a0ea5cdc518d8b9;hp=b1d51c6eab6635c1033c77ff354a21088eb87faa;hpb=0b31108e8d6240949592c9e344d0c5cee19f8f35;p=nepi.git diff --git a/src/nepi/testbeds/planetlab/metadata_v01.py b/src/nepi/testbeds/planetlab/metadata_v01.py index b1d51c6e..6fcc07d2 100644 --- a/src/nepi/testbeds/planetlab/metadata_v01.py +++ b/src/nepi/testbeds/planetlab/metadata_v01.py @@ -7,8 +7,9 @@ from constants import TESTBED_ID from nepi.core import metadata from nepi.core.attributes import Attribute from nepi.util import validation -from nepi.util.constants import STATUS_NOT_STARTED, STATUS_RUNNING, \ - STATUS_FINISHED +from nepi.util.constants import ApplicationStatus as AS, \ + FactoryCategories as FC, \ + ATTR_NEPI_TESTBED_ENVIRONMENT_SETUP import functools import os @@ -17,7 +18,11 @@ import os.path NODE = "Node" NODEIFACE = "NodeInterface" TUNIFACE = "TunInterface" +TAPIFACE = "TapInterface" APPLICATION = "Application" +DEPENDENCY = "Dependency" +NEPIDEPENDENCY = "NepiDependency" +NS3DEPENDENCY = "NS3Dependency" INTERNET = "Internet" NETPIPE = "NetPipe" @@ -39,13 +44,13 @@ def is_addrlist(attribute, value): if '/' in component: addr, mask = component.split('/',1) else: - addr, mask = component, 32 + addr, mask = component, '32' if mask is not None and not (mask and mask.isdigit()): # No empty or nonnumeric masks return False - if not validation.is_ip4_address(attribute, value): + if not validation.is_ip4_address(attribute, addr): # Address part must be ipv4 return False @@ -76,31 +81,80 @@ def is_portlist(attribute, value): ### Connection functions #### -def connect_node_iface_node(testbed_instance, node, iface): +def connect_node_iface_node(testbed_instance, node_guid, iface_guid): + node = testbed_instance._elements[node_guid] + iface = testbed_instance._elements[iface_guid] iface.node = node -def connect_node_iface_inet(testbed_instance, iface, inet): +def connect_node_iface_inet(testbed_instance, iface_guid, inet_guid): + iface = testbed_instance._elements[iface_guid] iface.has_internet = True -def connect_tun_iface_node(testbed_instance, node, iface): +def connect_tun_iface_node(testbed_instance, node_guid, iface_guid): + node = testbed_instance._elements[node_guid] + iface = testbed_instance._elements[iface_guid] if not node.emulation: raise RuntimeError, "Use of TUN interfaces requires emulation" iface.node = node node.required_vsys.update(('fd_tuntap', 'vif_up')) + node.required_packages.update(('python', 'python-crypto', 'python-setuptools', 'gcc')) -def connect_tun_iface_peer(proto, testbed_instance, iface, peer_iface): +def connect_tun_iface_peer(proto, testbed_instance, iface_guid, peer_iface_guid): + iface = testbed_instance._elements[iface_guid] + peer_iface = testbed_instance._elements[peer_iface_guid] iface.peer_iface = peer_iface iface.peer_proto = \ iface.tun_proto = proto + iface.tun_key = peer_iface.tun_key -def connect_app(testbed_instance, node, app): +def crossconnect_tun_iface_peer_init(proto, testbed_instance, iface_guid, peer_iface_data): + iface = testbed_instance._elements[iface_guid] + iface.peer_iface = None + iface.peer_addr = peer_iface_data.get("tun_addr") + iface.peer_proto = peer_iface_data.get("tun_proto") or proto + iface.peer_port = peer_iface_data.get("tun_port") + iface.tun_key = min(iface.tun_key, peer_iface_data.get("tun_key")) + iface.tun_proto = proto + + preconfigure_tuniface(testbed_instance, iface_guid) + +def crossconnect_tun_iface_peer_compl(proto, testbed_instance, iface_guid, peer_iface_data): + # refresh (refreshable) attributes for second-phase + iface = testbed_instance._elements[iface_guid] + iface.peer_addr = peer_iface_data.get("tun_addr") + iface.peer_proto = peer_iface_data.get("tun_proto") or proto + iface.peer_port = peer_iface_data.get("tun_port") + + postconfigure_tuniface(testbed_instance, iface_guid) + +def crossconnect_tun_iface_peer_both(proto, testbed_instance, iface_guid, peer_iface_data): + crossconnect_tun_iface_peer_init(proto, testbed_instance, iface_guid, peer_iface_data) + crossconnect_tun_iface_peer_compl(proto, testbed_instance, iface_guid, peer_iface_data) + +def connect_dep(testbed_instance, node_guid, app_guid): + node = testbed_instance._elements[node_guid] + app = testbed_instance._elements[app_guid] app.node = node if app.depends: node.required_packages.update(set( app.depends.split() )) + + if app.add_to_path: + if app.home_path and app.home_path not in node.pythonpath: + node.pythonpath.append(app.home_path) + + if app.env: + for envkey, envval in app.env.iteritems(): + envval = app._replace_paths(envval) + node.env[envkey].append(envval) + + if app.rpmFusion: + node.rpmFusion = True -def connect_node_netpipe(testbed_instance, node, netpipe): +def connect_node_netpipe(testbed_instance, node_guid, netpipe_guid): + node = testbed_instance._elements[node_guid] + netpipe = testbed_instance._elements[netpipe_guid] if not node.emulation: raise RuntimeError, "Use of NetPipes requires emulation" netpipe.node = node @@ -116,12 +170,17 @@ def create_node(testbed_instance, guid): # add constraint on number of (real) interfaces # by counting connected devices - dev_guids = testbed_instance.get_connected(guid, "node", "devs") + dev_guids = testbed_instance.get_connected(guid, "devs", "node") num_open_ifaces = sum( # count True values NODEIFACE == testbed_instance._get_factory_id(guid) for guid in dev_guids ) element.min_num_external_ifaces = num_open_ifaces + # require vroute vsys if we have routes to set up + routes = testbed_instance._add_route.get(guid) + if routes: + element.required_vsys.add("vroute") + testbed_instance.elements[guid] = element def create_nodeiface(testbed_instance, guid): @@ -132,11 +191,65 @@ def create_nodeiface(testbed_instance, guid): def create_tuniface(testbed_instance, guid): parameters = testbed_instance._get_parameters(guid) element = testbed_instance._make_tun_iface(parameters) + + # Set custom addresses, if there are any already + # Setting this early helps set up P2P links + if guid in testbed_instance._add_address and not (element.address or element.netmask or element.netprefix): + addresses = testbed_instance._add_address[guid] + for address in addresses: + (address, netprefix, broadcast) = address + element.add_address(address, netprefix, broadcast) + + testbed_instance.elements[guid] = element + +def create_tapiface(testbed_instance, guid): + parameters = testbed_instance._get_parameters(guid) + element = testbed_instance._make_tap_iface(parameters) + + # Set custom addresses, if there are any already + # Setting this early helps set up P2P links + if guid in testbed_instance._add_address and not (element.address or element.netmask or element.netprefix): + addresses = testbed_instance._add_address[guid] + for address in addresses: + (address, netprefix, broadcast) = address + element.add_address(address, netprefix, broadcast) + testbed_instance.elements[guid] = element def create_application(testbed_instance, guid): parameters = testbed_instance._get_parameters(guid) element = testbed_instance._make_application(parameters) + + # Just inject configuration stuff + element.home_path = "nepi-app-%s" % (guid,) + + testbed_instance.elements[guid] = element + +def create_dependency(testbed_instance, guid): + parameters = testbed_instance._get_parameters(guid) + element = testbed_instance._make_dependency(parameters) + + # Just inject configuration stuff + element.home_path = "nepi-dep-%s" % (guid,) + + testbed_instance.elements[guid] = element + +def create_nepi_dependency(testbed_instance, guid): + parameters = testbed_instance._get_parameters(guid) + element = testbed_instance._make_nepi_dependency(parameters) + + # Just inject configuration stuff + element.home_path = "nepi-nepi-%s" % (guid,) + + testbed_instance.elements[guid] = element + +def create_ns3_dependency(testbed_instance, guid): + parameters = testbed_instance._get_parameters(guid) + element = testbed_instance._make_ns3_dependency(parameters) + + # Just inject configuration stuff + element.home_path = "nepi-ns3-%s" % (guid,) + testbed_instance.elements[guid] = element def create_internet(testbed_instance, guid): @@ -170,7 +283,7 @@ def stop_application(testbed_instance, guid): def status_application(testbed_instance, guid): if guid not in testbed_instance.elements.keys(): - return STATUS_NOT_STARTED + return AS.STATUS_NOT_STARTED app = testbed_instance.elements[guid] return app.status() @@ -200,8 +313,8 @@ def configure_nodeiface(testbed_instance, guid): def preconfigure_tuniface(testbed_instance, guid): element = testbed_instance._elements[guid] - # Set custom addresses if any - if guid in testbed_instance._add_address: + # Set custom addresses if any, and if not set already + if guid in testbed_instance._add_address and not (element.address or element.netmask or element.netprefix): addresses = testbed_instance._add_address[guid] for address in addresses: (address, netprefix, broadcast) = address @@ -214,8 +327,9 @@ def preconfigure_tuniface(testbed_instance, guid): break # Set standard TUN attributes - element.tun_addr = element.external_iface.address - element.tun_port = 15000 + int(guid) + if (not element.tun_addr or not element.tun_port) and element.external_iface: + element.tun_addr = element.external_iface.address + element.tun_port = 15000 + int(guid) # Set enabled traces traces = testbed_instance._get_traces(guid) @@ -225,9 +339,23 @@ def preconfigure_tuniface(testbed_instance, guid): element.validate() # First-phase setup - element.prepare( - 'tun-%s' % (guid,), - id(element) < id(element.peer_iface) ) + if element.peer_proto: + if element.peer_iface and isinstance(element.peer_iface, testbed_instance._interfaces.TunIface): + # intra tun + listening = id(element) < id(element.peer_iface) + else: + # cross tun + if not element.tun_addr or not element.tun_port: + listening = True + elif not element.peer_addr or not element.peer_port: + listening = True + else: + # both have addresses... + # ...the one with the lesser address listens + listening = element.tun_addr < element.peer_addr + element.prepare( + 'tun-%s' % (guid,), + listening) def postconfigure_tuniface(testbed_instance, guid): element = testbed_instance._elements[guid] @@ -235,6 +363,12 @@ def postconfigure_tuniface(testbed_instance, guid): # Second-phase setup element.setup() +def wait_tuniface(testbed_instance, guid): + element = testbed_instance._elements[guid] + + # Second-phase setup + element.async_launch_wait() + def configure_node(testbed_instance, guid): node = testbed_instance._elements[guid] @@ -247,24 +381,25 @@ def configure_node(testbed_instance, guid): # Do some validations node.validate() - # recently provisioned nodes may not be up yet - sleeptime = 1.0 - while not node.is_alive(): - time.sleep(sleeptime) - sleeptime = min(30.0, sleeptime*1.5) - # this will be done in parallel in all nodes # this call only spawns the process node.install_dependencies() +def configure_node_routes(testbed_instance, guid): + node = testbed_instance._elements[guid] + routes = testbed_instance._add_route.get(guid) + + if routes: + devs = [ dev + for dev_guid in testbed_instance.get_connected(guid, "devs", "node") + for dev in ( testbed_instance._elements.get(dev_guid) ,) + if dev and isinstance(dev, testbed_instance._interfaces.TunIface) ] + + node.configure_routes(routes, devs) + def configure_application(testbed_instance, guid): app = testbed_instance._elements[guid] - # Just inject configuration stuff - app.home_path = "nepi-app-%s" % (guid,) - app.ident_path = testbed_instance.sliceSSHKey - app.slicename = testbed_instance.slicename - # Do some validations app.validate() @@ -272,7 +407,19 @@ def configure_application(testbed_instance, guid): app.node.wait_dependencies() # Install stuff - app.setup() + app.async_setup() + +def configure_dependency(testbed_instance, guid): + dep = testbed_instance._elements[guid] + + # Do some validations + dep.validate() + + # Wait for dependencies + dep.node.wait_dependencies() + + # Install stuff + dep.async_setup() def configure_netpipe(testbed_instance, guid): netpipe = testbed_instance._elements[guid] @@ -301,6 +448,13 @@ connector_types = dict({ "max": -1, "min": 0 }), + "deps": dict({ + "help": "Connector from node to application dependencies " + "(packages and applications that need to be installed)", + "name": "deps", + "max": -1, + "min": 0 + }), "inet": dict({ "help": "Connector from network interfaces to the internet", "name": "inet", @@ -332,6 +486,12 @@ connector_types = dict({ "max": 1, "min": 0 }), + "fd->": dict({ + "help": "TUN device file descriptor provider", + "name": "fd->", + "max": 1, + "min": 0 + }), }) connections = [ @@ -347,6 +507,12 @@ connections = [ "init_code": connect_tun_iface_node, "can_cross": False }), + dict({ + "from": (TESTBED_ID, NODE, "devs"), + "to": (TESTBED_ID, TAPIFACE, "node"), + "init_code": connect_tun_iface_node, + "can_cross": False + }), dict({ "from": (TESTBED_ID, NODEIFACE, "inet"), "to": (TESTBED_ID, INTERNET, "devs"), @@ -356,7 +522,25 @@ connections = [ dict({ "from": (TESTBED_ID, NODE, "apps"), "to": (TESTBED_ID, APPLICATION, "node"), - "init_code": connect_app, + "init_code": connect_dep, + "can_cross": False + }), + dict({ + "from": (TESTBED_ID, NODE, "deps"), + "to": (TESTBED_ID, DEPENDENCY, "node"), + "init_code": connect_dep, + "can_cross": False + }), + dict({ + "from": (TESTBED_ID, NODE, "deps"), + "to": (TESTBED_ID, NEPIDEPENDENCY, "node"), + "init_code": connect_dep, + "can_cross": False + }), + dict({ + "from": (TESTBED_ID, NODE, "deps"), + "to": (TESTBED_ID, NS3DEPENDENCY, "node"), + "init_code": connect_dep, "can_cross": False }), dict({ @@ -377,6 +561,58 @@ connections = [ "init_code": functools.partial(connect_tun_iface_peer,"udp"), "can_cross": False }), + dict({ + "from": (TESTBED_ID, TAPIFACE, "tcp"), + "to": (TESTBED_ID, TAPIFACE, "tcp"), + "init_code": functools.partial(connect_tun_iface_peer,"tcp"), + "can_cross": False + }), + dict({ + "from": (TESTBED_ID, TAPIFACE, "udp"), + "to": (TESTBED_ID, TAPIFACE, "udp"), + "init_code": functools.partial(connect_tun_iface_peer,"udp"), + "can_cross": False + }), + dict({ + "from": (TESTBED_ID, TUNIFACE, "tcp"), + "to": (None, None, "tcp"), + "init_code": functools.partial(crossconnect_tun_iface_peer_init,"tcp"), + "compl_code": functools.partial(crossconnect_tun_iface_peer_compl,"tcp"), + "can_cross": True + }), + dict({ + "from": (TESTBED_ID, TUNIFACE, "udp"), + "to": (None, None, "udp"), + "init_code": functools.partial(crossconnect_tun_iface_peer_init,"udp"), + "compl_code": functools.partial(crossconnect_tun_iface_peer_compl,"udp"), + "can_cross": True + }), + dict({ + "from": (TESTBED_ID, TUNIFACE, "fd->"), + "to": (None, None, "->fd"), + "compl_code": functools.partial(crossconnect_tun_iface_peer_both,"fd"), + "can_cross": True + }), + dict({ + "from": (TESTBED_ID, TAPIFACE, "tcp"), + "to": (None, None, "tcp"), + "init_code": functools.partial(crossconnect_tun_iface_peer_init,"tcp"), + "compl_code": functools.partial(crossconnect_tun_iface_peer_compl,"tcp"), + "can_cross": True + }), + dict({ + "from": (TESTBED_ID, TAPIFACE, "udp"), + "to": (None, None, "udp"), + "init_code": functools.partial(crossconnect_tun_iface_peer_init,"udp"), + "compl_code": functools.partial(crossconnect_tun_iface_peer_compl,"udp"), + "can_cross": True + }), + dict({ + "from": (TESTBED_ID, TAPIFACE, "fd->"), + "to": (None, None, "->fd"), + "compl_code": functools.partial(crossconnect_tun_iface_peer_both,"fd"), + "can_cross": True + }), ] attributes = dict({ @@ -508,6 +744,14 @@ attributes = dict({ "value": False, "validation_function": validation.is_bool }), + "pointopoint": dict({ + "name": "pointopoint", + "help": "If the interface is a P2P link, the remote endpoint's IP " + "should be set on this attribute.", + "type": Attribute.STRING, + "flags": Attribute.DesignOnly, + "validation_function": validation.is_string + }), "txqueuelen": dict({ "name": "mask", "help": "Transmission queue length (in packets)", @@ -554,6 +798,14 @@ attributes = dict({ "flags": Attribute.DesignOnly, "validation_function": validation.is_string }), + "rpm-fusion": dict({ + "name": "rpmFusion", + "help": "True if required packages can be found in the RpmFusion repository", + "type": Attribute.BOOL, + "flags": Attribute.DesignOnly, + "value": False, + "validation_function": validation.is_bool + }), "sources": dict({ "name": "sources", "help": "Space-separated list of regular files to be deployed in the working path prior to building. " @@ -684,17 +936,21 @@ traces = dict({ }), }) -create_order = [ INTERNET, NODE, NODEIFACE, TUNIFACE, NETPIPE, APPLICATION ] +create_order = [ INTERNET, NODE, NODEIFACE, TAPIFACE, TUNIFACE, NETPIPE, NEPIDEPENDENCY, NS3DEPENDENCY, DEPENDENCY, APPLICATION ] -configure_order = [ INTERNET, NODE, NODEIFACE, TUNIFACE, NETPIPE, APPLICATION ] +configure_order = [ INTERNET, NODE, NODEIFACE, TAPIFACE, TUNIFACE, NETPIPE, NEPIDEPENDENCY, NS3DEPENDENCY, DEPENDENCY, APPLICATION ] + +# Start (and prestart) node after ifaces, because the node needs the ifaces in order to set up routes +start_order = [ INTERNET, NODEIFACE, TAPIFACE, TUNIFACE, NODE, NETPIPE, NEPIDEPENDENCY, NS3DEPENDENCY, DEPENDENCY, APPLICATION ] factories_info = dict({ NODE: dict({ - "allow_routes": False, + "allow_routes": True, "help": "Virtualized Node (V-Server style)", - "category": "topology", + "category": FC.CATEGORY_NODES, "create_function": create_node, "preconfigure_function": configure_node, + "prestart_function": configure_node_routes, "box_attributes": [ "forward_X11", "hostname", @@ -706,13 +962,16 @@ factories_info = dict({ "max_reliability", "min_bandwidth", "max_bandwidth", + + # NEPI-in-NEPI attributes + ATTR_NEPI_TESTBED_ENVIRONMENT_SETUP, ], - "connector_types": ["devs", "apps", "pipes"] + "connector_types": ["devs", "apps", "pipes", "deps"] }), NODEIFACE: dict({ "has_addresses": True, "help": "External network interface - they cannot be brought up or down, and they MUST be connected to the internet.", - "category": "devices", + "category": FC.CATEGORY_DEVICES, "create_function": create_nodeiface, "preconfigure_function": configure_nodeiface, "box_attributes": [ ], @@ -720,21 +979,39 @@ factories_info = dict({ }), TUNIFACE: dict({ "allow_addresses": True, - "help": "Virtual TUN network interface", - "category": "devices", + "help": "Virtual TUN network interface (layer 3)", + "category": FC.CATEGORY_DEVICES, "create_function": create_tuniface, "preconfigure_function": preconfigure_tuniface, "configure_function": postconfigure_tuniface, + "prestart_function": wait_tuniface, "box_attributes": [ - "up", "device_name", "mtu", "snat", - "txqueuelen" + "up", "device_name", "mtu", "snat", "pointopoint", + "txqueuelen", + "tun_proto", "tun_addr", "tun_port", "tun_key" ], "traces": ["packets"], - "connector_types": ["node","udp","tcp"] + "connector_types": ["node","udp","tcp","fd->"] + }), + TAPIFACE: dict({ + "allow_addresses": True, + "help": "Virtual TAP network interface (layer 2)", + "category": FC.CATEGORY_DEVICES, + "create_function": create_tapiface, + "preconfigure_function": preconfigure_tuniface, + "configure_function": postconfigure_tuniface, + "prestart_function": wait_tuniface, + "box_attributes": [ + "up", "device_name", "mtu", "snat", "pointopoint", + "txqueuelen", + "tun_proto", "tun_addr", "tun_port", "tun_key" + ], + "traces": ["packets"], + "connector_types": ["node","udp","tcp","fd->"] }), APPLICATION: dict({ "help": "Generic executable command line application", - "category": "applications", + "category": FC.CATEGORY_APPLICATIONS, "create_function": create_application, "start_function": start_application, "status_function": status_application, @@ -742,19 +1019,47 @@ factories_info = dict({ "configure_function": configure_application, "box_attributes": ["command", "sudo", "stdin", "depends", "build-depends", "build", "install", - "sources" ], + "sources", "rpm-fusion" ], "connector_types": ["node"], - "traces": ["stdout", "stderr"] + "traces": ["stdout", "stderr", "buildlog"] + }), + DEPENDENCY: dict({ + "help": "Requirement for package or application to be installed on some node", + "category": FC.CATEGORY_APPLICATIONS, + "create_function": create_dependency, + "preconfigure_function": configure_dependency, + "box_attributes": ["depends", "build-depends", "build", "install", + "sources", "rpm-fusion" ], + "connector_types": ["node"], + "traces": ["buildlog"] + }), + NEPIDEPENDENCY: dict({ + "help": "Requirement for NEPI inside NEPI - required to run testbed instances inside a node", + "category": FC.CATEGORY_APPLICATIONS, + "create_function": create_nepi_dependency, + "preconfigure_function": configure_dependency, + "box_attributes": [ ], + "connector_types": ["node"], + "traces": ["buildlog"] + }), + NS3DEPENDENCY: dict({ + "help": "Requirement for NS3 inside NEPI - required to run NS3 testbed instances inside a node. It also needs NepiDependency.", + "category": FC.CATEGORY_APPLICATIONS, + "create_function": create_ns3_dependency, + "preconfigure_function": configure_dependency, + "box_attributes": [ ], + "connector_types": ["node"], + "traces": ["buildlog"] }), INTERNET: dict({ "help": "Internet routing", - "category": "topology", + "category": FC.CATEGORY_CHANNELS, "create_function": create_internet, "connector_types": ["devs"], }), NETPIPE: dict({ "help": "Link emulation", - "category": "topology", + "category": FC.CATEGORY_CHANNELS, "create_function": create_netpipe, "configure_function": configure_netpipe, "box_attributes": ["netpipe_mode", @@ -788,6 +1093,22 @@ testbed_attributes = dict({ "flags": Attribute.DesignOnly | Attribute.HasNoDefaultValue, "validation_function": validation.is_string }), + "plc_host": dict({ + "name": "plcHost", + "help": "The PlanetLab PLC API host", + "type": Attribute.STRING, + "value": "www.planet-lab.eu", + "flags": Attribute.DesignOnly, + "validation_function": validation.is_string + }), + "plc_url": dict({ + "name": "plcUrl", + "help": "The PlanetLab PLC API url pattern - %(hostname)s is replaced by plcHost.", + "type": Attribute.STRING, + "value": "https://%(hostname)s:443/PLCAPI/", + "flags": Attribute.DesignOnly, + "validation_function": validation.is_string + }), "slice_ssh_key": dict({ "name": "sliceSSHKey", "help": "The controller-local path to the slice user's ssh private key. " @@ -826,6 +1147,14 @@ class VersionedMetadataInfo(metadata.VersionedMetadataInfo): def configure_order(self): return configure_order + @property + def prestart_order(self): + return start_order + + @property + def start_order(self): + return start_order + @property def factories_info(self): return factories_info