From: Claudio-Daniel Freire Date: Mon, 2 May 2011 15:24:00 +0000 (+0200) Subject: - Merge with head X-Git-Tag: nepi_v2~86 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=8668fec103ad103cbf19923807a4141741b3620f;p=nepi.git - Merge with head - Add NS3Dependency object to PlanetLab, which makes possible the instantiation of NS3 TestbedController s. - Add support for environment modifiers, required by NS3Dependency - Better error logging (buildlog, installlog) for PlanetLab Dependency objects --- 8668fec103ad103cbf19923807a4141741b3620f diff --cc src/nepi/testbeds/ns3/execute.py index 5befa69c,5befa69c..2fca7f92 --- a/src/nepi/testbeds/ns3/execute.py +++ b/src/nepi/testbeds/ns3/execute.py @@@ -171,8 -171,8 +171,20 @@@ class TestbedController(testbed_impl.Te if bindings: path = [ bindings ] + path -- module = imp.find_module ('ns3', path) -- mod = imp.load_module ('ns3', *module) ++ try: ++ module = imp.find_module ('ns3', path) ++ mod = imp.load_module ('ns3', *module) ++ except ImportError: ++ # In some environments, ns3 per-se does not exist, ++ # only the low-level _ns3 ++ module = imp.find_module ('_ns3', path) ++ mod = imp.load_module ('_ns3', *module) ++ sys.modules["ns3"] = mod # install it as ns3 too ++ ++ # When using _ns3, we have to make sure we destroy ++ # the simulator when the process finishes ++ import atexit ++ atexit.register(mod.Simulator.Destroy) if simu_impl_type: value = mod.StringValue(simu_impl_type) diff --cc src/nepi/testbeds/planetlab/application.py index cbde9d29,cbde9d29..d550908e --- a/src/nepi/testbeds/planetlab/application.py +++ b/src/nepi/testbeds/planetlab/application.py @@@ -41,6 -41,6 +41,7 @@@ class Dependency(object) self.depends = None self.buildDepends = None self.sources = None ++ self.env = {} self.stdin = None self.stdout = None @@@ -223,7 -223,7 +224,7 @@@ if self.build: # Build sources (out,err),proc = server.popen_ssh_command( -- "cd %(home)s && mkdir -p build && cd build && %(command)s" % { ++ "cd %(home)s && mkdir -p build && cd build && ( %(command)s ) > ${HOME}/%(home)s/buildlog 2>&1 || ( tail ${HOME}/%(home)s/buildlog >&2 && false )" % { 'command' : self._replace_paths(self.build), 'home' : server.shell_escape(self.home_path), }, @@@ -258,7 -258,7 +259,7 @@@ if self.install: # Install application (out,err),proc = server.popen_ssh_command( -- "cd %(home)s && cd build && %(command)s" % { ++ "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/installlog >&2 && false )" % { 'command' : self._replace_paths(self.install), 'home' : server.shell_escape(self.home_path), }, @@@ -321,6 -321,6 +322,10 @@@ class Application(Dependency) command.write('export PATH=$PATH:%s\n' % ( ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath]) )) ++ if self.node.env: ++ for envkey, envvals in self.node.env.iteritems(): ++ for envval in envvals: ++ command.write('export %s=%s\n' % (envkey, envval)) command.write(self.command) command.seek(0) @@@ -421,13 -421,13 +426,8 @@@ class NepiDependency(Dependency): """ -- A Dependency is in every respect like an application. -- -- It depends on some packages, it may require building binaries, it must deploy -- them... -- -- But it has no command. Dependencies aren't ever started, or stopped, and have -- no status. ++ This dependency adds nepi itself to the python path, ++ so that you may run testbeds within PL nodes. """ # Class attribute holding a *weak* reference to the shared NEPI tar file @@@ -440,7 -440,7 +440,7 @@@ self._tarball = None -- self.depends = 'python python-ipaddrn python-setuptools' ++ self.depends = 'python python-ipaddr python-setuptools' # our sources are in our ad-hoc tarball self.sources = self.tarball.name @@@ -453,6 -453,6 +453,93 @@@ # unpack it into sources, and we're done self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,) ++ @property ++ def tarball(self): ++ if self._tarball is None: ++ shared_tar = self._shared_nepi_tar and self._shared_nepi_tar() ++ if shared_tar is not None: ++ self._tarball = shared_tar ++ else: ++ # Build an ad-hoc tarball ++ # Prebuilt ++ import nepi ++ import tempfile ++ ++ shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz') ++ ++ proc = subprocess.Popen( ++ ["tar", "czf", shared_tar.name, ++ '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'), ++ 'nepi'], ++ stdout = open("/dev/null","w"), ++ stdin = open("/dev/null","r")) ++ ++ if proc.wait(): ++ raise RuntimeError, "Failed to create nepi tarball" ++ ++ self._tarball = self._shared_nepi_tar = shared_tar ++ ++ return self._tarball ++ ++class NS3Dependency(Dependency): ++ """ ++ This dependency adds NS3 libraries to the library paths, ++ so that you may run the NS3 testbed within PL nodes. ++ ++ You'll also need the NepiDependency. ++ """ ++ ++ def __init__(self, api = None): ++ super(NS3Dependency, self).__init__(api) ++ ++ self.buildDepends = 'build-essential waf gcc gcc-c++ gccxml unzip' ++ ++ # We have to download the sources, untar, build... ++ pybindgen_source_url = "http://pybindgen.googlecode.com/files/pybindgen-0.15.0.zip" ++ pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip" ++ ns3_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/ns-3-dev/archive/tip.tar.gz" ++ self.build =("wget -q -c -O pybindgen-src.zip %(pybindgen_source_url)s && " # continue, to exploit the case when it has already been dl'ed ++ "wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && " ++ "wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && " ++ "unzip -n pybindgen-src.zip && " # Do not overwrite files, to exploit the case when it has already been built ++ "unzip -n pygccxml-1.0.0.zip && " ++ "mkdir -p ns3-src && " ++ "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && " ++ "rm -rf target && " # mv doesn't like unclean targets ++ "mkdir -p target && " ++ "cd pygccxml-1.0.0 && " ++ "python setup.py build && " ++ "python setup.py install --install-lib ${BUILD}/target && " ++ "python setup.py clean && " ++ "cd ../pybindgen-0.15.0 && " ++ "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && " ++ "./waf configure --prefix=${BUILD}/target -d release && " ++ "./waf && " ++ "./waf install && " ++ "./waf clean && " ++ "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && " ++ "rm -rf ${BUILD}/target/lib && " ++ "cd ../ns3-src && " ++ "./waf configure --prefix=${BUILD}/target -d release && " ++ "./waf &&" ++ "./waf install && " ++ "./waf clean" ++ % dict( ++ pybindgen_source_url = server.shell_escape(pybindgen_source_url), ++ pygccxml_source_url = server.shell_escape(pygccxml_source_url), ++ ns3_source_url = server.shell_escape(ns3_source_url), ++ )) ++ ++ # Just move ${BUILD}/target ++ self.install = ( ++ "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets ++ "mv -f ${BUILD}/target/* ${SOURCES}" ++ ) ++ ++ # Set extra environment paths ++ self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib" ++ self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib/libns3.so" ++ @property def tarball(self): if self._tarball is None: diff --cc src/nepi/testbeds/planetlab/execute.py index 32405f53,32405f53..8aecc76a --- a/src/nepi/testbeds/planetlab/execute.py +++ b/src/nepi/testbeds/planetlab/execute.py @@@ -222,4 -222,4 +222,6 @@@ class TestbedController(testbed_impl.Te def _make_nepi_dependency(self, parameters): return self._make_generic(parameters, self._app.NepiDependency) ++ def _make_ns3_dependency(self, parameters): ++ return self._make_generic(parameters, self._app.NS3Dependency) diff --cc src/nepi/testbeds/planetlab/metadata_v01.py index 396c920c,396c920c..9c85c048 --- a/src/nepi/testbeds/planetlab/metadata_v01.py +++ b/src/nepi/testbeds/planetlab/metadata_v01.py @@@ -20,6 -20,6 +20,7 @@@ TUNIFACE = "TunInterface APPLICATION = "Application" DEPENDENCY = "Dependency" NEPIDEPENDENCY = "NepiDependency" ++NS3DEPENDENCY = "NS3Dependency" INTERNET = "Internet" NETPIPE = "NetPipe" @@@ -114,6 -114,6 +115,11 @@@ def connect_dep(testbed_instance, node_ if app.add_to_path: if app.home_path and app.home_path not in node.pythonpath: node.pythonpath.append(app.home_path) ++ ++ if app.env: ++ for envkey, envval in app.env.iteritems(): ++ envval = app._replace_paths(envval) ++ node.env[envkey].append(envval) def connect_node_netpipe(testbed_instance, node_guid, netpipe_guid): node = testbed_instance._elements[node_guid] @@@ -178,6 -178,6 +184,15 @@@ def create_nepi_dependency(testbed_inst testbed_instance.elements[guid] = element ++def create_ns3_dependency(testbed_instance, guid): ++ parameters = testbed_instance._get_parameters(guid) ++ element = testbed_instance._make_ns3_dependency(parameters) ++ ++ # Just inject configuration stuff ++ element.home_path = "nepi-ns3-%s" % (guid,) ++ ++ testbed_instance.elements[guid] = element ++ def create_internet(testbed_instance, guid): parameters = testbed_instance._get_parameters(guid) element = testbed_instance._make_internet(parameters) @@@ -424,6 -424,6 +439,12 @@@ connections = "init_code": connect_dep, "can_cross": False }), ++ dict({ ++ "from": (TESTBED_ID, NODE, "deps"), ++ "to": (TESTBED_ID, NS3DEPENDENCY, "node"), ++ "init_code": connect_dep, ++ "can_cross": False ++ }), dict({ "from": (TESTBED_ID, NODE, "pipes"), "to": (TESTBED_ID, NETPIPE, "node"), @@@ -749,9 -749,9 +770,9 @@@ traces = dict( }), }) --create_order = [ INTERNET, NODE, NODEIFACE, TUNIFACE, NETPIPE, NEPIDEPENDENCY, DEPENDENCY, APPLICATION ] ++create_order = [ INTERNET, NODE, NODEIFACE, TUNIFACE, NETPIPE, NEPIDEPENDENCY, NS3DEPENDENCY, DEPENDENCY, APPLICATION ] --configure_order = [ INTERNET, NODE, NODEIFACE, TUNIFACE, NETPIPE, NEPIDEPENDENCY, DEPENDENCY, APPLICATION ] ++configure_order = [ INTERNET, NODE, NODEIFACE, TUNIFACE, NETPIPE, NEPIDEPENDENCY, NS3DEPENDENCY, DEPENDENCY, APPLICATION ] factories_info = dict({ NODE: dict({ @@@ -828,6 -828,6 +849,15 @@@ "configure_function": configure_dependency, "box_attributes": [ ], "connector_types": ["node"], ++ "traces": ["buildlog"] ++ }), ++ NS3DEPENDENCY: dict({ ++ "help": "Requirement for NS3 inside NEPI - required to run NS3 testbed instances inside a node. It also needs NepiDependency.", ++ "category": "applications", ++ "create_function": create_ns3_dependency, ++ "configure_function": configure_dependency, ++ "box_attributes": [ ], ++ "connector_types": ["node"], "traces": ["buildlog"] }), INTERNET: dict({ diff --cc src/nepi/testbeds/planetlab/node.py index 22143791,22143791..245e636b --- a/src/nepi/testbeds/planetlab/node.py +++ b/src/nepi/testbeds/planetlab/node.py @@@ -57,6 -57,6 +57,7 @@@ class Node(object) self.required_packages = set() self.required_vsys = set() self.pythonpath = [] ++ self.env = collections.defaultdict(list) # Testbed-derived attributes self.slicename = None diff --cc test/testbeds/planetlab/execute.py index fb73b3b3,fb73b3b3..68864e48 --- a/test/testbeds/planetlab/execute.py +++ b/test/testbeds/planetlab/execute.py @@@ -418,6 -418,6 +418,44 @@@ echo 'OKIDOKI # asserts at the end, to make sure there's proper cleanup self.assertEqual(ping_result, "") ++ ++ @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)") ++ def test_ns3_depends(self): ++ instance = self.make_instance() ++ ++ instance.defer_create(2, "Node") ++ instance.defer_create_set(2, "hostname", "onelab11.pl.sophia.inria.fr") ++ instance.defer_create(3, "NodeInterface") ++ instance.defer_connect(2, "devs", 3, "node") ++ instance.defer_create(4, "Internet") ++ instance.defer_connect(3, "inet", 4, "devs") ++ instance.defer_create(5, "NepiDependency") ++ instance.defer_connect(5, "node", 2, "deps") ++ instance.defer_create(6, "NS3Dependency") ++ instance.defer_connect(6, "node", 2, "deps") ++ instance.defer_create(12, "Application") ++ instance.defer_connect(12, "node", 2, "apps") ++ instance.defer_create_set(12, "command", "python -c 'import nepi.testbeds.ns3.execute ; tb = nepi.testbeds.ns3.execute.TestbedController(\"3_9_RC3\") ; mod = tb._load_ns3_module()'") ++ instance.defer_add_trace(12, "stderr") ++ ++ try: ++ instance.do_setup() ++ instance.do_create() ++ instance.do_connect_init() ++ instance.do_connect_compl() ++ instance.do_preconfigure() ++ instance.do_configure() ++ ++ instance.start() ++ while instance.status(12) != STATUS_FINISHED: ++ time.sleep(0.5) ++ ping_result = (instance.trace(12, "stderr") or "").strip() ++ instance.stop() ++ finally: ++ instance.shutdown() ++ ++ # asserts at the end, to make sure there's proper cleanup ++ self.assertEqual(ping_result, "") if __name__ == '__main__':