From 8d7986ddc88aeb0e14dfff08c9e857201fd1823a Mon Sep 17 00:00:00 2001 From: Alina Quereilhac Date: Mon, 8 Apr 2013 15:42:47 +0200 Subject: [PATCH] PlanetLab support toon-up: home_cleanup only nepi folders + make server support longer wait periods for ssh --- examples/streamming/planetlab_ccn_vlc.py | 75 +++++++++++++----- examples/streamming/planetlab_multiple_vlc.py | 77 ++++--------------- src/nepi/testbeds/planetlab/application.py | 25 +++++- src/nepi/testbeds/planetlab/node.py | 2 +- src/nepi/util/server.py | 14 ++-- 5 files changed, 103 insertions(+), 90 deletions(-) diff --git a/examples/streamming/planetlab_ccn_vlc.py b/examples/streamming/planetlab_ccn_vlc.py index 852b0d35..91f004f3 100644 --- a/examples/streamming/planetlab_ccn_vlc.py +++ b/examples/streamming/planetlab_ccn_vlc.py @@ -28,6 +28,9 @@ class MonitorInfo(object): self.type = type self.cpumem_monitor = None self.net_monitor = None + self.ccnd = None + self.ccncat = None + self.ccnseqwriter = None def create_slice(exp_desc, slicename, plc_host, pl_user, pl_pwd, pl_ssh_key, root_dir): @@ -170,11 +173,27 @@ def store_results(controller, monitors, results_dir, exp_label): # store monitoring results cpumem_stdout = controller.trace(mon.cpumem_monitor.guid, "stdout") net_stdout = controller.trace(mon.net_monitor.guid, "stdout") - results = dict({"cpumem": cpumem_stdout, "net": net_stdout}) - for name, stdout in results.iteritems(): + ccnd_error = controller.trace(mon.ccnd.guid, "stderr") + + ccncat_error = None + if mon.ccncat: + ccncat_error = controller.trace(mon.ccncat.guid, "stderr") + + ccnseqwriter_error = None + if mon.ccnseqwriter: + ccnseqwriter_error = controller.trace(mon.ccnseqwriter.guid, "stderr") + + results = dict({"cpumem": cpumem_stdout, "net": net_stdout, + "ccnd_stderr": ccnd_error, "ccncat_stderr": ccncat_error, + "ccnseqwriter_stderr": ccnseqwriter_error }) + + for name, result in results.iteritems(): + if not result: + continue + fpath = os.path.join(node_path, name) f = open(fpath, "w") - f.write(stdout) + f.write(result) f.close() # store node info file @@ -217,7 +236,7 @@ def get_options(): help="Path to directory to store results", type="str") parser.add_option("-l", "--label", dest="exp_label", default = exp_label, help="Label to identify experiment results", type="str") - parser.add_option("-t", "--time", dest="time_to_run", default = 2, + parser.add_option("-t", "--time", dest="time_to_run", default = 1, help="Time to run the experiment in hours", type="float") parser.add_option("-P", "--port", dest="port", help="Port to bind the CCNx daemon", type="int") @@ -259,12 +278,21 @@ if __name__ == '__main__': pl_inet = slice_desc.create("Internet") ### Level 0 - Root node - root_hostname = "chimay.infonet.fundp.ac.be" + root_hostname = "ple6.ipv6.lip6.fr" (root_node, root_iface) = create_node(root_hostname, pl_inet, slice_desc) ### Level 1 - Intermediate nodes l1_hostnames = dict() - l1_hostnames["uk"] = "planetlab4.cs.st-andrews.ac.uk" + l1_hostnames["fi"] = "planetlab-1.research.netlab.hut.fi" + l1_hostnames["se"] = "planetlab2.sics.se" + l1_hostnames["es"] = "planetlab1.um.es" + l1_hostnames["pt"] = "planetlab-um10.di.uminho.pt" + l1_hostnames["pl"] = "pandora.we.po.opole.pl" + l1_hostnames["it"] = "gschembra4.diit.unict.it" + l1_hostnames["de"] = "planetlab2.wiwi.hu-berlin.de" + l1_hostnames["fr"] = "planetlab1.u-strasbg.fr" + l1_hostnames["gr"] = "planetlab1.ics.forth.gr" + l1_hostnames["ch"] = "planetlab2.unineuchatel.ch" l1_ifaces = dict() l1_nodes = dict() @@ -277,10 +305,10 @@ if __name__ == '__main__': # Add CCN Daemon to root node ifaces = l1_ifaces.values() - create_ccnd(root_node, slice_desc, ifaces, port) + root_ccnd = create_ccnd(root_node, slice_desc, ifaces, port) # Publish video in root node - create_ccnpush(movie, root_node, slice_desc, port) + root_ccnseqwriter = create_ccnpush(movie, root_node, slice_desc, port) # Create monitor info object for root node root_mon = MonitorInfo(root_hostname, MonitorInfo.TYPE_ROOT) @@ -289,14 +317,22 @@ if __name__ == '__main__': # Add memory and cpu monitoring for root node root_mon.cpumem_monitor = create_cpumem_monitor(root_node, slice_desc) root_mon.net_monitor = create_net_monitor(root_node, slice_desc, ifaces) + root_mon.ccnd = root_ccnd + root_mon.ccnseqwriter = root_ccnseqwriter ### Level 2 - Leaf nodes l2_hostnames = dict() - l2_hostnames["uk"] = ["planetlab-1.imperial.ac.uk", - "planetlab3.xeno.cl.cam.ac.uk", - "planetlab1.xeno.cl.cam.ac.uk" - ] - + l2_hostnames["fi"] = ["planetlab1.rd.tut.fi",] + l2_hostnames["se"] = ["planetlab1.s3.kth.se",] + l2_hostnames["es"] = ["planetlab1.tlm.unavarra.es",] + l2_hostnames["pt"] = ["planet1.servers.ua.pt",] + l2_hostnames["pl"] = ["onelab3.warsaw.rd.tp.pl",] + l2_hostnames["it"] = ["gschembra3.diit.unict.it",] + l2_hostnames["de"] = ["iraplab1.iralab.uni-karlsruhe.de",] + l2_hostnames["fr"] = ["host3-plb.loria.fr",] + l2_hostnames["gr"] = ["kostis.di.uoa.gr",] + l2_hostnames["ch"] = ["planetlab04.cnds.unibe.ch",] + for country, hostnames in l2_hostnames.iteritems(): l2_ifaces = [] l1_hostname = l1_hostnames[country] @@ -310,10 +346,10 @@ if __name__ == '__main__': ### Level 2 - CCN & Monitoring # Add CCN Daemon to intermediate nodes - create_ccnd(pl_node, slice_desc, [l1_iface], port) + ccnd = create_ccnd(pl_node, slice_desc, [l1_iface], port) # Retrieve video in leaf node - create_ccnpull(pl_node, slice_desc, port) + ccncat = create_ccnpull(pl_node, slice_desc, port) # Create monitor info object for intermediate nodes mon = MonitorInfo(hostname, MonitorInfo.TYPE_LEAF) @@ -322,6 +358,8 @@ if __name__ == '__main__': # Add memory and cpu monitoring for intermediate nodes mon.cpumem_monitor = create_cpumem_monitor(pl_node, slice_desc) mon.net_monitor = create_net_monitor(pl_node, slice_desc, [l1_iface]) + mon.ccnd = ccnd + mon.ccncat = ccncat ### Level 1 - CCN & Monitoring @@ -329,7 +367,7 @@ if __name__ == '__main__': ifaces.extend(l2_ifaces) # Add CCN Daemon to intermediate nodes - create_ccnd(l1_node, slice_desc, ifaces, port) + ccnd = create_ccnd(l1_node, slice_desc, ifaces, port) # Create monitor info object for intermediate nodes mon = MonitorInfo(l1_hostname, MonitorInfo.TYPE_MID) @@ -338,6 +376,7 @@ if __name__ == '__main__': # Add memory and cpu monitoring for intermediate nodes mon.cpumem_monitor = create_cpumem_monitor(l1_node, slice_desc) mon.net_monitor = create_net_monitor(l1_node, slice_desc, ifaces) + mon.ccnd = ccnd xml = exp_desc.to_xml() @@ -348,8 +387,8 @@ if __name__ == '__main__': duration = time_to_run * 3600 # in seconds while not TERMINATE: time.sleep(1) - #if (time.time() - start_time) > duration: # elapsed time - # TERMINATE.append(None) + if (time.time() - start_time) > duration: # elapsed time + TERMINATE.append(None) controller.stop() diff --git a/examples/streamming/planetlab_multiple_vlc.py b/examples/streamming/planetlab_multiple_vlc.py index 9e131f4e..d3da14dc 100644 --- a/examples/streamming/planetlab_multiple_vlc.py +++ b/examples/streamming/planetlab_multiple_vlc.py @@ -38,6 +38,7 @@ class MonitorInfo(object): self.type = type self.cpumem_monitor = None self.net_monitor = None + self.vlc = None def create_slice(exp_desc, slicename, plc_host, pl_user, pl_pwd, pl_ssh_key, root_dir): @@ -83,7 +84,6 @@ def create_vlc_server(movie, pl_node, slice_desc): pl_app.set_attribute_value("sources", "%s" % movie) pl_app.set_attribute_value("command", - # "sudo -S dbus-uuidgen --ensure ; vlc -vvv -I dummy --vlm-conf VOD.vlm --rtsp-host=0.0.0.0:8554") "sudo -S dbus-uuidgen --ensure ; vlc -vvv -I dummy --vlm-conf VOD.vlm") pl_app.enable_trace("stdout") pl_app.enable_trace("stderr") @@ -117,7 +117,7 @@ def create_cpumem_monitor(pl_node, slice_desc): pl_app.set_attribute_value("label", label) pl_app.set_attribute_value("command", "while true ; do echo $(date +%Y%m%d%H%M%S%z) " \ - " $(top -b -n 1 | grep 'vlc' | head -1 | sed 's/\s\s*/ /g' | cut -d' ' -f9,10,11)" \ + " $(top -b -n 1 | grep 'vlc' | head -1 | sed 's/\s\s*/ /g' | sed 's/^\s//g' | cut -d' ' -f9,10,11)" \ "; sleep 1 ; done") pl_app.enable_trace("stdout") pl_app.enable_trace("stderr") @@ -174,7 +174,11 @@ def store_results(controller, monitors, results_dir, exp_label): # store monitoring results cpumem_stdout = controller.trace(mon.cpumem_monitor.guid, "stdout") net_stdout = controller.trace(mon.net_monitor.guid, "stdout") - results = dict({"cpumem": cpumem_stdout, "net": net_stdout}) + vlc_stderr = controller.trace(mon.vlc.guid, "stderr") + + results = dict({"cpumem": cpumem_stdout, "net": net_stdout, + "vlc_error": vlc_stderr}) + for name, stdout in results.iteritems(): fpath = os.path.join(node_path, name) f = open(fpath, "w") @@ -220,7 +224,7 @@ def get_options(): help="Path to directory to store results", type="str") parser.add_option("-l", "--label", dest="exp_label", default = exp_label, help="Label to identify experiment results", type="str") - parser.add_option("-t", "--time", dest="time_to_run", default = 2, + parser.add_option("-t", "--time", dest="time_to_run", default = 1, help="Time to run the experiment in hours", type="float") (options, args) = parser.parse_args() @@ -266,16 +270,18 @@ if __name__ == '__main__': monitors.append(root_mon) # Add VLC service - create_vlc_server(movie, root_node, slice_desc) + root_vlc = create_vlc_server(movie, root_node, slice_desc) # Add memory and cpu monitoring for root node root_mon.cpumem_monitor = create_cpumem_monitor(root_node, slice_desc) + # Add reference to vlc app + root_mon.vlc = root_vlc + # Create leaf nodes cli_apps = [] cli_ifaces = [] - """ hostnames = ["planetlab1.rd.tut.fi", "planetlab1.s3.kth.se", "planetlab1.tlm.unavarra.es", @@ -286,58 +292,6 @@ if __name__ == '__main__': "host3-plb.loria.fr", "kostis.di.uoa.gr", "planetlab04.cnds.unibe.ch"] - """ - - hostnames = ["planetlab1.rd.tut.fi", - "planetlab-2.research.netlab.hut.fi", - "planetlab2.willab.fi", - "planetlab3.hiit.fi", - "planetlab4.hiit.fi", - "planetlab1.s3.kth.se", - "itchy.comlab.bth.se", - "planetlab-1.ida.liu.se", - "scratchy.comlab.bth.se", - "planetlab2.s3.kth.se", - "planetlab1.tlm.unavarra.es", - "planetlab2.uc3m.es", - "planetlab2.upc.es", - "ait21.us.es", - "planetlab3.upc.es", - "planet1.servers.ua.pt", - "planetlab2.fct.ualg.pt", - "planetlab-1.tagus.ist.utl.pt", - "planetlab1.di.fct.unl.pt", - "planetlab1.fct.ualg.pt", - "onelab3.warsaw.rd.tp.pl", - "onelab1.warsaw.rd.tp.pl", - "prata.mimuw.edu.pl", - "onelab2.warsaw.rd.tp.pl", - "prometeusz.we.po.opole.pl", - "gschembra3.diit.unict.it", - "onelab6.iet.unipi.it", - "planetlab1.science.unitn.it", - "planetlab-1.ing.unimo.it", - "gschembra4.diit.unict.it", - "iraplab1.iralab.uni-karlsruhe.de", - "planetlab-1.fokus.fraunhofer.de", - "iraplab2.iralab.uni-karlsruhe.de", - "planet2.zib.de", - "planet2.inf.tu-dresden.de", - "host3-plb.loria.fr", - "inriarennes1.irisa.fr", - "inriarennes2.irisa.fr", - "peeramide.irisa.fr", - "pl1.bell-labs.fr", - "kostis.di.uoa.gr", - "pl001.ece.upatras.gr", - "planetlab1.ionio.gr", - "planetlab2.ionio.gr", - "planetlab2.cs.uoi.gr", - "planetlab04.cnds.unibe.ch", - "lsirextpc01.epfl.ch", - "planetlab2.csg.uzh.ch", - "lsirextpc02.epfl.ch", - "planetlab1.unineuchatel.ch"] for hostname in hostnames: pl_node, pl_iface = create_node(hostname, pl_inet, slice_desc) @@ -354,8 +308,11 @@ if __name__ == '__main__': node_mon.net_monitor = create_net_monitor(pl_node, slice_desc, [root_iface]) # Add VLC clients - app = create_vlc_client(root_node, pl_node, slice_desc) - cli_apps.append(app) + vlc = create_vlc_client(root_node, pl_node, slice_desc) + cli_apps.append(vlc) + + # Add reference to vlc app + node_mon.vlc = vlc # Add network monitoring for root node root_mon.net_monitor = create_net_monitor(root_node, slice_desc, cli_ifaces) diff --git a/src/nepi/testbeds/planetlab/application.py b/src/nepi/testbeds/planetlab/application.py index a5170a46..fe2994a2 100644 --- a/src/nepi/testbeds/planetlab/application.py +++ b/src/nepi/testbeds/planetlab/application.py @@ -344,7 +344,7 @@ class Dependency(object): 'home' : server.shell_escape(self.home_path), 'token' : server.shell_escape(self._master_token), } - + return cStringIO.StringIO(slavescript) def _do_launch_build(self): @@ -523,8 +523,24 @@ class Dependency(object): if self.sources: sources = self.sources.split(' ') - - # Copy all sources + + http_sources = list() + for source in list(sources): + if source.startswith("http") or source.startswith("https"): + http_sources.append(source) + sources.remove(source) + + # Download http sources + try: + for source in http_sources: + path = os.path.join(self.home_path, source.split("/")[-1]) + command = "wget -o %s %s" % (path, source) + self._popen_ssh(command) + except RuntimeError, e: + raise RuntimeError, "Failed wget source file %r: %s %s" \ + % (sources, e.args[0], e.args[1],) + + # Copy all other sources try: self._popen_scp( sources, @@ -1119,7 +1135,8 @@ class CCNxDaemon(Application): self.ccnx_0_5_1_sources = "http://yans.pl.sophia.inria.fr/libs/ccnx-0.5.1.tar.gz" #self.ccnx_0_6_0_sources = "http://www.ccnx.org/releases/ccnx-0.6.0.tar.gz" self.ccnx_0_6_0_sources = "http://yans.pl.sophia.inria.fr/libs/ccnx-0.6.0.tar.gz" - self.buildDepends = 'make gcc development-tools openssl-devel expat-devel libpcap-devel libxml2-devel' + #self.buildDepends = 'make gcc development-tools openssl-devel expat-devel libpcap-devel libxml2-devel' + self.buildDepends = 'make gcc openssl-devel expat-devel libpcap-devel libxml2-devel' self.ccnx_0_5_1_build = ( " ( " diff --git a/src/nepi/testbeds/planetlab/node.py b/src/nepi/testbeds/planetlab/node.py index 59485ef2..0710a1ac 100644 --- a/src/nepi/testbeds/planetlab/node.py +++ b/src/nepi/testbeds/planetlab/node.py @@ -612,7 +612,7 @@ class Node(object): self._logger.info("Cleaning up home on %s", self.hostname) cmds = [ - "find . -maxdepth 1 ! -name '.bash*' ! -name '.' -execdir rm -rf {} + " + "find . -maxdepth 1 -name 'nepi-*' -execdir rm -rf {} + " ] for cmd in cmds: diff --git a/src/nepi/util/server.py b/src/nepi/util/server.py index 2cdf7bda..0ece8038 100644 --- a/src/nepi/util/server.py +++ b/src/nepi/util/server.py @@ -608,7 +608,7 @@ def popen_ssh_command(command, host, port, user, agent, timeout = None, retry = 0, err_on_timeout = True, - connect_timeout = 90, + connect_timeout = 1200, persistent = True, hostip = None): """ @@ -624,7 +624,7 @@ def popen_ssh_command(command, host, port, user, agent, '-o', 'NoHostAuthenticationForLocalhost=yes', '-o', 'ConnectTimeout=%d' % (int(connect_timeout),), '-o', 'ConnectionAttempts=3', - '-o', 'ServerAliveInterval=60', + '-o', 'ServerAliveInterval=30', '-o', 'TCPKeepAlive=yes', '-l', user, hostip or host] if persistent and openssh_has_persist(): @@ -737,9 +737,9 @@ def popen_scp(source, dest, args = ['ssh', '-l', user, '-C', # Don't bother with localhost. Makes test easier '-o', 'NoHostAuthenticationForLocalhost=yes', - '-o', 'ConnectTimeout=90', + '-o', 'ConnectTimeout=1200', '-o', 'ConnectionAttempts=3', - '-o', 'ServerAliveInterval=60', + '-o', 'ServerAliveInterval=30', '-o', 'TCPKeepAlive=yes', host ] if openssh_has_persist(): @@ -871,9 +871,9 @@ def popen_scp(source, dest, args = ['scp', '-q', '-p', '-C', # Don't bother with localhost. Makes test easier '-o', 'NoHostAuthenticationForLocalhost=yes', - '-o', 'ConnectTimeout=90', + '-o', 'ConnectTimeout=1200', '-o', 'ConnectionAttempts=3', - '-o', 'ServerAliveInterval=60', + '-o', 'ServerAliveInterval=30', '-o', 'TCPKeepAlive=yes' ] if port: @@ -974,7 +974,7 @@ def popen_python(python_code, # Don't bother with localhost. Makes test easier '-o', 'NoHostAuthenticationForLocalhost=yes', '-o', 'ConnectionAttempts=3', - '-o', 'ServerAliveInterval=60', + '-o', 'ServerAliveInterval=30', '-o', 'TCPKeepAlive=yes', '-l', user, host] if agent: -- 2.43.0