PlanetLab support toon-up: home_cleanup only nepi folders + make server support longe...

diff --git a/examples/streamming/planetlab_ccn_vlc.py b/examples/streamming/planetlab_ccn_vlc.py
index 852b0d3..91f004f 100644 (file)
--- a/examples/streamming/planetlab_ccn_vlc.py
+++ b/examples/streamming/planetlab_ccn_vlc.py
@@ -28,6 +28,9 @@ class MonitorInfo(object):
         self.type = type
         self.cpumem_monitor = None
         self.net_monitor = None
+        self.ccnd = None
+        self.ccncat = None
+        self.ccnseqwriter = None
 
 def create_slice(exp_desc, slicename, plc_host, pl_user, pl_pwd, 
         pl_ssh_key, root_dir):
@@ -170,11 +173,27 @@ def store_results(controller, monitors, results_dir, exp_label):
         # store monitoring results
         cpumem_stdout = controller.trace(mon.cpumem_monitor.guid, "stdout")
         net_stdout = controller.trace(mon.net_monitor.guid, "stdout")
-        results = dict({"cpumem": cpumem_stdout, "net": net_stdout})
-        for name, stdout in results.iteritems():
+        ccnd_error = controller.trace(mon.ccnd.guid, "stderr")
+
+        ccncat_error = None
+        if mon.ccncat:
+            ccncat_error = controller.trace(mon.ccncat.guid, "stderr")
+
+        ccnseqwriter_error = None
+        if mon.ccnseqwriter:
+            ccnseqwriter_error = controller.trace(mon.ccnseqwriter.guid, "stderr")
+        
+        results = dict({"cpumem": cpumem_stdout, "net": net_stdout, 
+            "ccnd_stderr": ccnd_error, "ccncat_stderr": ccncat_error,
+            "ccnseqwriter_stderr": ccnseqwriter_error })
+
+        for name, result in results.iteritems():
+            if not result:
+                continue
+
             fpath = os.path.join(node_path, name)
             f = open(fpath, "w")
-            f.write(stdout)
+            f.write(result)
             f.close()
 
     # store node info file
@@ -217,7 +236,7 @@ def get_options():
             help="Path to directory to store results", type="str")
     parser.add_option("-l", "--label", dest="exp_label", default = exp_label, 
             help="Label to identify experiment results", type="str")
-    parser.add_option("-t", "--time", dest="time_to_run", default = 2, 
+    parser.add_option("-t", "--time", dest="time_to_run", default = 1, 
             help="Time to run the experiment in hours", type="float")
     parser.add_option("-P", "--port", dest="port", 
             help="Port to bind the CCNx daemon", type="int")
@@ -259,12 +278,21 @@ if __name__ == '__main__':
     pl_inet = slice_desc.create("Internet")
 
     ### Level 0 - Root node
-    root_hostname = "chimay.infonet.fundp.ac.be"
+    root_hostname = "ple6.ipv6.lip6.fr"
     (root_node, root_iface) = create_node(root_hostname, pl_inet, slice_desc)
 
     ### Level 1 - Intermediate nodes
     l1_hostnames = dict()
-    l1_hostnames["uk"] = "planetlab4.cs.st-andrews.ac.uk"
+    l1_hostnames["fi"] = "planetlab-1.research.netlab.hut.fi"
+    l1_hostnames["se"] = "planetlab2.sics.se"
+    l1_hostnames["es"] = "planetlab1.um.es"
+    l1_hostnames["pt"] = "planetlab-um10.di.uminho.pt"
+    l1_hostnames["pl"] = "pandora.we.po.opole.pl"
+    l1_hostnames["it"] = "gschembra4.diit.unict.it"
+    l1_hostnames["de"] = "planetlab2.wiwi.hu-berlin.de"
+    l1_hostnames["fr"] = "planetlab1.u-strasbg.fr"
+    l1_hostnames["gr"] = "planetlab1.ics.forth.gr"
+    l1_hostnames["ch"] = "planetlab2.unineuchatel.ch"
     l1_ifaces = dict()
     l1_nodes = dict()
     
@@ -277,10 +305,10 @@ if __name__ == '__main__':
     
     # Add CCN Daemon to root node
     ifaces = l1_ifaces.values()
-    create_ccnd(root_node, slice_desc, ifaces, port)
+    root_ccnd = create_ccnd(root_node, slice_desc, ifaces, port)
 
     # Publish video in root node
-    create_ccnpush(movie, root_node, slice_desc, port)
+    root_ccnseqwriter = create_ccnpush(movie, root_node, slice_desc, port)
 
     # Create monitor info object for root node
     root_mon = MonitorInfo(root_hostname, MonitorInfo.TYPE_ROOT)
@@ -289,14 +317,22 @@ if __name__ == '__main__':
     # Add memory and cpu monitoring for root node
     root_mon.cpumem_monitor = create_cpumem_monitor(root_node, slice_desc)
     root_mon.net_monitor = create_net_monitor(root_node, slice_desc, ifaces)
+    root_mon.ccnd = root_ccnd
+    root_mon.ccnseqwriter = root_ccnseqwriter
 
     ### Level 2 - Leaf nodes
     l2_hostnames = dict()
-    l2_hostnames["uk"] = ["planetlab-1.imperial.ac.uk",
-        "planetlab3.xeno.cl.cam.ac.uk",
-        "planetlab1.xeno.cl.cam.ac.uk"
-    ]
-    
+    l2_hostnames["fi"] = ["planetlab1.rd.tut.fi",]
+    l2_hostnames["se"] = ["planetlab1.s3.kth.se",]
+    l2_hostnames["es"] = ["planetlab1.tlm.unavarra.es",]
+    l2_hostnames["pt"] = ["planet1.servers.ua.pt",]
+    l2_hostnames["pl"] = ["onelab3.warsaw.rd.tp.pl",]
+    l2_hostnames["it"] = ["gschembra3.diit.unict.it",]
+    l2_hostnames["de"] = ["iraplab1.iralab.uni-karlsruhe.de",]
+    l2_hostnames["fr"] = ["host3-plb.loria.fr",]
+    l2_hostnames["gr"] = ["kostis.di.uoa.gr",]
+    l2_hostnames["ch"] = ["planetlab04.cnds.unibe.ch",]
+
     for country, hostnames in l2_hostnames.iteritems():
         l2_ifaces = []
         l1_hostname = l1_hostnames[country]
@@ -310,10 +346,10 @@ if __name__ == '__main__':
             ### Level 2 - CCN & Monitoring
         
             # Add CCN Daemon to intermediate nodes
-            create_ccnd(pl_node, slice_desc, [l1_iface], port)
+            ccnd = create_ccnd(pl_node, slice_desc, [l1_iface], port)
 
             # Retrieve video in leaf node
-            create_ccnpull(pl_node, slice_desc, port)
+            ccncat = create_ccnpull(pl_node, slice_desc, port)
 
             # Create monitor info object for intermediate nodes
             mon = MonitorInfo(hostname, MonitorInfo.TYPE_LEAF)
@@ -322,6 +358,8 @@ if __name__ == '__main__':
             # Add memory and cpu monitoring for intermediate nodes
             mon.cpumem_monitor = create_cpumem_monitor(pl_node, slice_desc)
             mon.net_monitor = create_net_monitor(pl_node, slice_desc, [l1_iface])
+            mon.ccnd = ccnd
+            mon.ccncat = ccncat
 
         ### Level 1 - CCN & Monitoring
 
@@ -329,7 +367,7 @@ if __name__ == '__main__':
         ifaces.extend(l2_ifaces)
 
         # Add CCN Daemon to intermediate nodes
-        create_ccnd(l1_node, slice_desc, ifaces, port)
+        ccnd = create_ccnd(l1_node, slice_desc, ifaces, port)
 
         # Create monitor info object for intermediate nodes
         mon = MonitorInfo(l1_hostname, MonitorInfo.TYPE_MID)
@@ -338,6 +376,7 @@ if __name__ == '__main__':
         # Add memory and cpu monitoring for intermediate nodes
         mon.cpumem_monitor = create_cpumem_monitor(l1_node, slice_desc)
         mon.net_monitor = create_net_monitor(l1_node, slice_desc, ifaces)
+        mon.ccnd = ccnd
 
     xml = exp_desc.to_xml()
    
@@ -348,8 +387,8 @@ if __name__ == '__main__':
     duration = time_to_run * 3600 # in seconds
     while not TERMINATE:
         time.sleep(1)
-        #if (time.time() - start_time) > duration: # elapsed time
-        #    TERMINATE.append(None)
+        if (time.time() - start_time) > duration: # elapsed time
+            TERMINATE.append(None)
 
     controller.stop()
  

diff --git a/examples/streamming/planetlab_multiple_vlc.py b/examples/streamming/planetlab_multiple_vlc.py
index 9e131f4..d3da14d 100644 (file)
--- a/examples/streamming/planetlab_multiple_vlc.py
+++ b/examples/streamming/planetlab_multiple_vlc.py
@@ -38,6 +38,7 @@ class MonitorInfo(object):
         self.type = type
         self.cpumem_monitor = None
         self.net_monitor = None
+        self.vlc = None
 
 def create_slice(exp_desc, slicename, plc_host, pl_user, pl_pwd, 
         pl_ssh_key, root_dir):
@@ -83,7 +84,6 @@ def create_vlc_server(movie, pl_node, slice_desc):
 
     pl_app.set_attribute_value("sources", "%s" % movie)
     pl_app.set_attribute_value("command",
-    #        "sudo -S dbus-uuidgen --ensure ; vlc -vvv -I dummy --vlm-conf VOD.vlm --rtsp-host=0.0.0.0:8554")
         "sudo -S dbus-uuidgen --ensure ; vlc -vvv -I dummy --vlm-conf VOD.vlm")
     pl_app.enable_trace("stdout")
     pl_app.enable_trace("stderr")
@@ -117,7 +117,7 @@ def create_cpumem_monitor(pl_node, slice_desc):
     pl_app.set_attribute_value("label", label)
     pl_app.set_attribute_value("command", 
             "while true ; do echo $(date +%Y%m%d%H%M%S%z) " \
-            " $(top -b -n 1 | grep 'vlc' | head -1 | sed 's/\s\s*/ /g' | cut -d' ' -f9,10,11)" \
+            " $(top -b -n 1 | grep 'vlc' | head -1 | sed 's/\s\s*/ /g' | sed 's/^\s//g' | cut -d' ' -f9,10,11)" \
             "; sleep 1 ; done")
     pl_app.enable_trace("stdout")
     pl_app.enable_trace("stderr")
@@ -174,7 +174,11 @@ def store_results(controller, monitors, results_dir, exp_label):
         # store monitoring results
         cpumem_stdout = controller.trace(mon.cpumem_monitor.guid, "stdout")
         net_stdout = controller.trace(mon.net_monitor.guid, "stdout")
-        results = dict({"cpumem": cpumem_stdout, "net": net_stdout})
+        vlc_stderr = controller.trace(mon.vlc.guid, "stderr")
+
+        results = dict({"cpumem": cpumem_stdout, "net": net_stdout, 
+            "vlc_error": vlc_stderr})
+
         for name, stdout in results.iteritems():
             fpath = os.path.join(node_path, name)
             f = open(fpath, "w")
@@ -220,7 +224,7 @@ def get_options():
             help="Path to directory to store results", type="str")
     parser.add_option("-l", "--label", dest="exp_label", default = exp_label, 
             help="Label to identify experiment results", type="str")
-    parser.add_option("-t", "--time", dest="time_to_run", default = 2, 
+    parser.add_option("-t", "--time", dest="time_to_run", default = 1, 
             help="Time to run the experiment in hours", type="float")
 
     (options, args) = parser.parse_args()
@@ -266,16 +270,18 @@ if __name__ == '__main__':
     monitors.append(root_mon)
 
     # Add VLC service
-    create_vlc_server(movie, root_node, slice_desc)
+    root_vlc = create_vlc_server(movie, root_node, slice_desc)
     
     # Add memory and cpu monitoring for root node
     root_mon.cpumem_monitor = create_cpumem_monitor(root_node, slice_desc)
 
+    # Add reference to vlc app 
+    root_mon.vlc = root_vlc
+
     # Create leaf nodes
     cli_apps = []
     cli_ifaces = []
 
-    """
     hostnames = ["planetlab1.rd.tut.fi", 
             "planetlab1.s3.kth.se", 
             "planetlab1.tlm.unavarra.es", 
@@ -286,58 +292,6 @@ if __name__ == '__main__':
             "host3-plb.loria.fr", 
             "kostis.di.uoa.gr", 
             "planetlab04.cnds.unibe.ch"]
-    """
-
-    hostnames = ["planetlab1.rd.tut.fi",
-            "planetlab-2.research.netlab.hut.fi",
-            "planetlab2.willab.fi",
-            "planetlab3.hiit.fi",
-            "planetlab4.hiit.fi",
-            "planetlab1.s3.kth.se", 
-            "itchy.comlab.bth.se",
-            "planetlab-1.ida.liu.se",
-            "scratchy.comlab.bth.se",
-            "planetlab2.s3.kth.se", 
-            "planetlab1.tlm.unavarra.es", 
-            "planetlab2.uc3m.es",
-            "planetlab2.upc.es",
-            "ait21.us.es",
-            "planetlab3.upc.es",
-            "planet1.servers.ua.pt",
-            "planetlab2.fct.ualg.pt",
-            "planetlab-1.tagus.ist.utl.pt",
-            "planetlab1.di.fct.unl.pt",
-            "planetlab1.fct.ualg.pt",
-            "onelab3.warsaw.rd.tp.pl",
-            "onelab1.warsaw.rd.tp.pl",
-            "prata.mimuw.edu.pl",
-            "onelab2.warsaw.rd.tp.pl",
-            "prometeusz.we.po.opole.pl",
-            "gschembra3.diit.unict.it",
-            "onelab6.iet.unipi.it",
-            "planetlab1.science.unitn.it",
-            "planetlab-1.ing.unimo.it",
-            "gschembra4.diit.unict.it",
-            "iraplab1.iralab.uni-karlsruhe.de", 
-            "planetlab-1.fokus.fraunhofer.de",
-            "iraplab2.iralab.uni-karlsruhe.de",
-            "planet2.zib.de",
-            "planet2.inf.tu-dresden.de",
-            "host3-plb.loria.fr",
-            "inriarennes1.irisa.fr",
-            "inriarennes2.irisa.fr",
-            "peeramide.irisa.fr",
-            "pl1.bell-labs.fr", 
-            "kostis.di.uoa.gr",
-            "pl001.ece.upatras.gr",
-            "planetlab1.ionio.gr",
-            "planetlab2.ionio.gr",
-            "planetlab2.cs.uoi.gr", 
-            "planetlab04.cnds.unibe.ch",
-            "lsirextpc01.epfl.ch",
-            "planetlab2.csg.uzh.ch",
-            "lsirextpc02.epfl.ch",
-            "planetlab1.unineuchatel.ch"]
 
     for hostname in hostnames:
         pl_node, pl_iface = create_node(hostname, pl_inet, slice_desc)
@@ -354,8 +308,11 @@ if __name__ == '__main__':
         node_mon.net_monitor = create_net_monitor(pl_node, slice_desc, [root_iface])
 
         # Add VLC clients
-        app = create_vlc_client(root_node, pl_node, slice_desc)
-        cli_apps.append(app)
+        vlc = create_vlc_client(root_node, pl_node, slice_desc)
+        cli_apps.append(vlc)
+
+        # Add reference to vlc app 
+        node_mon.vlc = vlc
 
     # Add network monitoring for root node
     root_mon.net_monitor = create_net_monitor(root_node, slice_desc, cli_ifaces)

diff --git a/src/nepi/testbeds/planetlab/application.py b/src/nepi/testbeds/planetlab/application.py
index a5170a4..fe2994a 100644 (file)
--- a/src/nepi/testbeds/planetlab/application.py
+++ b/src/nepi/testbeds/planetlab/application.py
@@ -344,7 +344,7 @@ class Dependency(object):
             'home' : server.shell_escape(self.home_path),
             'token' : server.shell_escape(self._master_token),
         }
-        
+       
         return cStringIO.StringIO(slavescript)
          
     def _do_launch_build(self):
@@ -523,8 +523,24 @@ class Dependency(object):
             
         if self.sources:
             sources = self.sources.split(' ')
-            
-            # Copy all sources
+
+            http_sources = list()
+            for source in list(sources):
+                if source.startswith("http") or source.startswith("https"):
+                    http_sources.append(source)
+                    sources.remove(source)
+
+            # Download http sources
+            try:
+                for source in http_sources:
+                    path = os.path.join(self.home_path, source.split("/")[-1])
+                    command = "wget -o %s %s" % (path, source)
+                    self._popen_ssh(command)
+            except RuntimeError, e:
+                raise RuntimeError, "Failed wget source file %r: %s %s" \
+                        % (sources, e.args[0], e.args[1],)
+
+            # Copy all other sources
             try:
                 self._popen_scp(
                     sources,
@@ -1119,7 +1135,8 @@ class CCNxDaemon(Application):
         self.ccnx_0_5_1_sources = "http://yans.pl.sophia.inria.fr/libs/ccnx-0.5.1.tar.gz"
         #self.ccnx_0_6_0_sources = "http://www.ccnx.org/releases/ccnx-0.6.0.tar.gz"
         self.ccnx_0_6_0_sources = "http://yans.pl.sophia.inria.fr/libs/ccnx-0.6.0.tar.gz"
-        self.buildDepends = 'make gcc development-tools openssl-devel expat-devel libpcap-devel libxml2-devel'
+        #self.buildDepends = 'make gcc development-tools openssl-devel expat-devel libpcap-devel libxml2-devel'
+        self.buildDepends = 'make gcc openssl-devel expat-devel libpcap-devel libxml2-devel'
 
         self.ccnx_0_5_1_build = (
             " ( "

diff --git a/src/nepi/testbeds/planetlab/node.py b/src/nepi/testbeds/planetlab/node.py
index 59485ef..0710a1a 100644 (file)
--- a/src/nepi/testbeds/planetlab/node.py
+++ b/src/nepi/testbeds/planetlab/node.py
@@ -612,7 +612,7 @@ class Node(object):
         self._logger.info("Cleaning up home on %s", self.hostname)
         
         cmds = [
-            "find . -maxdepth 1 ! -name '.bash*' ! -name '.' -execdir rm -rf {} + "
+            "find . -maxdepth 1 -name 'nepi-*' -execdir rm -rf {} + "
         ]
 
         for cmd in cmds:

diff --git a/src/nepi/util/server.py b/src/nepi/util/server.py
index 2cdf7bd..0ece803 100644 (file)
--- a/src/nepi/util/server.py
+++ b/src/nepi/util/server.py
@@ -608,7 +608,7 @@ def popen_ssh_command(command, host, port, user, agent,
         timeout = None,
         retry = 0,
         err_on_timeout = True,
-        connect_timeout = 90,
+        connect_timeout = 1200,
         persistent = True,
         hostip = None):
     """
@@ -624,7 +624,7 @@ def popen_ssh_command(command, host, port, user, agent,
             '-o', 'NoHostAuthenticationForLocalhost=yes',
             '-o', 'ConnectTimeout=%d' % (int(connect_timeout),),
             '-o', 'ConnectionAttempts=3',
-            '-o', 'ServerAliveInterval=60',
+            '-o', 'ServerAliveInterval=30',
             '-o', 'TCPKeepAlive=yes',
             '-l', user, hostip or host]
     if persistent and openssh_has_persist():
@@ -737,9 +737,9 @@ def popen_scp(source, dest,
         args = ['ssh', '-l', user, '-C',
                 # Don't bother with localhost. Makes test easier
                 '-o', 'NoHostAuthenticationForLocalhost=yes',
-                '-o', 'ConnectTimeout=90',
+                '-o', 'ConnectTimeout=1200',
                 '-o', 'ConnectionAttempts=3',
-                '-o', 'ServerAliveInterval=60',
+                '-o', 'ServerAliveInterval=30',
                 '-o', 'TCPKeepAlive=yes',
                 host ]
         if openssh_has_persist():
@@ -871,9 +871,9 @@ def popen_scp(source, dest,
         args = ['scp', '-q', '-p', '-C',
                 # Don't bother with localhost. Makes test easier
                 '-o', 'NoHostAuthenticationForLocalhost=yes',
-                '-o', 'ConnectTimeout=90',
+                '-o', 'ConnectTimeout=1200',
                 '-o', 'ConnectionAttempts=3',
-                '-o', 'ServerAliveInterval=60',
+                '-o', 'ServerAliveInterval=30',
                 '-o', 'TCPKeepAlive=yes' ]
                 
         if port:
@@ -974,7 +974,7 @@ def popen_python(python_code,
                 # Don't bother with localhost. Makes test easier
                 '-o', 'NoHostAuthenticationForLocalhost=yes',
                 '-o', 'ConnectionAttempts=3',
-                '-o', 'ServerAliveInterval=60',
+                '-o', 'ServerAliveInterval=30',
                 '-o', 'TCPKeepAlive=yes',
                 '-l', user, host]
         if agent: