adding ccnx tree topology streamming over Internet example

diff --git a/examples/streamming/planetlab_ccn_vlc.py b/examples/streamming/planetlab_ccn_vlc.py
new file mode 100644 (file)
index 0000000..852b0d3
--- /dev/null
+++ b/examples/streamming/planetlab_ccn_vlc.py
@@ -0,0 +1,360 @@
+#!/usr/bin/env python
+
+from nepi.core.design import ExperimentDescription, FactoriesProvider
+from nepi.core.execute import ExperimentController
+from nepi.util.constants import ApplicationStatus as AS
+from optparse import OptionParser, SUPPRESS_HELP
+import os
+import tempfile
+import time
+import uuid
+
+# Trak SIGTERM, and set global termination flag instead of dying
+import signal
+TERMINATE = []
+def _finalize(sig,frame):
+    global TERMINATE
+    TERMINATE.append(None)
+signal.signal(signal.SIGTERM, _finalize)
+signal.signal(signal.SIGINT, _finalize)
+
+class MonitorInfo(object):
+    TYPE_ROOT = "root"
+    TYPE_MID  = "middle"
+    TYPE_LEAF = "leaf"
+
+    def __init__(self, hostname, type):
+        self.hostname = hostname
+        self.type = type
+        self.cpumem_monitor = None
+        self.net_monitor = None
+
+def create_slice(exp_desc, slicename, plc_host, pl_user, pl_pwd, 
+        pl_ssh_key, root_dir):
+    pl_provider = FactoriesProvider("planetlab")
+    slice_desc = exp_desc.add_testbed_description(pl_provider)
+    slice_desc.set_attribute_value("homeDirectory", root_dir)
+    slice_desc.set_attribute_value("slice", slicename)
+    slice_desc.set_attribute_value("sliceSSHKey", pl_ssh_key)
+    slice_desc.set_attribute_value("authUser", pl_user)
+    slice_desc.set_attribute_value("authPass", pl_pwd)
+    slice_desc.set_attribute_value("plcHost", plc_host)
+    # Kills all running processes before starting the experiment
+    slice_desc.set_attribute_value("cleanProc", True)
+    # NOTICE: Setting 'cleanHome' to 'True' will erase all previous
+    # folders in the sliver Home directory, including result files!
+    slice_desc.set_attribute_value("cleanHome", True)
+    slice_desc.set_attribute_value("plLogLevel", "DEBUG")
+    return slice_desc
+ 
+def create_node(hostname, pl_inet, slice_desc):
+    pl_node = slice_desc.create("Node")
+    pl_node.set_attribute_value("hostname", hostname)
+    pl_node.set_attribute_value("label", "%d" % pl_node.guid)
+    pl_node.set_attribute_value("operatingSystem", "f12")
+    pl_iface = slice_desc.create("NodeInterface")
+    pl_iface.set_attribute_value("label", "iface_%d" % pl_node.guid)
+    pl_iface.connector("inet").connect(pl_inet.connector("devs"))
+    pl_node.connector("devs").connect(pl_iface.connector("node"))
+    return pl_node, pl_iface
+
+def create_ccnd(pl_node, slice_desc, pl_ifaces, port):
+    pl_app = slice_desc.create("CCNxDaemon")
+    pl_app.set_attribute_value("ccnxVersion", "ccnx-0.5.1")
+    
+    # We use a wildcard to replace the public IP address of the node during runtime,
+    # once this IP is known
+    routes = "|".join(map(lambda pl_iface: "udp {#[%s].addr[0].[Address]#}" % 
+        pl_iface.get_attribute_value("label"), pl_ifaces))
+    
+    # Add unicast ccn routes 
+    pl_app.set_attribute_value("ccnRoutes", routes)
+
+    # Use a specific port to bind the CCNx daemon
+    if port:
+        pl_app.set_attribute_value("ccnLocalPort", port)
+
+    pl_app.enable_trace("stdout")
+    pl_app.enable_trace("stderr")
+    pl_app.connector("node").connect(pl_node.connector("apps"))
+    return pl_app
+
+def create_ccnpush(movie, pl_node, slice_desc, port):
+    pl_app = slice_desc.create("Application")
+    pl_app.set_attribute_value("stdin", movie)
+
+    command = "ccnseqwriter ccnx:/VIDEO"
+    if port:
+        command = "CCN_LOCAL_PORT=%d %s " % (port, command)
+
+    pl_app.set_attribute_value("command", command)
+
+    pl_app.enable_trace("stdout")
+    pl_app.enable_trace("stderr")
+    pl_app.connector("node").connect(pl_node.connector("apps"))
+    return pl_app
+
+def create_ccnpull(pl_node, slice_desc, port):
+    pl_app = slice_desc.create("Application")
+    pl_app.set_attribute_value("rpmFusion", True)
+    pl_app.set_attribute_value("depends", "vlc")
+
+    command = " sudo -S dbus-uuidgen --ensure ; while true ; do ccncat ccnx:/VIDEO"
+    if port:
+        command = "CCN_LOCAL_PORT=%d %s " % (port, command)
+
+    command += " | vlc -I dummy - vlc://quit > /dev/null ; done"
+    pl_app.set_attribute_value("command", command)
+    
+    pl_app.enable_trace("stdout")
+    pl_app.enable_trace("stderr")
+    pl_app.connector("node").connect(pl_node.connector("apps"))
+    return pl_app
+
+def create_cpumem_monitor(pl_node, slice_desc):
+    label = "%d_cpumem" % pl_node.guid
+    pl_app = slice_desc.create("Application")
+    pl_app.set_attribute_value("label", label)
+    pl_app.set_attribute_value("command", 
+            "while true; do echo $(date +%Y%m%d%H%M%S%z) "\
+            " $(top -b -n 1 |  grep 'bash\|python' | sed 's/\s\s*/ /g' | "\
+            " sed 's/^\s//g' | cut -d' ' -f9,10,11 | awk '{ sum1 +=$1; sum2 += $2; } "\
+            " END {printf \"%2.1f %2.1f 0:00.00\", sum1, sum2;}'); sleep 1 ; done ")
+
+    pl_app.enable_trace("stdout")
+    pl_app.enable_trace("stderr")
+    pl_node.connector("apps").connect(pl_app.connector("node"))
+    return pl_app
+
+def create_net_monitor(pl_node, slice_desc, pl_ifaces):
+    label = "%d_net" % pl_node.guid
+    hosts = " or ".join(map(lambda pl_iface: " ( host {#[%s].addr[0].[Address]#} ) " % 
+        pl_iface.get_attribute_value("label"), pl_ifaces))
+    pl_app = slice_desc.create("Application")
+    pl_app.set_attribute_value("label", label)
+    pl_app.set_attribute_value("rpmFusion", True)
+    pl_app.set_attribute_value("sudo", True)
+    pl_app.set_attribute_value("depends", "tcpdump pv")
+    pl_app.set_attribute_value("command", 
+            "tcpdump -l -i eth0 -nNqttf '(%s)' -w - | pv -fbt >/dev/null 2>>{#[%s].trace[stdout].[name]#}" %
+            (hosts, label))
+    pl_app.enable_trace("stdout")
+    pl_app.enable_trace("stderr")
+    pl_node.connector("apps").connect(pl_app.connector("node"))
+    return pl_app
+
+def store_results(controller, monitors, results_dir, exp_label):
+    # create results directory for experiment
+    root_path = os.path.join(results_dir, exp_label)
+
+    print "STORING RESULTS in ", root_path
+
+    try:
+        os.makedirs(root_path)
+    except OSError:
+        pass
+
+    # collect information on nodes
+    hosts_info = ""
+
+    for mon in monitors:
+        hosts_info += "%s %s\n" % (mon.hostname, mon.type)
+
+        # create a subdir per hostname
+        node_path = os.path.join(root_path, mon.hostname)
+        try:
+            os.makedirs(node_path)
+        except OSError:
+            pass
+
+        # store monitoring results
+        cpumem_stdout = controller.trace(mon.cpumem_monitor.guid, "stdout")
+        net_stdout = controller.trace(mon.net_monitor.guid, "stdout")
+        results = dict({"cpumem": cpumem_stdout, "net": net_stdout})
+        for name, stdout in results.iteritems():
+            fpath = os.path.join(node_path, name)
+            f = open(fpath, "w")
+            f.write(stdout)
+            f.close()
+
+    # store node info file
+    fpath = os.path.join(root_path, "hosts")
+    f = open(fpath, "w")
+    f.write(hosts_info)
+    f.close()
+
+def get_options():
+    slicename = os.environ.get("PL_SLICE")
+    pl_host = os.environ.get("PL_HOST", "www.planet-lab.eu")
+    pl_ssh_key = os.environ.get(
+        "PL_SSH_KEY",
+        "%s/.ssh/id_rsa_planetlab" % (os.environ['HOME'],) )
+    pl_user = os.environ.get('PL_USER')
+    pl_pwd = os.environ.get('PL_PASS')
+    exp_label = "%s" % uuid.uuid4()
+
+    usage = "usage: %prog -s <pl_slice> -H <pl_host> -k <ssh_key> -u <pl_user> \
+-m <movie> -p <pl_password> -r <results-dir> -l <experiment-label> \
+-P <ccnd-port>"
+
+    parser = OptionParser(usage=usage)
+    parser.add_option("-s", "--slicename", dest="slicename", 
+            help="PlanetLab slicename", default=slicename, type="str")
+    parser.add_option("-H", "--pl-host", dest="pl_host", 
+            help="PlanetLab site (e.g. www.planet-lab.eu)", 
+            default=pl_host, type="str")
+    parser.add_option("-k", "--ssh-key", dest="pl_ssh_key", 
+            help="Path to private ssh key used for PlanetLab authentication", 
+            default=pl_ssh_key, type="str")
+    parser.add_option("-u", "--pl-user", dest="pl_user", 
+            help="PlanetLab account user (i.e. Registration email address)", 
+            default=pl_user, type="str")
+    parser.add_option("-p", "--pl-pwd", dest="pl_pwd", 
+            help="PlanetLab account password", default=pl_pwd, type="str")
+    parser.add_option("-m", "--movie", dest="movie", 
+            help="Stream movie", type="str")
+    parser.add_option("-r", "--results", dest="results_dir", default = "/tmp", 
+            help="Path to directory to store results", type="str")
+    parser.add_option("-l", "--label", dest="exp_label", default = exp_label, 
+            help="Label to identify experiment results", type="str")
+    parser.add_option("-t", "--time", dest="time_to_run", default = 2, 
+            help="Time to run the experiment in hours", type="float")
+    parser.add_option("-P", "--port", dest="port", 
+            help="Port to bind the CCNx daemon", type="int")
+
+    (options, args) = parser.parse_args()
+
+    if not options.movie:
+        parser.error("movie is a required argument")
+
+    return (options.slicename, options.pl_host, options.pl_user, 
+            options.pl_pwd, options.pl_ssh_key, options.movie,
+            options.results_dir, options.exp_label, options.time_to_run,
+            options.port)
+
+if __name__ == '__main__':
+    root_dir = tempfile.mkdtemp()
+    (pl_slice, 
+            pl_host, 
+            pl_user, 
+            pl_pwd, 
+            pl_ssh_key, 
+            movie, 
+            results_dir,
+            exp_label,
+            time_to_run,
+            port) = get_options()
+
+    # list to store information on monitoring apps per node
+    monitors = []
+    
+    # Create the experiment description object
+    exp_desc = ExperimentDescription()
+
+    # Create slice
+    slice_desc = create_slice(exp_desc, pl_slice, pl_host, pl_user, pl_pwd,
+        pl_ssh_key, root_dir)
+   
+    # Create the Internet box object
+    pl_inet = slice_desc.create("Internet")
+
+    ### Level 0 - Root node
+    root_hostname = "chimay.infonet.fundp.ac.be"
+    (root_node, root_iface) = create_node(root_hostname, pl_inet, slice_desc)
+
+    ### Level 1 - Intermediate nodes
+    l1_hostnames = dict()
+    l1_hostnames["uk"] = "planetlab4.cs.st-andrews.ac.uk"
+    l1_ifaces = dict()
+    l1_nodes = dict()
+    
+    for country, hostname in l1_hostnames.iteritems():
+        pl_node, pl_iface = create_node(hostname, pl_inet, slice_desc)
+        l1_ifaces[country] = pl_iface
+        l1_nodes[country] = pl_node
+
+    ### Level 0 - CCN & Monitoring
+    
+    # Add CCN Daemon to root node
+    ifaces = l1_ifaces.values()
+    create_ccnd(root_node, slice_desc, ifaces, port)
+
+    # Publish video in root node
+    create_ccnpush(movie, root_node, slice_desc, port)
+
+    # Create monitor info object for root node
+    root_mon = MonitorInfo(root_hostname, MonitorInfo.TYPE_ROOT)
+    monitors.append(root_mon)
+   
+    # Add memory and cpu monitoring for root node
+    root_mon.cpumem_monitor = create_cpumem_monitor(root_node, slice_desc)
+    root_mon.net_monitor = create_net_monitor(root_node, slice_desc, ifaces)
+
+    ### Level 2 - Leaf nodes
+    l2_hostnames = dict()
+    l2_hostnames["uk"] = ["planetlab-1.imperial.ac.uk",
+        "planetlab3.xeno.cl.cam.ac.uk",
+        "planetlab1.xeno.cl.cam.ac.uk"
+    ]
+    
+    for country, hostnames in l2_hostnames.iteritems():
+        l2_ifaces = []
+        l1_hostname = l1_hostnames[country]
+        l1_iface = l1_ifaces[country]
+        l1_node = l1_nodes[country]
+        
+        for hostname in hostnames:
+            pl_node, pl_iface = create_node(hostname, pl_inet, slice_desc)
+            l2_ifaces.append(pl_iface)
+
+            ### Level 2 - CCN & Monitoring
+        
+            # Add CCN Daemon to intermediate nodes
+            create_ccnd(pl_node, slice_desc, [l1_iface], port)
+
+            # Retrieve video in leaf node
+            create_ccnpull(pl_node, slice_desc, port)
+
+            # Create monitor info object for intermediate nodes
+            mon = MonitorInfo(hostname, MonitorInfo.TYPE_LEAF)
+            monitors.append(mon)
+       
+            # Add memory and cpu monitoring for intermediate nodes
+            mon.cpumem_monitor = create_cpumem_monitor(pl_node, slice_desc)
+            mon.net_monitor = create_net_monitor(pl_node, slice_desc, [l1_iface])
+
+        ### Level 1 - CCN & Monitoring
+
+        ifaces = [root_iface]
+        ifaces.extend(l2_ifaces)
+
+        # Add CCN Daemon to intermediate nodes
+        create_ccnd(l1_node, slice_desc, ifaces, port)
+
+        # Create monitor info object for intermediate nodes
+        mon = MonitorInfo(l1_hostname, MonitorInfo.TYPE_MID)
+        monitors.append(mon)
+       
+        # Add memory and cpu monitoring for intermediate nodes
+        mon.cpumem_monitor = create_cpumem_monitor(l1_node, slice_desc)
+        mon.net_monitor = create_net_monitor(l1_node, slice_desc, ifaces)
+
+    xml = exp_desc.to_xml()
+   
+    controller = ExperimentController(xml, root_dir)
+    controller.start()
+
+    start_time = time.time()
+    duration = time_to_run * 3600 # in seconds
+    while not TERMINATE:
+        time.sleep(1)
+        #if (time.time() - start_time) > duration: # elapsed time
+        #    TERMINATE.append(None)
+
+    controller.stop()
+ 
+    # store results in results dir
+    store_results(controller, monitors, results_dir, exp_label)
+   
+    controller.shutdown()
+

diff --git a/examples/streamming/planetlab_multiple_vlc.py b/examples/streamming/planetlab_multiple_vlc.py
index 1067001..9e131f4 100644 (file)
--- a/examples/streamming/planetlab_multiple_vlc.py
+++ b/examples/streamming/planetlab_multiple_vlc.py
@@ -20,7 +20,6 @@ transmitted per stream are traced to files.
 
 """
 
-
 # Trak SIGTERM, and set global termination flag instead of dying
 import signal
 TERMINATE = []
@@ -69,13 +68,6 @@ def create_node(hostname, pl_inet, slice_desc):
     pl_node.connector("devs").connect(pl_iface.connector("node"))
     return pl_node, pl_iface
 
-def fix_keys_app(slice_desc, pl_node):
-    pl_app = slice_desc.create("Application")
-    pl_app.set_attribute_value("command", "yum reinstall -y --nogpgcheck fedora-release")
-    pl_app.set_attribute_value("sudo", True)
-    pl_app.connector("node").connect(pl_node.connector("apps"))
-    return pl_app
-
 def create_vlc_server(movie, pl_node, slice_desc):
     mv = os.path.basename(movie)
     pl_app = slice_desc.create("Application")
@@ -125,14 +117,14 @@ def create_cpumem_monitor(pl_node, slice_desc):
     pl_app.set_attribute_value("label", label)
     pl_app.set_attribute_value("command", 
             "while true ; do echo $(date +%Y%m%d%H%M%S%z) " \
-            " $(top -b -n 1 | grep 'vlc' | head -1 | sed 's/\s\s*/ /g' | cut -d' ' -f10,11,12)" \
+            " $(top -b -n 1 | grep 'vlc' | head -1 | sed 's/\s\s*/ /g' | cut -d' ' -f9,10,11)" \
             "; sleep 1 ; done")
     pl_app.enable_trace("stdout")
     pl_app.enable_trace("stderr")
     pl_node.connector("apps").connect(pl_app.connector("node"))
     return pl_app
 
-def create_net_monitor(pl_node, slice_desc, ifaces):
+def create_net_monitor(pl_node, slice_desc, pl_ifaces):
     """ This function creates a monitoring application for the
     amount of bytes transmitted/received by the vlc application.
 
@@ -140,8 +132,8 @@ def create_net_monitor(pl_node, slice_desc, ifaces):
     'total-Mbytes total-time'
     """
     label = "%d_net" % pl_node.guid
-    hosts = " or ".join(map(lambda iface: " ( host {#[%s].addr[0].[Address]#} ) " % 
-        iface.get_attribute_value("label"), ifaces))
+    hosts = " or ".join(map(lambda pl_iface: " ( host {#[%s].addr[0].[Address]#} ) " % 
+        pl_iface.get_attribute_value("label"), pl_ifaces))
     pl_app = slice_desc.create("Application")
     pl_app.set_attribute_value("label", label)
     pl_app.set_attribute_value("rpmFusion", True)
@@ -195,7 +187,6 @@ def store_results(controller, monitors, results_dir, exp_label):
     f.write(hosts_info)
     f.close()
 
-
 def get_options():
     slicename = os.environ.get("PL_SLICE")
     pl_host = os.environ.get("PL_HOST", "www.planet-lab.eu")
@@ -274,9 +265,6 @@ if __name__ == '__main__':
     root_mon = MonitorInfo(hostname, MonitorInfo.TYPE_ROOT)
     monitors.append(root_mon)
 
-    # Fix GPG key problem in node
-    #fix_keys_app(slice_desc, root_node)
-
     # Add VLC service
     create_vlc_server(movie, root_node, slice_desc)
     
@@ -287,6 +275,7 @@ if __name__ == '__main__':
     cli_apps = []
     cli_ifaces = []
 
+    """
     hostnames = ["planetlab1.rd.tut.fi", 
             "planetlab1.s3.kth.se", 
             "planetlab1.tlm.unavarra.es", 
@@ -297,14 +286,63 @@ if __name__ == '__main__':
             "host3-plb.loria.fr", 
             "kostis.di.uoa.gr", 
             "planetlab04.cnds.unibe.ch"]
-    
+    """
+
+    hostnames = ["planetlab1.rd.tut.fi",
+            "planetlab-2.research.netlab.hut.fi",
+            "planetlab2.willab.fi",
+            "planetlab3.hiit.fi",
+            "planetlab4.hiit.fi",
+            "planetlab1.s3.kth.se", 
+            "itchy.comlab.bth.se",
+            "planetlab-1.ida.liu.se",
+            "scratchy.comlab.bth.se",
+            "planetlab2.s3.kth.se", 
+            "planetlab1.tlm.unavarra.es", 
+            "planetlab2.uc3m.es",
+            "planetlab2.upc.es",
+            "ait21.us.es",
+            "planetlab3.upc.es",
+            "planet1.servers.ua.pt",
+            "planetlab2.fct.ualg.pt",
+            "planetlab-1.tagus.ist.utl.pt",
+            "planetlab1.di.fct.unl.pt",
+            "planetlab1.fct.ualg.pt",
+            "onelab3.warsaw.rd.tp.pl",
+            "onelab1.warsaw.rd.tp.pl",
+            "prata.mimuw.edu.pl",
+            "onelab2.warsaw.rd.tp.pl",
+            "prometeusz.we.po.opole.pl",
+            "gschembra3.diit.unict.it",
+            "onelab6.iet.unipi.it",
+            "planetlab1.science.unitn.it",
+            "planetlab-1.ing.unimo.it",
+            "gschembra4.diit.unict.it",
+            "iraplab1.iralab.uni-karlsruhe.de", 
+            "planetlab-1.fokus.fraunhofer.de",
+            "iraplab2.iralab.uni-karlsruhe.de",
+            "planet2.zib.de",
+            "planet2.inf.tu-dresden.de",
+            "host3-plb.loria.fr",
+            "inriarennes1.irisa.fr",
+            "inriarennes2.irisa.fr",
+            "peeramide.irisa.fr",
+            "pl1.bell-labs.fr", 
+            "kostis.di.uoa.gr",
+            "pl001.ece.upatras.gr",
+            "planetlab1.ionio.gr",
+            "planetlab2.ionio.gr",
+            "planetlab2.cs.uoi.gr", 
+            "planetlab04.cnds.unibe.ch",
+            "lsirextpc01.epfl.ch",
+            "planetlab2.csg.uzh.ch",
+            "lsirextpc02.epfl.ch",
+            "planetlab1.unineuchatel.ch"]
+
     for hostname in hostnames:
         pl_node, pl_iface = create_node(hostname, pl_inet, slice_desc)
         cli_ifaces.append(pl_iface)
 
-        # Fix GPG key problem in node
-        #fix_keys_app(slice_desc, root_node)
- 
         # Create monitor info object for root node
         node_mon = MonitorInfo(hostname, MonitorInfo.TYPE_LEAF)
         monitors.append(node_mon)

diff --git a/src/nepi/testbeds/planetlab/application.py b/src/nepi/testbeds/planetlab/application.py
index 55883f4..a5170a4 100644 (file)
--- a/src/nepi/testbeds/planetlab/application.py
+++ b/src/nepi/testbeds/planetlab/application.py
@@ -585,7 +585,8 @@ class Dependency(object):
             except RuntimeError, e:
                 if self.check_bad_host(e.args[0], e.args[1]):
                     self.node.blacklist()
-                raise RuntimeError, "Failed install build sources: %s %s" % (e.args[0], e.args[1],)
+                raise RuntimeError, "Failed install build sources on node %s: %s %s" % (
+                        self.node.hostname, e.args[0], e.args[1],)
 
     def set_master(self, master):
         self._master = master
@@ -753,7 +754,6 @@ class Application(Dependency):
             stdout = 'stdout' if self.stdout else '/dev/null',
             stderr = 'stderr' if self.stderr else '/dev/null',
             sudo = self.sudo,
-            
             host = self.node.hostname,
             port = None,
             user = self.node.slicename,

diff --git a/src/nepi/testbeds/planetlab/rspawn.py b/src/nepi/testbeds/planetlab/rspawn.py
index b2e978e..8be8ac5 100644 (file)
--- a/src/nepi/testbeds/planetlab/rspawn.py
+++ b/src/nepi/testbeds/planetlab/rspawn.py
@@ -81,6 +81,7 @@ def remote_spawn(command, pidfile, stdout='/dev/null', stderr=STDOUT, stdin='/de
             'gohome' : 'cd %s ; ' % (server.shell_escape(home),) if home else '',
             'create' : 'mkdir -p %s ; ' % (server.shell_escape,) if create_home else '',
         }
+
     (out,err),proc = server.popen_ssh_command(
         cmd,
         host = host,
@@ -94,7 +95,7 @@ def remote_spawn(command, pidfile, stdout='/dev/null', stderr=STDOUT, stdin='/de
         )
     
     if proc.wait():
-        raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
+        raise RuntimeError, "Failed to set up application on host %s: %s %s" % (host, out,err,)
 
     return (out,err),proc
 

diff --git a/src/nepi/util/server.py b/src/nepi/util/server.py
index c53cc4e..2cdf7bd 100644 (file)
--- a/src/nepi/util/server.py
+++ b/src/nepi/util/server.py
@@ -608,7 +608,7 @@ def popen_ssh_command(command, host, port, user, agent,
         timeout = None,
         retry = 0,
         err_on_timeout = True,
-        connect_timeout = 30,
+        connect_timeout = 90,
         persistent = True,
         hostip = None):
     """
@@ -624,7 +624,7 @@ def popen_ssh_command(command, host, port, user, agent,
             '-o', 'NoHostAuthenticationForLocalhost=yes',
             '-o', 'ConnectTimeout=%d' % (int(connect_timeout),),
             '-o', 'ConnectionAttempts=3',
-            '-o', 'ServerAliveInterval=30',
+            '-o', 'ServerAliveInterval=60',
             '-o', 'TCPKeepAlive=yes',
             '-l', user, hostip or host]
     if persistent and openssh_has_persist():
@@ -737,9 +737,9 @@ def popen_scp(source, dest,
         args = ['ssh', '-l', user, '-C',
                 # Don't bother with localhost. Makes test easier
                 '-o', 'NoHostAuthenticationForLocalhost=yes',
-                '-o', 'ConnectTimeout=30',
+                '-o', 'ConnectTimeout=90',
                 '-o', 'ConnectionAttempts=3',
-                '-o', 'ServerAliveInterval=30',
+                '-o', 'ServerAliveInterval=60',
                 '-o', 'TCPKeepAlive=yes',
                 host ]
         if openssh_has_persist():
@@ -871,9 +871,9 @@ def popen_scp(source, dest,
         args = ['scp', '-q', '-p', '-C',
                 # Don't bother with localhost. Makes test easier
                 '-o', 'NoHostAuthenticationForLocalhost=yes',
-                '-o', 'ConnectTimeout=30',
+                '-o', 'ConnectTimeout=90',
                 '-o', 'ConnectionAttempts=3',
-                '-o', 'ServerAliveInterval=30',
+                '-o', 'ServerAliveInterval=60',
                 '-o', 'TCPKeepAlive=yes' ]
                 
         if port:
@@ -974,7 +974,7 @@ def popen_python(python_code,
                 # Don't bother with localhost. Makes test easier
                 '-o', 'NoHostAuthenticationForLocalhost=yes',
                 '-o', 'ConnectionAttempts=3',
-                '-o', 'ServerAliveInterval=30',
+                '-o', 'ServerAliveInterval=60',
                 '-o', 'TCPKeepAlive=yes',
                 '-l', user, host]
         if agent: