expose stack trace in error message when node is considered unresponsive

[nepi.git] / src / nepi / resources / linux / node.py

diff --git a/src/nepi/resources/linux/node.py b/src/nepi/resources/linux/node.py
index d17e4df..0896ed0 100644 (file)
--- a/src/nepi/resources/linux/node.py
+++ b/src/nepi/resources/linux/node.py
@@ -3,9 +3,8 @@
 #    Copyright (C) 2013 INRIA
 #
 #    This program is free software: you can redistribute it and/or modify
 #    Copyright (C) 2013 INRIA
 #
 #    This program is free software: you can redistribute it and/or modify

-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
+#    it under the terms of the GNU General Public License version 2 as
+#    published by the Free Software Foundation;

 #
 #    This program is distributed in the hope that it will be useful,
 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
 #
 #    This program is distributed in the hope that it will be useful,
 #    but WITHOUT ANY WARRANTY; without even the implied warranty of


@@ -18,8 +17,8 @@
 # Author: Alina Quereilhac <alina.quereilhac@inria.fr>
 
 from nepi.execution.attribute import Attribute, Flags, Types
 # Author: Alina Quereilhac <alina.quereilhac@inria.fr>
 
 from nepi.execution.attribute import Attribute, Flags, Types

-from nepi.execution.resource import ResourceManager, clsinit_copy, \
-        ResourceState, reschedule_delay
+from nepi.execution.resource import (ResourceManager, clsinit_copy, 
+                                     ResourceState)

 from nepi.resources.linux import rpmfuncs, debfuncs 
 from nepi.util import sshfuncs, execfuncs
 from nepi.util.sshfuncs import ProcStatus
 from nepi.resources.linux import rpmfuncs, debfuncs 
 from nepi.util import sshfuncs, execfuncs
 from nepi.util.sshfuncs import ProcStatus


@@ -50,12 +49,12 @@ class OSType:
     """
     Supported flavors of Linux OS
     """
     """
     Supported flavors of Linux OS
     """

-    FEDORA_8 = "f8"
-    FEDORA_12 = "f12"
-    FEDORA_14 = "f14"
-    FEDORA = "fedora"
-    UBUNTU = "ubuntu"
-    DEBIAN = "debian"
+    DEBIAN = 1 
+    UBUNTU = 1 << 1 
+    FEDORA = 1 << 2
+    FEDORA_8 = 1 << 3 | FEDORA 
+    FEDORA_12 = 1 << 4 | FEDORA 
+    FEDORA_14 = 1 << 5 | FEDORA 

 
 @clsinit_copy
 class LinuxNode(ResourceManager):
 
 @clsinit_copy
 class LinuxNode(ResourceManager):


@@ -141,70 +140,95 @@ class LinuxNode(ResourceManager):
                     source compilation, file download, etc)
 
     """
                     source compilation, file download, etc)
 
     """

-    _rtype = "LinuxNode"
+    _rtype = "linux::Node"

     _help = "Controls Linux host machines ( either localhost or a host " \
             "that can be accessed using a SSH key)"
     _help = "Controls Linux host machines ( either localhost or a host " \
             "that can be accessed using a SSH key)"

-    _backend_type = "linux"
+    _platform = "linux"

 
     @classmethod
     def _register_attributes(cls):
 
     @classmethod
     def _register_attributes(cls):

-        hostname = Attribute("hostname", "Hostname of the machine",
-                flags = Flags.ExecReadOnly)
+        cls._register_attribute(Attribute(
+            "hostname", "Hostname of the machine",
+            flags = Flags.Design))

 
 

-        username = Attribute("username", "Local account username", 
-                flags = Flags.Credential)
+        cls._register_attribute(Attribute(
+            "username", "Local account username", 
+            flags = Flags.Credential))

 
 

-        port = Attribute("port", "SSH port", flags = Flags.ExecReadOnly)
+        cls._register_attribute(Attribute(
+            "port", "SSH port",
+            flags = Flags.Design))

         
         

-        home = Attribute("home",
-                "Experiment home directory to store all experiment related files",
-                flags = Flags.ExecReadOnly)
+        cls._register_attribute(Attribute(
+            "home",
+            "Experiment home directory to store all experiment related files",
+            flags = Flags.Design))

         
         

-        identity = Attribute("identity", "SSH identity file",
-                flags = Flags.Credential)
+        cls._register_attribute(Attribute(
+            "identity", "SSH identity file",
+            flags = Flags.Credential))

         
         

-        server_key = Attribute("serverKey", "Server public key", 
-                flags = Flags.ExecReadOnly)
+        cls._register_attribute(Attribute(
+            "serverKey", "Server public key", 
+            flags = Flags.Design))

         
         

-        clean_home = Attribute("cleanHome", "Remove all nepi files and directories "
-                " from node home folder before starting experiment", 
-                type = Types.Bool,
-                default = False,
-                flags = Flags.ExecReadOnly)
-
-        clean_experiment = Attribute("cleanExperiment", "Remove all files and directories " 
-                " from a previous same experiment, before the new experiment starts", 
-                type = Types.Bool,
-                default = False,
-                flags = Flags.ExecReadOnly)
+        cls._register_attribute(Attribute(
+            "cleanHome",
+            "Remove all nepi files and directories "
+            " from node home folder before starting experiment", 
+            type = Types.Bool,
+            default = False,
+            flags = Flags.Design))
+
+        cls._register_attribute(Attribute(
+            "cleanExperiment", "Remove all files and directories " 
+            " from a previous same experiment, before the new experiment starts", 
+            type = Types.Bool,
+            default = False,
+            flags = Flags.Design))

         
         

-        clean_processes = Attribute("cleanProcesses", 
-                "Kill all running processes before starting experiment",
-                type = Types.Bool,
-                default = False,
-                flags = Flags.ExecReadOnly)
+        cls._register_attribute(Attribute(
+            "cleanProcesses", 
+            "Kill all running processes before starting experiment",
+            type = Types.Bool,
+            default = False,
+            flags = Flags.Design))

         
         

-        tear_down = Attribute("tearDown", "Bash script to be executed before " + \
-                "releasing the resource",
-                flags = Flags.ExecReadOnly)
-
-        cls._register_attribute(hostname)
-        cls._register_attribute(username)
-        cls._register_attribute(port)
-        cls._register_attribute(home)
-        cls._register_attribute(identity)
-        cls._register_attribute(server_key)
-        cls._register_attribute(clean_home)
-        cls._register_attribute(clean_experiment)
-        cls._register_attribute(clean_processes)
-        cls._register_attribute(tear_down)
+        cls._register_attribute(Attribute(
+            "cleanProcessesAfter", 
+            """Kill all running processes after starting experiment
+            This might be dangerous when using user root""",
+            type = Types.Bool,
+            default = True,
+            flags = Flags.Design))
+        
+        cls._register_attribute(Attribute(
+            "tearDown",
+            "Bash script to be executed before releasing the resource",
+            flags = Flags.Design))
+
+        cls._register_attribute(Attribute(
+            "gatewayUser",
+            "Gateway account username",
+            flags = Flags.Design))
+
+        cls._register_attribute(Attribute(
+            "gateway",
+            "Hostname of the gateway machine",
+            flags = Flags.Design))
+
+        cls._register_attribute(Attribute(
+            "ip",
+            "Linux host public IP address. "
+            "Must not be modified by the user unless hostname is 'localhost'",
+            flags = Flags.Design))

 
     def __init__(self, ec, guid):
         super(LinuxNode, self).__init__(ec, guid)
         self._os = None
         # home directory at Linux host
         self._home_dir = ""
 
     def __init__(self, ec, guid):
         super(LinuxNode, self).__init__(ec, guid)
         self._os = None
         # home directory at Linux host
         self._home_dir = ""

-        
+

         # lock to prevent concurrent applications on the same node,
         # to execute commands at the same time. There are potential
         # concurrency issues when using SSH to a same host from 
         # lock to prevent concurrent applications on the same node,
         # to execute commands at the same time. There are potential
         # concurrency issues when using SSH to a same host from 


@@ -213,21 +237,25 @@ class LinuxNode(ResourceManager):
         # of a file or folder prior to its creation, and another 
         # application creating the same file or folder in between.
         self._node_lock = threading.Lock()
         # of a file or folder prior to its creation, and another 
         # application creating the same file or folder in between.
         self._node_lock = threading.Lock()

-    
+        

     def log_message(self, msg):
     def log_message(self, msg):

-        return " guid %d - host %s - %s " % (self.guid, 
-                self.get("hostname"), msg)
+        return " guid {} - host {} - {} "\
+            .format(self.guid, self.get("hostname"), msg)

 
     @property
     def home_dir(self):
         home = self.get("home") or ""
         if not home.startswith("/"):
 
     @property
     def home_dir(self):
         home = self.get("home") or ""
         if not home.startswith("/"):

-           home = os.path.join(self._home_dir, home) 
-        return home
+            home = os.path.join(self._home_dir, home) 
+            return home
+
+    @property
+    def nepi_home(self):
+        return os.path.join(self.home_dir, ".nepi")

 
     @property
     def usr_dir(self):
 
     @property
     def usr_dir(self):

-        return os.path.join(self.home_dir, "nepi-usr")
+        return os.path.join(self.nepi_home, "nepi-usr")

 
     @property
     def lib_dir(self):
 
     @property
     def lib_dir(self):


@@ -247,7 +275,7 @@ class LinuxNode(ResourceManager):
 
     @property
     def exp_dir(self):
 
     @property
     def exp_dir(self):

-        return os.path.join(self.home_dir, "nepi-exp")
+        return os.path.join(self.nepi_home, "nepi-exp")

 
     @property
     def exp_home(self):
 
     @property
     def exp_home(self):


@@ -255,7 +283,7 @@ class LinuxNode(ResourceManager):
 
     @property
     def node_home(self):
 
     @property
     def node_home(self):

-        return os.path.join(self.exp_home, "node-%d" % self.guid)
+        return os.path.join(self.exp_home, "node-{}".format(self.guid))

 
     @property
     def run_home(self):
 
     @property
     def run_home(self):


@@ -266,29 +294,29 @@ class LinuxNode(ResourceManager):
         if self._os:
             return self._os
 
         if self._os:
             return self._os
 

-        if (not self.get("hostname") or not self.get("username")):
+        if not self.localhost and not self.get("username"):

             msg = "Can't resolve OS, insufficient data "
             self.error(msg)
             raise RuntimeError, msg
 
         out = self.get_os()
 
             msg = "Can't resolve OS, insufficient data "
             self.error(msg)
             raise RuntimeError, msg
 
         out = self.get_os()
 

-        if out.find("Fedora release 8") == 0:
-            self._os = OSType.FEDORA_8
-        elif out.find("Fedora release 12") == 0:
-            self._os = OSType.FEDORA_12
-        elif out.find("Fedora release 14") == 0:
-            self._os = OSType.FEDORA_14
-        elif out.find("Fedora release") == 0:
-            self._os = OSType.FEDORA
-        elif out.find("Debian") == 0: 
+        if out.find("Debian") == 0: 

             self._os = OSType.DEBIAN
         elif out.find("Ubuntu") ==0:
             self._os = OSType.UBUNTU
             self._os = OSType.DEBIAN
         elif out.find("Ubuntu") ==0:
             self._os = OSType.UBUNTU

+        elif out.find("Fedora release") == 0:
+            self._os = OSType.FEDORA
+            if out.find("Fedora release 8") == 0:
+                self._os = OSType.FEDORA_8
+            elif out.find("Fedora release 12") == 0:
+                self._os = OSType.FEDORA_12
+            elif out.find("Fedora release 14") == 0:
+                self._os = OSType.FEDORA_14

         else:
             msg = "Unsupported OS"
             self.error(msg, out)
         else:
             msg = "Unsupported OS"
             self.error(msg, out)

-            raise RuntimeError, "%s - %s " %( msg, out )
+            raise RuntimeError("{} - {} ".format(msg, out))

 
         return self._os
 
 
         return self._os
 


@@ -300,32 +328,32 @@ class LinuxNode(ResourceManager):
         out = ""
         try:
             (out, err), proc = self.execute("cat /etc/issue", 
         out = ""
         try:
             (out, err), proc = self.execute("cat /etc/issue", 

-                    with_lock = True,
-                    blocking = True)
+                                            with_lock = True,
+                                            blocking = True)

         except:
             trace = traceback.format_exc()
         except:
             trace = traceback.format_exc()

-            msg = "Error detecting OS: %s " % trace
+            msg = "Error detecting OS: {} ".format(trace)

             self.error(msg, out, err)
             self.error(msg, out, err)

-        
+    

         return out
 
     @property
     def use_deb(self):
         return out
 
     @property
     def use_deb(self):

-        return self.os in [OSType.DEBIAN, OSType.UBUNTU]
+        return (self.os & (OSType.DEBIAN|OSType.UBUNTU))

 
     @property
     def use_rpm(self):
 
     @property
     def use_rpm(self):

-        return self.os in [OSType.FEDORA_12, OSType.FEDORA_14, OSType.FEDORA_8,
-                OSType.FEDORA]
+        return (self.os & OSType.FEDORA)

 
     @property
     def localhost(self):
 
     @property
     def localhost(self):

-        return self.get("hostname") in ['localhost', '127.0.0.7', '::1']
+        return self.get("hostname") in ['localhost', '127.0.0.1', '::1']

 
     def do_provision(self):
         # check if host is alive
         if not self.is_alive():
 
     def do_provision(self):
         # check if host is alive
         if not self.is_alive():

-            msg = "Deploy failed. Unresponsive node %s" % self.get("hostname")
+            trace = traceback.format_exc()
+            msg = "Deploy failed. Unresponsive node {} -- traceback {}".format(self.get("hostname"), trace)

             self.error(msg)
             raise RuntimeError, msg
 
             self.error(msg)
             raise RuntimeError, msg
 


@@ -336,18 +364,28 @@ class LinuxNode(ResourceManager):
 
         if self.get("cleanHome"):
             self.clean_home()
 
         if self.get("cleanHome"):
             self.clean_home()

- 
+            

         if self.get("cleanExperiment"):
             self.clean_experiment()
         if self.get("cleanExperiment"):
             self.clean_experiment()

-    
-        # Create shared directory structure
-        self.mkdir(self.lib_dir)
-        self.mkdir(self.bin_dir)
-        self.mkdir(self.src_dir)
-        self.mkdir(self.share_dir)
+            
+        # Create shared directory structure and node home directory
+        paths = [self.lib_dir, 
+                 self.bin_dir, 
+                 self.src_dir, 
+                 self.share_dir, 
+                 self.node_home]

 
 

-        # Create experiment node home directory
-        self.mkdir(self.node_home)
+        self.mkdir(paths)
+
+        # Get Public IP address if possible
+        if not self.get("ip"):
+            try:
+                ip = sshfuncs.gethostbyname(self.get("hostname"))
+                self.set("ip", ip)
+            except:
+                if self.get("gateway") is None:
+                    msg = "Local DNS can not resolve hostname {}".format(self.get("hostname"))
+                    self.error(msg)

 
         super(LinuxNode, self).do_provision()
 
 
         super(LinuxNode, self).do_provision()
 


@@ -363,7 +401,7 @@ class LinuxNode(ResourceManager):
         ifaces = self.get_connected(LinuxInterface.get_rtype())
         for iface in ifaces:
             if iface.state < ResourceState.READY:
         ifaces = self.get_connected(LinuxInterface.get_rtype())
         for iface in ifaces:
             if iface.state < ResourceState.READY:

-                self.ec.schedule(reschedule_delay, self.deploy)
+                self.ec.schedule(self.reschedule_delay, self.deploy)

                 return 
 
         super(LinuxNode, self).do_deploy()
                 return 
 
         super(LinuxNode, self).do_deploy()


@@ -374,14 +412,15 @@ class LinuxNode(ResourceManager):
             # Node needs to wait until all associated RMs are released
             # before it can be released
             if rm.state != ResourceState.RELEASED:
             # Node needs to wait until all associated RMs are released
             # before it can be released
             if rm.state != ResourceState.RELEASED:

-                self.ec.schedule(reschedule_delay, self.release)
+                self.ec.schedule(self.reschedule_delay, self.release)

                 return 
 
         tear_down = self.get("tearDown")
         if tear_down:
             self.execute(tear_down)
 
                 return 
 
         tear_down = self.get("tearDown")
         if tear_down:
             self.execute(tear_down)
 

-        self.clean_processes()
+        if self.get("cleanProcessesAfter"):
+            self.clean_processes()

 
         super(LinuxNode, self).do_release()
 
 
         super(LinuxNode, self).do_release()
 


@@ -391,12 +430,38 @@ class LinuxNode(ResourceManager):
 
     def clean_processes(self):
         self.info("Cleaning up processes")
 
     def clean_processes(self):
         self.info("Cleaning up processes")

+
+        if self.localhost:
+            return 

         
         

-        cmd = ("sudo -S killall tcpdump || /bin/true ; " +
-            "sudo -S kill $(ps aux | grep '[n]epi' | awk '{print $2}') || /bin/true ; " +
-            "sudo -S killall -u %s || /bin/true ; " % self.get("username"))
+        if self.get("username") != 'root':
+            cmd = ("sudo -S killall tcpdump || /bin/true ; " +
+                   "sudo -S kill -9 $(ps aux | grep '[.]nepi' | awk '{print $2}') || /bin/true ; " +
+                   "sudo -S killall -u {} || /bin/true ; ".format(self.get("username")))
+        else:
+            if self.state >= ResourceState.READY:
+                import pickle
+                pids = pickle.load(open("/tmp/save.proc", "rb"))
+                pids_temp = dict()
+                ps_aux = "ps aux |awk '{print $2,$11}'"
+                (out, err), proc = self.execute(ps_aux)
+                if len(out) != 0:
+                    for line in out.strip().split("\n"):
+                        parts = line.strip().split(" ")
+                        pids_temp[parts[0]] = parts[1]
+                    kill_pids = set(pids_temp.items()) - set(pids.items())
+                    kill_pids = ' '.join(dict(kill_pids).keys())
+
+                    cmd = ("killall tcpdump || /bin/true ; " +
+                           "kill $(ps aux | grep '[.]nepi' | awk '{print $2}') || /bin/true ; " +
+                           "kill {} || /bin/true ; ".format(kill_pids))
+                else:
+                    cmd = ("killall tcpdump || /bin/true ; " +
+                           "kill $(ps aux | grep '[.]nepi' | awk '{print $2}') || /bin/true ; ")
+            else:
+                cmd = ("killall tcpdump || /bin/true ; " +
+                       "kill $(ps aux | grep '[.]nepi' | awk '{print $2}') || /bin/true ; ")

 
 

-        out = err = ""

         (out, err), proc = self.execute(cmd, retry = 1, with_lock = True)
 
     def clean_home(self):
         (out, err), proc = self.execute(cmd, retry = 1, with_lock = True)
 
     def clean_home(self):


@@ -404,8 +469,8 @@ class LinuxNode(ResourceManager):
         """
         self.info("Cleaning up home")
         
         """
         self.info("Cleaning up home")
         

-        cmd = "cd %s ; find . -maxdepth 1 \( -name 'nepi-usr' -o -name 'nepi-exp' \) -execdir rm -rf {} + " % (
-                self.home_dir )
+        cmd = "cd {} ; find . -maxdepth 1 -name \.nepi -execdir rm -rf {{}} + "\
+              .format(self.home_dir)

 
         return self.execute(cmd, with_lock = True)
 
 
         return self.execute(cmd, with_lock = True)
 


@@ -416,83 +481,81 @@ class LinuxNode(ResourceManager):
         """
         self.info("Cleaning up experiment files")
         
         """
         self.info("Cleaning up experiment files")
         

-        cmd = "cd %s ; find . -maxdepth 1 -name '%s' -execdir rm -rf {} + " % (
-                self.exp_dir,
-                self.ec.exp_id )
-            
+        cmd = "cd {} ; find . -maxdepth 1 -name '{}' -execdir rm -rf {{}} + "\
+              .format(self.exp_dir, self.ec.exp_id)
+        

         return self.execute(cmd, with_lock = True)
 
     def execute(self, command,
         return self.execute(cmd, with_lock = True)
 
     def execute(self, command,

-            sudo = False,
-            stdin = None, 
-            env = None,
-            tty = False,
-            forward_x11 = False,
-            timeout = None,
-            retry = 3,
-            err_on_timeout = True,
-            connect_timeout = 30,
-            strict_host_checking = False,
-            persistent = True,
-            blocking = True,
-            with_lock = False
-            ):
+                sudo = False,
+                env = None,
+                tty = False,
+                forward_x11 = False,
+                retry = 3,
+                connect_timeout = 30,
+                strict_host_checking = False,
+                persistent = True,
+                blocking = True,
+                with_lock = False
+    ):

         """ Notice that this invocation will block until the
         execution finishes. If this is not the desired behavior,
         use 'run' instead."""
 
         if self.localhost:
         """ Notice that this invocation will block until the
         execution finishes. If this is not the desired behavior,
         use 'run' instead."""
 
         if self.localhost:

-            (out, err), proc = execfuncs.lexec(command, 
-                    user = user,
-                    sudo = sudo,
-                    stdin = stdin,
-                    env = env)
+            (out, err), proc = execfuncs.lexec(
+                command, 
+                user = self.get("username"), # still problem with localhost
+                sudo = sudo,
+                env = env)

         else:
             if with_lock:
         else:
             if with_lock:

+                # If the execute command is blocking, we don't want to keep
+                # the node lock. This lock is used to avoid race conditions
+                # when creating the ControlMaster sockets. A more elegant
+                # solution is needed.

                 with self._node_lock:
                     (out, err), proc = sshfuncs.rexec(
                         command, 
                         host = self.get("hostname"),
                         user = self.get("username"),
                         port = self.get("port"),
                 with self._node_lock:
                     (out, err), proc = sshfuncs.rexec(
                         command, 
                         host = self.get("hostname"),
                         user = self.get("username"),
                         port = self.get("port"),

+                        gwuser = self.get("gatewayUser"),
+                        gw = self.get("gateway"),

                         agent = True,
                         sudo = sudo,
                         agent = True,
                         sudo = sudo,

-                        stdin = stdin,

                         identity = self.get("identity"),
                         server_key = self.get("serverKey"),
                         env = env,
                         tty = tty,
                         forward_x11 = forward_x11,
                         identity = self.get("identity"),
                         server_key = self.get("serverKey"),
                         env = env,
                         tty = tty,
                         forward_x11 = forward_x11,

-                        timeout = timeout,

                         retry = retry,
                         retry = retry,

-                        err_on_timeout = err_on_timeout,

                         connect_timeout = connect_timeout,
                         persistent = persistent,
                         blocking = blocking, 
                         strict_host_checking = strict_host_checking
                         connect_timeout = connect_timeout,
                         persistent = persistent,
                         blocking = blocking, 
                         strict_host_checking = strict_host_checking

-                        )
+                    )

             else:
                 (out, err), proc = sshfuncs.rexec(
                     command, 
                     host = self.get("hostname"),
                     user = self.get("username"),
                     port = self.get("port"),
             else:
                 (out, err), proc = sshfuncs.rexec(
                     command, 
                     host = self.get("hostname"),
                     user = self.get("username"),
                     port = self.get("port"),

+                    gwuser = self.get("gatewayUser"),
+                    gw = self.get("gateway"),

                     agent = True,
                     sudo = sudo,
                     agent = True,
                     sudo = sudo,

-                    stdin = stdin,

                     identity = self.get("identity"),
                     server_key = self.get("serverKey"),
                     env = env,
                     tty = tty,
                     forward_x11 = forward_x11,
                     identity = self.get("identity"),
                     server_key = self.get("serverKey"),
                     env = env,
                     tty = tty,
                     forward_x11 = forward_x11,

-                    timeout = timeout,

                     retry = retry,
                     retry = retry,

-                    err_on_timeout = err_on_timeout,

                     connect_timeout = connect_timeout,
                     persistent = persistent,
                     blocking = blocking, 
                     strict_host_checking = strict_host_checking
                     connect_timeout = connect_timeout,
                     persistent = persistent,
                     blocking = blocking, 
                     strict_host_checking = strict_host_checking

-                    )
+                )

 
         return (out, err), proc
 
 
         return (out, err), proc
 


@@ -503,19 +566,20 @@ class LinuxNode(ResourceManager):
             stdout = 'stdout', 
             stderr = 'stderr', 
             sudo = False,
             stdout = 'stdout', 
             stderr = 'stderr', 
             sudo = False,

-            tty = False):
+            tty = False,
+            strict_host_checking = False):

         
         

-        self.debug("Running command '%s'" % command)
+        self.debug("Running command '{}'".format(command))

         
         if self.localhost:
         
         if self.localhost:

-            (out, err), proc = execfuncs.lspawn(command, pidfile, 
-                    stdout = stdout, 
-                    stderr = stderr, 
-                    stdin = stdin, 
-                    home = home, 
-                    create_home = create_home, 
-                    sudo = sudo,
-                    user = user) 
+            (out, err), proc = execfuncs.lspawn(
+                command, pidfile,
+                home = home, 
+                create_home = create_home, 
+                stdin = stdin or '/dev/null',
+                stdout = stdout or '/dev/null',
+                stderr = stderr or '/dev/null',
+                sudo = sudo) 

         else:
             with self._node_lock:
                 (out, err), proc = sshfuncs.rspawn(
         else:
             with self._node_lock:
                 (out, err), proc = sshfuncs.rspawn(


@@ -523,18 +587,21 @@ class LinuxNode(ResourceManager):
                     pidfile = pidfile,
                     home = home,
                     create_home = create_home,
                     pidfile = pidfile,
                     home = home,
                     create_home = create_home,

-                    stdin = stdin if stdin is not None else '/dev/null',
-                    stdout = stdout if stdout else '/dev/null',
-                    stderr = stderr if stderr else '/dev/null',
+                    stdin = stdin or '/dev/null',
+                    stdout = stdout or '/dev/null',
+                    stderr = stderr or '/dev/null',

                     sudo = sudo,
                     host = self.get("hostname"),
                     user = self.get("username"),
                     port = self.get("port"),
                     sudo = sudo,
                     host = self.get("hostname"),
                     user = self.get("username"),
                     port = self.get("port"),

+                    gwuser = self.get("gatewayUser"),
+                    gw = self.get("gateway"),

                     agent = True,
                     identity = self.get("identity"),
                     server_key = self.get("serverKey"),
                     agent = True,
                     identity = self.get("identity"),
                     server_key = self.get("serverKey"),

-                    tty = tty
-                    )
+                    tty = tty,
+                    strict_host_checking = strict_host_checking
+                )

 
         return (out, err), proc
 
 
         return (out, err), proc
 


@@ -548,10 +615,13 @@ class LinuxNode(ResourceManager):
                     host = self.get("hostname"),
                     user = self.get("username"),
                     port = self.get("port"),
                     host = self.get("hostname"),
                     user = self.get("username"),
                     port = self.get("port"),

+                    gwuser = self.get("gatewayUser"),
+                    gw = self.get("gateway"),

                     agent = True,
                     identity = self.get("identity"),
                     agent = True,
                     identity = self.get("identity"),

-                    server_key = self.get("serverKey")
-                    )
+                    server_key = self.get("serverKey"),
+                    strict_host_checking = False
+                )

         
         return pidtuple
 
         
         return pidtuple
 


@@ -561,14 +631,17 @@ class LinuxNode(ResourceManager):
         else:
             with self._node_lock:
                 status = sshfuncs.rstatus(
         else:
             with self._node_lock:
                 status = sshfuncs.rstatus(

-                        pid, ppid,
-                        host = self.get("hostname"),
-                        user = self.get("username"),
-                        port = self.get("port"),
-                        agent = True,
-                        identity = self.get("identity"),
-                        server_key = self.get("serverKey")
-                        )
+                    pid, ppid,
+                    host = self.get("hostname"),
+                    user = self.get("username"),
+                    port = self.get("port"),
+                    gwuser = self.get("gatewayUser"),
+                    gw = self.get("gateway"),
+                    agent = True,
+                    identity = self.get("identity"),
+                    server_key = self.get("serverKey"),
+                    strict_host_checking = False
+                )

            
         return status
     
            
         return status
     


@@ -587,24 +660,29 @@ class LinuxNode(ResourceManager):
                         host = self.get("hostname"),
                         user = self.get("username"),
                         port = self.get("port"),
                         host = self.get("hostname"),
                         user = self.get("username"),
                         port = self.get("port"),

+                        gwuser = self.get("gatewayUser"),
+                        gw = self.get("gateway"),

                         agent = True,
                         sudo = sudo,
                         identity = self.get("identity"),
                         agent = True,
                         sudo = sudo,
                         identity = self.get("identity"),

-                        server_key = self.get("serverKey")
-                        )
+                        server_key = self.get("serverKey"),
+                        strict_host_checking = False
+                    )

 
         return (out, err), proc
 
     def copy(self, src, dst):
         if self.localhost:
 
         return (out, err), proc
 
     def copy(self, src, dst):
         if self.localhost:

-            (out, err), proc = execfuncs.lcopy(source, dest, 
-                    recursive = True,
-                    strict_host_checking = False)
+            (out, err), proc = execfuncs.lcopy(
+                src, dst, 
+                recursive = True)

         else:
             with self._node_lock:
                 (out, err), proc = sshfuncs.rcopy(
                     src, dst, 
                     port = self.get("port"),
         else:
             with self._node_lock:
                 (out, err), proc = sshfuncs.rcopy(
                     src, dst, 
                     port = self.get("port"),

+                    gwuser = self.get("gatewayUser"),
+                    gw = self.get("gateway"),

                     identity = self.get("identity"),
                     server_key = self.get("serverKey"),
                     recursive = True,
                     identity = self.get("identity"),
                     server_key = self.get("serverKey"),
                     recursive = True,


@@ -612,14 +690,21 @@ class LinuxNode(ResourceManager):
 
         return (out, err), proc
 
 
         return (out, err), proc
 

-    def upload(self, src, dst, text = False, overwrite = True):
+    def upload(self, src, dst, text = False, overwrite = True,
+               raise_on_error = True):

         """ Copy content to destination
 
         """ Copy content to destination
 

-           src  content to copy. Can be a local file, directory or a list of files
+        src  string with the content to copy. Can be:
+            - plain text
+            - a string with the path to a local file
+            - a string with a semi-colon separeted list of local files
+            - a string with a local directory

 
 

-           dst  destination path on the remote host (remote is always self.host)
+        dst  string with destination path on the remote host (remote is 
+            always self.host)

 
 

-           text src is text input, it must be stored into a temp file before uploading
+        text src is text input, it must be stored into a temp file before 
+        uploading

         """
         # If source is a string input 
         f = None
         """
         # If source is a string input 
         f = None


@@ -632,29 +717,50 @@ class LinuxNode(ResourceManager):
             src = f.name
 
         # If dst files should not be overwritten, check that the files do not
             src = f.name
 
         # If dst files should not be overwritten, check that the files do not

-        # exits already 
+        # exits already
+        if isinstance(src, str):
+            src = map(str.strip, src.split(";"))
+    

         if overwrite == False:
             src = self.filter_existing_files(src, dst)
             if not src:
         if overwrite == False:
             src = self.filter_existing_files(src, dst)
             if not src:

-                return ("", ""), None 
+                return ("", ""), None

 
         if not self.localhost:
             # Build destination as <user>@<server>:<path>
 
         if not self.localhost:
             # Build destination as <user>@<server>:<path>

-            dst = "%s@%s:%s" % (self.get("username"), self.get("hostname"), dst)
+            dst = "{}@{}:{}".format(self.get("username"), self.get("hostname"), dst)

 
 

-        result = self.copy(src, dst)
+        ((out, err), proc) = self.copy(src, dst)

 
         # clean up temp file
         if f:
             os.remove(f.name)
 
 
         # clean up temp file
         if f:
             os.remove(f.name)
 

-        return result
+        if err:
+            msg = " Failed to upload files - src: {} dst: {}".format(";".join(src), dst)
+            self.error(msg, out, err)
+            
+            msg = "{} out: {} err: {}".format(msg, out, err)
+            if raise_on_error:
+                raise RuntimeError, msg
+
+        return ((out, err), proc)

 
 

-    def download(self, src, dst):
+    def download(self, src, dst, raise_on_error = True):

         if not self.localhost:
             # Build destination as <user>@<server>:<path>
         if not self.localhost:
             # Build destination as <user>@<server>:<path>

-            src = "%s@%s:%s" % (self.get("username"), self.get("hostname"), src)
-        return self.copy(src, dst)
+            src = "{}@{}:{}".format(self.get("username"), self.get("hostname"), src)
+
+        ((out, err), proc) = self.copy(src, dst)
+
+        if err:
+            msg = " Failed to download files - src: {} dst: {}".format(";".join(src), dst) 
+            self.error(msg, out, err)
+
+            if raise_on_error:
+                raise RuntimeError, msg
+
+        return ((out, err), proc)

 
     def install_packages_command(self, packages):
         command = ""
 
     def install_packages_command(self, packages):
         command = ""


@@ -669,7 +775,9 @@ class LinuxNode(ResourceManager):
 
         return command
 
 
         return command
 

-    def install_packages(self, packages, home, run_home = None):
+    def install_packages(self, packages, home,
+                         run_home = None,
+                         raise_on_error = True):

         """ Install packages in the Linux host.
 
         'home' is the directory to upload the package installation script.
         """ Install packages in the Linux host.
 
         'home' is the directory to upload the package installation script.


@@ -680,17 +788,18 @@ class LinuxNode(ResourceManager):
         run_home = run_home or home
 
         (out, err), proc = self.run_and_wait(command, run_home, 
         run_home = run_home or home
 
         (out, err), proc = self.run_and_wait(command, run_home, 

-            shfile = os.path.join(home, "instpkg.sh"),
-            pidfile = "instpkg_pidfile",
-            ecodefile = "instpkg_exitcode",
-            stdout = "instpkg_stdout", 
-            stderr = "instpkg_stderr",
-            overwrite = False,
-            raise_on_error = True)
+                                             shfile = os.path.join(home, "instpkg.sh"),
+                                             pidfile = "instpkg_pidfile",
+                                             ecodefile = "instpkg_exitcode",
+                                             stdout = "instpkg_stdout", 
+                                             stderr = "instpkg_stderr",
+                                             overwrite = False,
+                                             raise_on_error = raise_on_error)

 
         return (out, err), proc 
 
 
         return (out, err), proc 
 

-    def remove_packages(self, packages, home, run_home = None):
+    def remove_packages(self, packages, home, run_home = None,
+                        raise_on_error = True):

         """ Uninstall packages from the Linux host.
 
         'home' is the directory to upload the package un-installation script.
         """ Uninstall packages from the Linux host.
 
         'home' is the directory to upload the package un-installation script.


@@ -708,37 +817,55 @@ class LinuxNode(ResourceManager):
         run_home = run_home or home
 
         (out, err), proc = self.run_and_wait(command, run_home, 
         run_home = run_home or home
 
         (out, err), proc = self.run_and_wait(command, run_home, 

-            shfile = os.path.join(home, "rmpkg.sh"),
-            pidfile = "rmpkg_pidfile",
-            ecodefile = "rmpkg_exitcode",
-            stdout = "rmpkg_stdout", 
-            stderr = "rmpkg_stderr",
-            overwrite = False,
-            raise_on_error = True)
-         
+                                             shfile = os.path.join(home, "rmpkg.sh"),
+                                             pidfile = "rmpkg_pidfile",
+                                             ecodefile = "rmpkg_exitcode",
+                                             stdout = "rmpkg_stdout", 
+                                             stderr = "rmpkg_stderr",
+                                             overwrite = False,
+                                             raise_on_error = raise_on_error)
+        

         return (out, err), proc 
 
         return (out, err), proc 
 

-    def mkdir(self, path, clean = False):
+    def mkdir(self, paths, clean = False):
+        """ Paths is either a single remote directory path to create,
+        or a list of directories to create.
+        """

         if clean:
         if clean:

-            self.rmdir(path)
+            self.rmdir(paths)

 
 

-        return self.execute("mkdir -p %s" % path, with_lock = True)
+        if isinstance(paths, str):
+            paths = [paths]

 
 

-    def rmdir(self, path):
-        return self.execute("rm -rf %s" % path, with_lock = True)
-        
+        cmd = " ; ".join(["mkdir -p {}".format(path) for path in paths])
+
+        return self.execute(cmd, with_lock = True)
+
+    def rmdir(self, paths):
+        """ Paths is either a single remote directory path to delete,
+        or a list of directories to delete.
+        """
+
+        if isinstance(paths, str):
+            paths = [paths]
+
+        cmd = " ; ".join(map(lambda path: "rm -rf {}".format(path), paths))
+
+        return self.execute(cmd, with_lock = True)
+    

     def run_and_wait(self, command, home, 
     def run_and_wait(self, command, home, 

-            shfile = "cmd.sh",
-            env = None,
-            overwrite = True,
-            pidfile = "pidfile", 
-            ecodefile = "exitcode", 
-            stdin = None, 
-            stdout = "stdout", 
-            stderr = "stderr", 
-            sudo = False,
-            tty = False,
-            raise_on_error = False):
+                     shfile="cmd.sh",
+                     env=None,
+                     overwrite=True,
+                     wait_run=True,
+                     pidfile="pidfile", 
+                     ecodefile="exitcode", 
+                     stdin=None, 
+                     stdout="stdout", 
+                     stderr="stderr", 
+                     sudo=False,
+                     tty=False,
+                     raise_on_error=True):

         """
         Uploads the 'command' to a bash script in the host.
         Then runs the script detached in background in the host, and
         """
         Uploads the 'command' to a bash script in the host.
         Then runs the script detached in background in the host, and


@@ -749,53 +876,54 @@ class LinuxNode(ResourceManager):
             shfile = os.path.join(home, shfile)
 
         self.upload_command(command, 
             shfile = os.path.join(home, shfile)
 
         self.upload_command(command, 

-            shfile = shfile, 
-            ecodefile = ecodefile, 
-            env = env,
-            overwrite = overwrite)
+                            shfile = shfile, 
+                            ecodefile = ecodefile, 
+                            env = env,
+                            overwrite = overwrite)

 
 

-        command = "bash %s" % shfile
+        command = "bash {}".format(shfile)

         # run command in background in remote host
         (out, err), proc = self.run(command, home, 
         # run command in background in remote host
         (out, err), proc = self.run(command, home, 

-                pidfile = pidfile,
-                stdin = stdin, 
-                stdout = stdout, 
-                stderr = stderr, 
-                sudo = sudo,
-                tty = tty)
+                                    pidfile = pidfile,
+                                    stdin = stdin, 
+                                    stdout = stdout, 
+                                    stderr = stderr, 
+                                    sudo = sudo,
+                                    tty = tty)

 
         # check no errors occurred
         if proc.poll():
 
         # check no errors occurred
         if proc.poll():

-            msg = " Failed to run command '%s' " % command
+            msg = " Failed to run command '{}' ".format(command)

             self.error(msg, out, err)
             if raise_on_error:
                 raise RuntimeError, msg
 
         # Wait for pid file to be generated
         pid, ppid = self.wait_pid(
             self.error(msg, out, err)
             if raise_on_error:
                 raise RuntimeError, msg
 
         # Wait for pid file to be generated
         pid, ppid = self.wait_pid(

-                home = home, 
-                pidfile = pidfile, 
-                raise_on_error = raise_on_error)
+            home = home, 
+            pidfile = pidfile, 
+            raise_on_error = raise_on_error)

 
 

-        # wait until command finishes to execute
-        self.wait_run(pid, ppid)
-      
-        (eout, err), proc = self.check_errors(home,
-            ecodefile = ecodefile,
-            stderr = stderr)
+        if wait_run:
+            # wait until command finishes to execute
+            self.wait_run(pid, ppid)
+            
+            (eout, err), proc = self.check_errors(home,
+                                                  ecodefile = ecodefile,
+                                                  stderr = stderr)

 
 

-        # Out is what was written in the stderr file
-        if err:
-            msg = " Failed to run command '%s' " % command
-            self.error(msg, eout, err)
+            # Out is what was written in the stderr file
+            if err:
+                msg = " Failed to run command '{}' ".format(command)
+                self.error(msg, eout, err)

 
 

-            if raise_on_error:
-                raise RuntimeError, msg
+                if raise_on_error:
+                    raise RuntimeError, msg

 
         (out, oerr), proc = self.check_output(home, stdout)
         
         return (out, err), proc
 
         (out, oerr), proc = self.check_output(home, stdout)
         
         return (out, err), proc

-
+        

     def exitcode(self, home, ecodefile = "exitcode"):
         """
         Get the exit code of an application.
     def exitcode(self, home, ecodefile = "exitcode"):
         """
         Get the exit code of an application.


@@ -819,22 +947,20 @@ class LinuxNode(ResourceManager):
         return ExitCode.ERROR
 
     def upload_command(self, command, 
         return ExitCode.ERROR
 
     def upload_command(self, command, 

-            shfile = "cmd.sh",
-            ecodefile = "exitcode",
-            overwrite = True,
-            env = None):
+                       shfile="cmd.sh",
+                       ecodefile="exitcode",
+                       overwrite=True,
+                       env=None):

         """ Saves the command as a bash script file in the remote host, and
         forces to save the exit code of the command execution to the ecodefile
         """
 
         if not (command.strip().endswith(";") or command.strip().endswith("&")):
             command += ";"
         """ Saves the command as a bash script file in the remote host, and
         forces to save the exit code of the command execution to the ecodefile
         """
 
         if not (command.strip().endswith(";") or command.strip().endswith("&")):
             command += ";"

-      
+            

         # The exit code of the command will be stored in ecodefile
         # The exit code of the command will be stored in ecodefile

-        command = " { %(command)s } ; echo $? > %(ecodefile)s ;" % {
-                'command': command,
-                'ecodefile': ecodefile,
-                } 
+        command = " {{ {command} }} ; echo $? > {ecodefile} ;"\
+                  .format(command=command, ecodefile=ecodefile)

 
         # Export environment
         environ = self.format_environment(env)
 
         # Export environment
         environ = self.format_environment(env)


@@ -842,9 +968,9 @@ class LinuxNode(ResourceManager):
         # Add environ to command
         command = environ + command
 
         # Add environ to command
         command = environ + command
 

-        return self.upload(command, shfile, text = True, overwrite = overwrite)
+        return self.upload(command, shfile, text=True, overwrite=overwrite)

 
 

-    def format_environment(self, env, inline = False):
+    def format_environment(self, env, inline=False):

         """ Formats the environment variables for a command to be executed
         either as an inline command
         (i.e. export PYTHONPATH=src/..; export LALAL= ..;python script.py) or 
         """ Formats the environment variables for a command to be executed
         either as an inline command
         (i.e. export PYTHONPATH=src/..; export LALAL= ..;python script.py) or 


@@ -856,11 +982,11 @@ class LinuxNode(ResourceManager):
         env = re.sub(r'\s+', ' ', env.strip())
 
         sep = ";" if inline else "\n"
         env = re.sub(r'\s+', ' ', env.strip())
 
         sep = ";" if inline else "\n"

-        return sep.join(map(lambda e: " export %s" % e, env.split(" "))) + sep 
+        return sep.join([" export {}".format(e) for e in env.split(" ")]) + sep 

 
     def check_errors(self, home, 
 
     def check_errors(self, home, 

-            ecodefile = "exitcode", 
-            stderr = "stderr"):
+                     ecodefile = "exitcode", 
+                     stderr = "stderr"):

         """ Checks whether errors occurred while running a command.
         It first checks the exit code for the command, and only if the
         exit code is an error one it returns the error output.
         """ Checks whether errors occurred while running a command.
         It first checks the exit code for the command, and only if the
         exit code is an error one it returns the error output.


@@ -873,7 +999,7 @@ class LinuxNode(ResourceManager):
         ecode = self.exitcode(home, ecodefile)
 
         if ecode in [ ExitCode.CORRUPTFILE, ExitCode.ERROR ]:
         ecode = self.exitcode(home, ecodefile)
 
         if ecode in [ ExitCode.CORRUPTFILE, ExitCode.ERROR ]:

-            err = "Error retrieving exit code status from file %s/%s" % (home, ecodefile)
+            err = "Error retrieving exit code status from file {}/{}".format(home, ecodefile)

         elif ecode > 0 or ecode == ExitCode.FILENOTFOUND:
             # The process returned an error code or didn't exist. 
             # Check standard error.
         elif ecode > 0 or ecode == ExitCode.FILENOTFOUND:
             # The process returned an error code or didn't exist. 
             # Check standard error.


@@ -884,9 +1010,9 @@ class LinuxNode(ResourceManager):
             # (cat returns 1 for error "No such file or directory")
             if ecode == ExitCode.FILENOTFOUND and proc.poll() == 1: 
                 err = "" 
             # (cat returns 1 for error "No such file or directory")
             if ecode == ExitCode.FILENOTFOUND and proc.poll() == 1: 
                 err = "" 

-            
+                

         return ("", err), proc
         return ("", err), proc

- 
+    

     def wait_pid(self, home, pidfile = "pidfile", raise_on_error = False):
         """ Waits until the pid file for the command is generated, 
             and returns the pid and ppid of the process """
     def wait_pid(self, home, pidfile = "pidfile", raise_on_error = False):
         """ Waits until the pid file for the command is generated, 
             and returns the pid and ppid of the process """


@@ -903,10 +1029,9 @@ class LinuxNode(ResourceManager):
                 time.sleep(delay)
                 delay = delay * 1.5
         else:
                 time.sleep(delay)
                 delay = delay * 1.5
         else:

-            msg = " Failed to get pid for pidfile %s/%s " % (
-                    home, pidfile )
+            msg = " Failed to get pid for pidfile {}/{} ".format(home, pidfile )

             self.error(msg)
             self.error(msg)

-            
+    

             if raise_on_error:
                 raise RuntimeError, msg
 
             if raise_on_error:
                 raise RuntimeError, msg
 


@@ -934,8 +1059,8 @@ class LinuxNode(ResourceManager):
 
     def check_output(self, home, filename):
         """ Retrives content of file """
 
     def check_output(self, home, filename):
         """ Retrives content of file """

-        (out, err), proc = self.execute("cat %s" % 
-            os.path.join(home, filename), retry = 1, with_lock = True)
+        (out, err), proc = self.execute(
+            "cat {}".format(os.path.join(home, filename)), retry = 1, with_lock = True)

         return (out, err), proc
 
     def is_alive(self):
         return (out, err), proc
 
     def is_alive(self):


@@ -953,20 +1078,21 @@ class LinuxNode(ResourceManager):
         # until the result is not empty string
         try:
             (out, err), proc = self.execute("echo 'ALIVE'",
         # until the result is not empty string
         try:
             (out, err), proc = self.execute("echo 'ALIVE'",

-                    blocking = True,
-                    with_lock = True)
-    
+                                            blocking = True,
+                                            with_lock = True)
+            

             if out.find("ALIVE") > -1:
                 return True
         except:
             trace = traceback.format_exc()
             if out.find("ALIVE") > -1:
                 return True
         except:
             trace = traceback.format_exc()

-            msg = "Unresponsive host. Error reaching host: %s " % trace
+            msg = "Unresponsive host. Error reaching host: {} ".format(trace)

 
         self.error(msg, out, err)
         return False
 
     def find_home(self):
 
         self.error(msg, out, err)
         return False
 
     def find_home(self):

-        """ Retrieves host home directory
+        """ 
+        Retrieves host home directory

         """
         # The underlying SSH layer will sometimes return an empty
         # output (even if the command was executed without errors).
         """
         # The underlying SSH layer will sometimes return an empty
         # output (even if the command was executed without errors).


@@ -975,40 +1101,40 @@ class LinuxNode(ResourceManager):
         msg = "Impossible to retrieve HOME directory"
         try:
             (out, err), proc = self.execute("echo ${HOME}",
         msg = "Impossible to retrieve HOME directory"
         try:
             (out, err), proc = self.execute("echo ${HOME}",

-                    blocking = True,
-                    with_lock = True)
-    
+                                            blocking = True,
+                                            with_lock = True)
+            

             if out.strip() != "":
                 self._home_dir =  out.strip()
         except:
             trace = traceback.format_exc()
             if out.strip() != "":
                 self._home_dir =  out.strip()
         except:
             trace = traceback.format_exc()

-            msg = "Impossible to retrieve HOME directory" % trace
+            msg = "Impossible to retrieve HOME directory {}".format(trace)

 
         if not self._home_dir:
 
         if not self._home_dir:

-            self.error(msg, out, err)
+            self.error(msg)

             raise RuntimeError, msg
 
     def filter_existing_files(self, src, dst):
         """ Removes files that already exist in the Linux host from src list
         """
         # construct a dictionary with { dst: src }
             raise RuntimeError, msg
 
     def filter_existing_files(self, src, dst):
         """ Removes files that already exist in the Linux host from src list
         """
         # construct a dictionary with { dst: src }

-        dests = dict(map(lambda x: ( os.path.join(dst, os.path.basename(x) ),  x ), 
-            src.strip().split(" ") ) ) if src.strip().find(" ") != -1 else dict({dst: src})
+        dests = { os.path.join(dst, os.path.basename(s)) : s for s in src } \
+                if len(src) > 1 else {dst: src[0]}

 
         command = []
         for d in dests.keys():
 
         command = []
         for d in dests.keys():

-            command.append(" [ -f %(dst)s ] && echo '%(dst)s' " % {'dst' : d} )
+            command.append(" [ -f {dst} ] && echo '{dst}' ".format(dst=d) )

 
         command = ";".join(command)
 
         (out, err), proc = self.execute(command, retry = 1, with_lock = True)
 
         command = ";".join(command)
 
         (out, err), proc = self.execute(command, retry = 1, with_lock = True)

-    
+        

         for d in dests.keys():
             if out.find(d) > -1:
                 del dests[d]
 
         if not dests:
         for d in dests.keys():
             if out.find(d) > -1:
                 del dests[d]
 
         if not dests:

-            return ""
+            return []

 
 

-        return " ".join(dests.values())
+        return dests.values()