X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=src%2Fnepi%2Fresources%2Flinux%2Fns3%2Fns3simulation.py;h=0a44d7aebd15cb80ff7b85e30cbc3d7037f7fef9;hb=6285ca51026efb69642eea9dfc7c480e722d84a9;hp=f820eabf57bcc2bc981c5fa467b3bada1b83eded;hpb=5e877924fe463f4016226e4f5c6b91da6134c6a8;p=nepi.git diff --git a/src/nepi/resources/linux/ns3/ns3simulation.py b/src/nepi/resources/linux/ns3/ns3simulation.py index f820eabf..0a44d7ae 100644 --- a/src/nepi/resources/linux/ns3/ns3simulation.py +++ b/src/nepi/resources/linux/ns3/ns3simulation.py @@ -3,9 +3,8 @@ # Copyright (C) 2014 INRIA # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation; # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -20,19 +19,21 @@ from nepi.execution.attribute import Attribute, Flags, Types from nepi.execution.trace import Trace, TraceAttr from nepi.execution.resource import ResourceManager, clsinit_copy, \ - ResourceState, reschedule_delay + ResourceState, ResourceFactory from nepi.resources.linux.application import LinuxApplication from nepi.util.timefuncs import tnow, tdiffsec from nepi.resources.ns3.ns3simulation import NS3Simulation -from nepi.resources.ns3.ns3wrapper import GLOBAL_VALUE_UUID +from nepi.resources.ns3.ns3wrapper import SIMULATOR_UUID, GLOBAL_VALUE_UUID, \ + IPV4_GLOBAL_ROUTING_HELPER_UUID from nepi.resources.linux.ns3.ns3client import LinuxNS3Client import os import time +import threading @clsinit_copy class LinuxNS3Simulation(LinuxApplication, NS3Simulation): - _rtype = "LinuxNS3Simulation" + _rtype = "linux::ns3::Simulation" @classmethod def _register_attributes(cls): @@ -61,10 +62,6 @@ class LinuxNS3Simulation(LinuxApplication, NS3Simulation): type = Types.Bool, flags = Flags.Design) - stop_time = Attribute("stopTime", - "Time to stop the simulation", - flags = Flags.Design) - ns_log = Attribute("nsLog", "NS_LOG environment variable. " \ " Will only generate output if ns-3 is compiled in DEBUG mode. ", @@ -75,12 +72,60 @@ class LinuxNS3Simulation(LinuxApplication, NS3Simulation): type = Types.Bool, flags = Flags.Design) + enable_dump = Attribute("enableDump", + "Enable dumping the remote executed ns-3 commands to a script " + "in order to later reproduce and debug the experiment", + type = Types.Bool, + default = False, + flags = Flags.Design) + + build_mode = Attribute("buildMode", + "Mode used to build ns-3 with waf. One if: debug, release, oprimized ", + default = "optimized", + allowed = ["debug", "release", "optimized"], + type = Types.Enumerate, + flags = Flags.Design) + + ns3_version = Attribute("ns3Version", + "Version of ns-3 to install from nsam repo", + default = "ns-3.20", + #default = "ns-3-dev", + flags = Flags.Design) + + pybindgen_version = Attribute("pybindgenVersion", + "Version of pybindgen to install from bazar repo", + default = "868", + #default = "876", + flags = Flags.Design) + + dce_version = Attribute("dceVersion", + "Version of dce to install from nsam repo (tag branch for repo)", + #default = "dce-1.3", + default = "dce-dev", + flags = Flags.Design) + + populate_routing_tables = Attribute("populateRoutingTables", + "Invokes Ipv4GlobalRoutingHelper.PopulateRoutingTables() ", + default = False, + type = Types.Bool, + flags = Flags.Design) + + stoptime = Attribute("StopTime", + "Time at which the simulation will stop", + flags = Flags.Design) + cls._register_attribute(impl_type) cls._register_attribute(sched_type) cls._register_attribute(check_sum) - cls._register_attribute(stop_time) cls._register_attribute(ns_log) + cls._register_attribute(enable_dump) cls._register_attribute(verbose) + cls._register_attribute(build_mode) + cls._register_attribute(ns3_version) + cls._register_attribute(pybindgen_version) + cls._register_attribute(dce_version) + cls._register_attribute(populate_routing_tables) + cls._register_attribute(stoptime) def __init__(self, ec, guid): LinuxApplication.__init__(self, ec, guid) @@ -88,7 +133,9 @@ class LinuxNS3Simulation(LinuxApplication, NS3Simulation): self._client = None self._home = "ns3-simu-%s" % self.guid - self._socket_name = "ns3simu-%s" % os.urandom(8).encode('hex') + self._socket_name = "ns3-%s.sock" % os.urandom(4).encode('hex') + self._enable_dce = None + self._dce_helper = None @property def socket_name(self): @@ -98,15 +145,19 @@ class LinuxNS3Simulation(LinuxApplication, NS3Simulation): def remote_socket(self): return os.path.join(self.run_home, self.socket_name) - @property - def local_socket(self): - if self.node.get('hostname') in ['localhost', '127.0.0.01']: - return self.remote_socket - - return os.path.join("/", "tmp", self.socket_name) - def trace(self, name, attr = TraceAttr.ALL, block = 512, offset = 0): - self._client.flush() + # stout needs to get flushed on the ns-3 server side, else we will + # get an empty stream. We try twice to retrieve the stream + # if we get empty stdout since the stream might not be + # flushed immediately. + if name.endswith("stdout"): + self._client.flush() + result = LinuxApplication.trace(self, name, attr, block, offset) + if result: + return result + # Let the stream be flushed + time.sleep(1) + return LinuxApplication.trace(self, name, attr, block, offset) def upload_sources(self): @@ -120,6 +171,14 @@ class LinuxNS3Simulation(LinuxApplication, NS3Simulation): os.path.join(self.node.src_dir, "ns3wrapper", "ns3wrapper.py"), overwrite = False) + # upload ns3 wrapper debug python script + ns3_wrapper_debug = os.path.join(os.path.dirname(__file__), "..", "..", "ns3", + "ns3wrapper_debug.py") + + self.node.upload(ns3_wrapper_debug, + os.path.join(self.node.src_dir, "ns3wrapper", "ns3wrapper_debug.py"), + overwrite = False) + # upload ns3_server python script ns3_server = os.path.join(os.path.dirname(__file__), "..", "..", "ns3", "ns3server.py") @@ -137,6 +196,17 @@ class LinuxNS3Simulation(LinuxApplication, NS3Simulation): os.path.join(self.node.src_dir, "%s.tar.gz" % self.pygccxml_version), overwrite = False) + # Upload user defined ns-3 sources + self.node.mkdir(os.path.join(self.node.src_dir, "ns-3")) + src_dir = os.path.join(self.node.src_dir, "ns-3") + + super(LinuxNS3Simulation, self).upload_sources(src_dir = src_dir) + + def upload_extra_sources(self, sources = None, src_dir = None): + return super(LinuxNS3Simulation, self).upload_sources( + sources = sources, + src_dir = src_dir) + def upload_start_command(self): command = self.get("command") env = self.get("env") @@ -158,32 +228,31 @@ class LinuxNS3Simulation(LinuxApplication, NS3Simulation): # Run the ns3wrapper self._run_in_background() + # Wait until the remote socket is created + self.wait_remote_socket() + def configure(self): - if self._attrs.get("simulatorImplementationType").has_changed(): + if self.has_changed("simulatorImplementationType"): simu_type = self.get("simulatorImplementationType") stype = self.create("StringValue", simu_type) self.invoke(GLOBAL_VALUE_UUID, "Bind", "SimulatorImplementationType", stype) - if self._attrs.get("checksumEnabled").has_changed(): + if self.has_changed("checksumEnabled"): check_sum = self.get("checksumEnabled") btrue = self.create("BooleanValue", check_sum) self.invoke(GLOBAL_VALUE_UUID, "Bind", "ChecksumEnabled", btrue) - if self._attrs.get("schedulerType").has_changed(): + if self.has_changed("schedulerType"): sched_type = self.get("schedulerType") stype = self.create("StringValue", sched_type) self.invoke(GLOBAL_VALUE_UUID, "Bind", "SchedulerType", btrue) - if self._attrs.get("stopTime").has_changed(): - stop_time = self.get("stopTime") - self.stop(time = stop_time) - def do_deploy(self): if not self.node or self.node.state < ResourceState.READY: self.debug("---- RESCHEDULING DEPLOY ---- node state %s " % self.node.state ) # ccnd needs to wait until node is deployed and running - self.ec.schedule(reschedule_delay, self.deploy) + self.ec.schedule(self.reschedule_delay, self.deploy) else: if not self.get("command"): self.set("command", self._start_command) @@ -191,6 +260,18 @@ class LinuxNS3Simulation(LinuxApplication, NS3Simulation): if not self.get("depends"): self.set("depends", self._dependencies) + if self.get("sources"): + sources = self.get("sources") + source = sources.split(" ")[0] + basename = os.path.basename(source) + version = ( basename.strip().replace(".tar.gz", "") + .replace(".tar","") + .replace(".gz","") + .replace(".zip","") ) + + self.set("ns3Version", version) + self.set("sources", source) + if not self.get("build"): self.set("build", self._build) @@ -205,15 +286,6 @@ class LinuxNS3Simulation(LinuxApplication, NS3Simulation): # Create client self._client = LinuxNS3Client(self) - - # Wait until local socket is created - for i in [1, 5, 15, 30, 60]: - if os.path.exists(self.local_socket): - break - time.sleep(i) - - if not os.path.exists(self.local_socket): - raise RuntimeError("Problem starting socat") self.configure() @@ -226,22 +298,29 @@ class LinuxNS3Simulation(LinuxApplication, NS3Simulation): self.info("Starting") if self.state == ResourceState.READY: - self._client.start() + if self.get("populateRoutingTables") == True: + self.invoke(IPV4_GLOBAL_ROUTING_HELPER_UUID, "PopulateRoutingTables") + + time = self.get("StopTime") + if time: + self._client.stop(time=time) + + self._client.start() self.set_started() else: msg = " Failed to execute command '%s'" % command self.error(msg, out, err) - raise RuntimeError, msg + raise RuntimeError(msg) def do_stop(self): """ Stops simulation execution """ if self.state == ResourceState.STARTED: - self._client.stop() - self._client.shutdown() - LinuxApplication.do_stop(self) + if not self.get("StopTime"): + self._client.stop() + self.set_stopped() def do_release(self): self.info("Releasing resource") @@ -251,19 +330,70 @@ class LinuxNS3Simulation(LinuxApplication, NS3Simulation): self.node.execute(tear_down) self.do_stop() + self._client.shutdown() + LinuxApplication.do_stop(self) super(LinuxApplication, self).do_release() + @property + def state(self): + super(LinuxApplication, self).state + + if self._state == ResourceState.STARTED: + try: + is_finished = self.invoke(SIMULATOR_UUID, "isFinished") + + if is_finished: + self.set_stopped() + except: + msg = "Simulator failed. Can not retrieve state" + out = "" + + import traceback + err = traceback.format_exc() + self.error(msg, out, err) + self.do_fail() + + return self._state + + @property + def enable_dce(self): + if self._enable_dce is None: + from nepi.resources.ns3.ns3dceapplication import NS3BaseDceApplication + rclass = ResourceFactory.get_resource_type( + NS3BaseDceApplication.get_rtype()) + + self._enable_dce = False + for guid in self.ec.resources: + rm = self.ec.get_resource(guid) + if isinstance(rm, rclass): + self._enable_dce = True + + from nepi.resources.ns3.ns3dcehelper import NS3DceHelper + self._dce_helper = NS3DceHelper(self) + break + + return self._enable_dce + + @property + def dce_helper(self): + return self._dce_helper + @property def _start_command(self): command = [] - command.append("PYTHONPATH=$PYTHONPATH:${SRC}/ns3wrapper/") - command.append("python ${SRC}/ns3wrapper/ns3server.py -S %s" % self.remote_socket ) + command.append("PYTHONPATH=$PYTHONPATH:${SRC}/ns3wrapper/") + + command.append("python ${SRC}/ns3wrapper/ns3server.py -S %s" % \ + os.path.basename(self.remote_socket) ) ns_log = self.get("nsLog") if ns_log: - command.append("-L %s" % ns_log) + command.append("-L '%s'" % ns_log) + + if self.get("enableDump"): + command.append("-D") if self.get("verbose"): command.append("-v") @@ -274,35 +404,151 @@ class LinuxNS3Simulation(LinuxApplication, NS3Simulation): @property def _dependencies(self): if self.node.use_rpm: - return ( " gcc gcc-c++ python python-devel mercurial bzr tcpdump socat gccxml") + return ( " gcc gcc-c++ python python-devel mercurial bzr tcpdump socat gccxml unzip") elif self.node.use_deb: - return ( " gcc g++ python python-dev mercurial bzr tcpdump socat gccxml python-pygccxml") + return ( " gcc g++ python python-dev mercurial bzr tcpdump socat gccxml python-pygccxml unzip") return "" @property def ns3_repo(self): - return "http://code.nsnam.org" + return "http://code.nsnam.org" @property - def ns3_version(self): - return "ns-3.19" + def pygccxml_version(self): + return "pygccxml-1.0.0" @property - def pybindgen_version(self): - return "834" + def dce_repo(self): + return "http://code.nsnam.org/ns-3-dce" + #eturn "http://code.nsnam.org/epmancini/ns-3-dce" @property - def pygccxml_version(self): - return "pygccxml-1.0.0" + def dce_version(self): + dce_version = self.get("dceVersion") + return dce_version or "dce-dev" + + @property + def ns3_build_location(self): + location = "${BIN}/ns-3/%(ns3_version)s%(dce_version)s/%(build_mode)s/build" \ + % { + "ns3_version": self.get("ns3Version"), + "dce_version": "-%s" % self.get("dceVersion") \ + if self.enable_dce else "", + "build_mode": self.get("buildMode"), + } + + return location + + + @property + def ns3_src_location(self): + location = "${SRC}/ns-3/%(ns3_version)s" \ + % { + "ns3_version": self.get("ns3Version"), + } + + return location + + @property + def dce_src_location(self): + location = "${SRC}/ns-3-dce/%(dce_version)s" \ + % { + "dce_version": self.get("dceVersion"), + } + + return location + + @property + def _clone_ns3_command(self): + source = self.get("sources") + + if not source: + clone_ns3_cmd = "hg clone %(ns3_repo)s/%(ns3_version)s %(ns3_src)s" \ + % { + "ns3_version": self.get("ns3Version"), + "ns3_repo": self.ns3_repo, + "ns3_src": self.ns3_src_location, + } + else: + if source.find(".tar.gz") > -1: + clone_ns3_cmd = ( + "tar xzf ${SRC}/ns-3/%(basename)s " + " --strip-components=1 -C %(ns3_src)s" + ) % { + "basename": os.path.basename(source), + "ns3_src": self.ns3_src_location, + } + elif source.find(".tar") > -1: + clone_ns3_cmd = ( + "tar xf ${SRC}/ns-3/%(basename)s " + " --strip-components=1 -C %(ns3_src)s" + ) % { + "basename": os.path.basename(source), + "ns3_src": self.ns3_src_location, + } + elif source.find(".zip") > -1: + basename = os.path.basename(source) + bare_basename = basename.replace(".zip", "") + + clone_ns3_cmd = ( + "unzip ${SRC}/ns-3/%(basename)s && " + "mv ${SRC}/ns-3/%(bare_basename)s %(ns3_src)s" + ) % { + "bare_basename": basename_name, + "basename": basename, + "ns3_src": self.ns3_src_location, + } + + return clone_ns3_cmd + + @property + def _clone_dce_command(self): + clone_dce_cmd = " echo 'DCE will not be built' " + + if self.enable_dce: + dce_version = self.dce_version + dce_tag = "" + if dce_version != "dce-dev": + dce_tag = "-r %s" % dce_version + + clone_dce_cmd = ( + # DCE installation + # Test if dce is alredy cloned + " ( " + " ( " + " ( test -d %(dce_src)s ) " + " && echo 'dce binaries found, nothing to do'" + " ) " + " ) " + " || " + # Get dce source code + " ( " + " mkdir -p %(dce_src)s && " + " hg clone %(dce_repo)s %(dce_tag)s %(dce_src)s" + " ) " + ) % { + "dce_repo": self.dce_repo, + "dce_tag": dce_tag, + "dce_src": self.dce_src_location, + } + + return clone_dce_cmd @property def _build(self): - return ( - # Test if ns-3 is alredy installed + # If the user defined local sources for ns-3, we uncompress the sources + # on the remote sources directory. Else we clone ns-3 from the official repo. + clone_ns3_cmd = self._clone_ns3_command + clone_dce_cmd = self._clone_dce_command + + ns3_build_cmd = ( + # NS3 installation + "( " " ( " - " (( " - " ( test -d ${SRC}/%(ns3_version)s ) || (test -d ${NS3BINDINGS:='None'} && test -d ${NS3LIBRARIES:='None'}) ) && " - " echo 'binaries found, nothing to do' )" + # Test if ns-3 is alredy cloned + " ((( test -d %(ns3_src)s ) || " + " ( test -d ${NS3BINDINGS:='None'} && test -d ${NS3LIBRARIES:='None'})) " + " && echo 'ns-3 binaries found, nothing to do' )" " ) " " || " # If not, install ns-3 and its dependencies @@ -317,6 +563,7 @@ class LinuxNS3Simulation(LinuxApplication, NS3Simulation): " ( " " tar xf ${SRC}/%(pygccxml_version)s.tar.gz -C ${SRC} && " " cd ${SRC}/%(pygccxml_version)s && " + " python setup.py build && " " sudo -S python setup.py install " " ) " " ) " @@ -324,66 +571,166 @@ class LinuxNS3Simulation(LinuxApplication, NS3Simulation): " && " " ( " " ( " - " test -d ${BIN}/pybindgen && " + " test -d ${SRC}/pybindgen/%(pybindgen_version)s && " " echo 'binaries found, nothing to do' " " ) " " || " # If not, clone and build " ( cd ${SRC} && " - " bzr checkout lp:pybindgen -r %(pybindgen_version)s && " - " cd ${SRC}/pybindgen && " + " mkdir -p ${SRC}/pybindgen && " + " bzr checkout lp:pybindgen -r %(pybindgen_version)s ${SRC}/pybindgen/%(pybindgen_version)s && " + " cd ${SRC}/pybindgen/%(pybindgen_version)s && " " ./waf configure && " " ./waf " " ) " " ) " - " && " - # Clone and build ns-3 + " && " + # Get ns-3 source code " ( " - " hg clone %(ns3_repo)s/%(ns3_version)s ${SRC}/%(ns3_version)s && " - " cd ${SRC}/%(ns3_version)s && " - " ./waf configure -d optimized && " - " ./waf " - " ) " + " mkdir -p %(ns3_src)s && " + " %(clone_ns3_cmd)s " + " ) " " ) " - ) % ({ - 'ns3_repo': self.ns3_repo, - 'ns3_version': self.ns3_version, - 'pybindgen_version': self.pybindgen_version, - 'pygccxml_version': self.pygccxml_version - }) + ") " + " && " + "( " + " %(clone_dce_cmd)s " + ") " + ) % { + "ns3_src": self.ns3_src_location, + "pybindgen_version": self.get("pybindgenVersion"), + "pygccxml_version": self.pygccxml_version, + "clone_ns3_cmd": clone_ns3_cmd, + "clone_dce_cmd": clone_dce_cmd, + } + + return ns3_build_cmd + + @property + def _install_dce_command(self): + install_dce_cmd = " echo 'DCE will not be installed'" + + if self.enable_dce: + install_dce_cmd = ( + " ( " + " ((test -d %(ns3_build)s/bin_dce ) && " + " echo 'dce binaries found, nothing to do' )" + " ) " + " ||" + " ( " + # If not, copy build to dce + " cd %(dce_src)s && " + " rm -rf %(dce_src)s/build && " + " ./waf configure %(enable_opt)s --with-pybindgen=${SRC}/pybindgen/%(pybindgen_version)s " + " --prefix=%(ns3_build)s --with-ns3=%(ns3_build)s && " + " ./waf build && " + " ./waf install && " + " [ ! -e %(ns3_build)s/lib/python/site-packages/ns/dce.so ] && " + " mv %(ns3_build)s/lib*/python*/site-packages/ns/dce.so %(ns3_build)s/lib/python/site-packages/ns/ " + " )" + ) % { + "pybindgen_version": self.get("pybindgenVersion"), + "enable_opt": "--enable-opt" if self.get("buildMode") == "optimized" else "", + "ns3_build": self.ns3_build_location, + "dce_src": self.dce_src_location, + } + + return install_dce_cmd @property def _install(self): - return ( - # Test if ns-3 is alredy cloned + install_dce_cmd = self._install_dce_command + + install_ns3_cmd = ( + # Test if ns-3 is alredy installed + "(" " ( " - " ( ( (test -d ${BIN}/%(ns3_version)s/build ) || " + " ( ( (test -d %(ns3_build)s/lib ) || " " (test -d ${NS3BINDINGS:='None'} && test -d ${NS3LIBRARIES:='None'}) ) && " " echo 'binaries found, nothing to do' )" " ) " " ||" " ( " # If not, copy ns-3 build to bin - " mkdir -p ${BIN}/%(ns3_version)s && " - " mv ${SRC}/%(ns3_version)s/build ${BIN}/%(ns3_version)s/build " + " mkdir -p %(ns3_build)s && " + " cd %(ns3_src)s && " + " rm -rf %(ns3_src)s/build && " + " ./waf configure -d %(build_mode)s --with-pybindgen=${SRC}/pybindgen/%(pybindgen_version)s " + " --prefix=%(ns3_build)s && " + " ./waf build && " + " ./waf install && " + " mv %(ns3_build)s/lib*/python* %(ns3_build)s/lib/python " " )" - ) % ({ - 'ns3_version': self.ns3_version - }) + ") " + " && " + "( " + " %(install_dce_cmd)s " + ") " + ) % { + "pybindgen_version": self.get("pybindgenVersion"), + "build_mode": self.get("buildMode"), + "install_dce_cmd": install_dce_cmd, + "ns3_build": self.ns3_build_location, + "ns3_src": self.ns3_src_location, + } + + return install_ns3_cmd @property def _environment(self): env = [] - env.append("NS3BINDINGS=${NS3BINDINGS:=${BIN}/%(ns3_version)s/build/bindings/python/}" % ({ - 'ns3_version': self.ns3_version, - })) - env.append("NS3LIBRARIES=${NS3LIBRARIES:=${BIN}/%(ns3_version)s/build/}" % ({ - 'ns3_version': self.ns3_version, - })) + env.append("PYTHONPATH=$PYTHONPATH:${NS3BINDINGS:=%(ns3_build)s/lib/python/site-packages}" % { + "ns3_build": self.ns3_build_location + }) + # If NS3LIBRARIES is defined and not empty, assign its value, + # if not assign ns3_build_home/lib/ to NS3LIBRARIES and LD_LIBARY_PATH + env.append("LD_LIBRARY_PATH=${NS3LIBRARIES:=%(ns3_build)s/lib}" % { + "ns3_build": self.ns3_build_location + }) + env.append("DCE_PATH=$NS3LIBRARIES/../bin_dce") + env.append("DCE_ROOT=$NS3LIBRARIES/..") return " ".join(env) + def replace_paths(self, command): + """ + Replace all special path tags with shell-escaped actual paths. + """ + return ( command + .replace("${USR}", self.node.usr_dir) + .replace("${LIB}", self.node.lib_dir) + .replace("${BIN}", self.node.bin_dir) + .replace("${SRC}", self.node.src_dir) + .replace("${SHARE}", self.node.share_dir) + .replace("${EXP}", self.node.exp_dir) + .replace("${EXP_HOME}", self.node.exp_home) + .replace("${APP_HOME}", self.app_home) + .replace("${RUN_HOME}", self.run_home) + .replace("${NODE_HOME}", self.node.node_home) + .replace("${HOME}", self.node.home_dir) + # If NS3LIBRARIES is defined and not empty, use that value, + # if not use ns3_build_home/lib/ + .replace("${BIN_DCE}", "${NS3LIBRARIES-%s/lib}/../bin_dce" % \ + self.ns3_build_location) + ) + def valid_connection(self, guid): # TODO: Validate! return True + def wait_remote_socket(self): + """ Waits until the remote socket is created + """ + command = " [ -e %s ] && echo 'DONE' " % self.remote_socket + + for i in xrange(200): + (out, err), proc = self.node.execute(command, retry = 1, + with_lock = True) + + if out.find("DONE") > -1: + break + else: + raise RuntimeError("Remote socket not found at %s" % \ + self.remote_socket) + +