# Author: Alina Quereilhac <alina.quereilhac@inria.fr>
from nepi.execution.attribute import Attribute, Flags, Types
-from nepi.execution.resource import ResourceManager, clsinit_copy, \
- ResourceState
+from nepi.execution.resource import (ResourceManager, clsinit_copy,
+ ResourceState)
from nepi.resources.linux import rpmfuncs, debfuncs
from nepi.util import sshfuncs, execfuncs
from nepi.util.sshfuncs import ProcStatus
@classmethod
def _register_attributes(cls):
- hostname = Attribute("hostname", "Hostname of the machine",
- flags = Flags.Design)
+ cls._register_attribute(Attribute(
+ "hostname", "Hostname of the machine",
+ flags = Flags.Design))
- username = Attribute("username", "Local account username",
- flags = Flags.Credential)
+ cls._register_attribute(Attribute(
+ "username", "Local account username",
+ flags = Flags.Credential))
- port = Attribute("port", "SSH port", flags = Flags.Design)
+ cls._register_attribute(Attribute(
+ "port", "SSH port",
+ flags = Flags.Design))
- home = Attribute("home",
- "Experiment home directory to store all experiment related files",
- flags = Flags.Design)
+ cls._register_attribute(Attribute(
+ "home",
+ "Experiment home directory to store all experiment related files",
+ flags = Flags.Design))
- identity = Attribute("identity", "SSH identity file",
- flags = Flags.Credential)
+ cls._register_attribute(Attribute(
+ "identity", "SSH identity file",
+ flags = Flags.Credential))
- server_key = Attribute("serverKey", "Server public key",
- flags = Flags.Design)
+ cls._register_attribute(Attribute(
+ "serverKey", "Server public key",
+ flags = Flags.Design))
- clean_home = Attribute("cleanHome", "Remove all nepi files and directories "
- " from node home folder before starting experiment",
- type = Types.Bool,
- default = False,
- flags = Flags.Design)
-
- clean_experiment = Attribute("cleanExperiment", "Remove all files and directories "
- " from a previous same experiment, before the new experiment starts",
- type = Types.Bool,
- default = False,
- flags = Flags.Design)
+ cls._register_attribute(Attribute(
+ "cleanHome",
+ "Remove all nepi files and directories "
+ " from node home folder before starting experiment",
+ type = Types.Bool,
+ default = False,
+ flags = Flags.Design))
+
+ cls._register_attribute(Attribute(
+ "cleanExperiment", "Remove all files and directories "
+ " from a previous same experiment, before the new experiment starts",
+ type = Types.Bool,
+ default = False,
+ flags = Flags.Design))
- clean_processes = Attribute("cleanProcesses",
- "Kill all running processes before starting experiment",
- type = Types.Bool,
- default = False,
- flags = Flags.Design)
+ cls._register_attribute(Attribute(
+ "cleanProcesses",
+ "Kill all running processes before starting experiment",
+ type = Types.Bool,
+ default = False,
+ flags = Flags.Design))
- clean_processes_after = Attribute("cleanProcessesAfter",
- """Kill all running processes after starting experiment
-This might be dangerous when using user root""",
- type = Types.Bool,
- default = True,
- flags = Flags.Design)
+ cls._register_attribute(Attribute(
+ "cleanProcessesAfter",
+ """Kill all running processes after starting experiment
+ This might be dangerous when using user root""",
+ type = Types.Bool,
+ default = True,
+ flags = Flags.Design))
- tear_down = Attribute("tearDown", "Bash script to be executed before " + \
- "releasing the resource",
- flags = Flags.Design)
-
- gateway_user = Attribute("gatewayUser", "Gateway account username",
- flags = Flags.Design)
-
- gateway = Attribute("gateway", "Hostname of the gateway machine",
- flags = Flags.Design)
-
- ip = Attribute("ip", "Linux host public IP address. "
- "Must not be modified by the user unless hostname is 'localhost'",
- flags = Flags.Design)
-
- cls._register_attribute(hostname)
- cls._register_attribute(username)
- cls._register_attribute(port)
- cls._register_attribute(home)
- cls._register_attribute(identity)
- cls._register_attribute(server_key)
- cls._register_attribute(clean_home)
- cls._register_attribute(clean_experiment)
- cls._register_attribute(clean_processes)
- cls._register_attribute(clean_processes_after)
- cls._register_attribute(tear_down)
- cls._register_attribute(gateway_user)
- cls._register_attribute(gateway)
- cls._register_attribute(ip)
+ cls._register_attribute(Attribute(
+ "tearDown",
+ "Bash script to be executed before releasing the resource",
+ flags = Flags.Design))
+
+ cls._register_attribute(Attribute(
+ "gatewayUser",
+ "Gateway account username",
+ flags = Flags.Design))
+
+ cls._register_attribute(Attribute(
+ "gateway",
+ "Hostname of the gateway machine",
+ flags = Flags.Design))
+
+ cls._register_attribute(Attribute(
+ "ip",
+ "Linux host public IP address. "
+ "Must not be modified by the user unless hostname is 'localhost'",
+ flags = Flags.Design))
def __init__(self, ec, guid):
super(LinuxNode, self).__init__(ec, guid)
# of a file or folder prior to its creation, and another
# application creating the same file or folder in between.
self._node_lock = threading.Lock()
-
+
def log_message(self, msg):
- return " guid %d - host %s - %s " % (self.guid,
- self.get("hostname"), msg)
+ return " guid {} - host {} - {} "\
+ .format(self.guid, self.get("hostname"), msg)
@property
def home_dir(self):
home = self.get("home") or ""
if not home.startswith("/"):
- home = os.path.join(self._home_dir, home)
- return home
+ home = os.path.join(self._home_dir, home)
+ return home
@property
def nepi_home(self):
@property
def node_home(self):
- return os.path.join(self.exp_home, "node-%d" % self.guid)
+ return os.path.join(self.exp_home, "node-{}".format(self.guid))
@property
def run_home(self):
else:
msg = "Unsupported OS"
self.error(msg, out)
- raise RuntimeError, "%s - %s " %( msg, out )
+ raise RuntimeError("{} - {} ".format(msg, out))
return self._os
out = ""
try:
(out, err), proc = self.execute("cat /etc/issue",
- with_lock = True,
- blocking = True)
+ with_lock = True,
+ blocking = True)
except:
trace = traceback.format_exc()
- msg = "Error detecting OS: %s " % trace
+ msg = "Error detecting OS: {} ".format(trace)
self.error(msg, out, err)
-
+
return out
@property
def do_provision(self):
# check if host is alive
if not self.is_alive():
- msg = "Deploy failed. Unresponsive node %s" % self.get("hostname")
+ msg = "Deploy failed. Unresponsive node {}".format(self.get("hostname"))
self.error(msg)
raise RuntimeError, msg
if self.get("cleanHome"):
self.clean_home()
-
+
if self.get("cleanExperiment"):
self.clean_experiment()
-
+
# Create shared directory structure and node home directory
paths = [self.lib_dir,
- self.bin_dir,
- self.src_dir,
- self.share_dir,
- self.node_home]
+ self.bin_dir,
+ self.src_dir,
+ self.share_dir,
+ self.node_home]
self.mkdir(paths)
self.set("ip", ip)
except:
if self.get("gateway") is None:
- msg = "Local DNS can not resolve hostname %s" % self.get("hostname")
+ msg = "Local DNS can not resolve hostname {}".format(self.get("hostname"))
self.error(msg)
super(LinuxNode, self).do_provision()
if self.get("username") != 'root':
cmd = ("sudo -S killall tcpdump || /bin/true ; " +
- "sudo -S kill -9 $(ps aux | grep '[.]nepi' | awk '{print $2}') || /bin/true ; " +
- "sudo -S killall -u %s || /bin/true ; " % self.get("username"))
+ "sudo -S kill -9 $(ps aux | grep '[.]nepi' | awk '{print $2}') || /bin/true ; " +
+ "sudo -S killall -u {} || /bin/true ; ".format(self.get("username")))
else:
if self.state >= ResourceState.READY:
import pickle
kill_pids = ' '.join(dict(kill_pids).keys())
cmd = ("killall tcpdump || /bin/true ; " +
- "kill $(ps aux | grep '[.]nepi' | awk '{print $2}') || /bin/true ; " +
- "kill %s || /bin/true ; " % kill_pids)
+ "kill $(ps aux | grep '[.]nepi' | awk '{print $2}') || /bin/true ; " +
+ "kill {} || /bin/true ; ".format(kill_pids))
else:
cmd = ("killall tcpdump || /bin/true ; " +
- "kill $(ps aux | grep '[.]nepi' | awk '{print $2}') || /bin/true ; ")
+ "kill $(ps aux | grep '[.]nepi' | awk '{print $2}') || /bin/true ; ")
else:
cmd = ("killall tcpdump || /bin/true ; " +
- "kill $(ps aux | grep '[.]nepi' | awk '{print $2}') || /bin/true ; ")
+ "kill $(ps aux | grep '[.]nepi' | awk '{print $2}') || /bin/true ; ")
(out, err), proc = self.execute(cmd, retry = 1, with_lock = True)
"""
self.info("Cleaning up home")
- cmd = "cd %s ; find . -maxdepth 1 -name \.nepi -execdir rm -rf {} + " % (
- self.home_dir )
+ cmd = "cd {} ; find . -maxdepth 1 -name \.nepi -execdir rm -rf {} + "\
+ .format(self.home_dir)
return self.execute(cmd, with_lock = True)
"""
self.info("Cleaning up experiment files")
- cmd = "cd %s ; find . -maxdepth 1 -name '%s' -execdir rm -rf {} + " % (
- self.exp_dir,
- self.ec.exp_id )
-
+ cmd = "cd {} ; find . -maxdepth 1 -name '{}' -execdir rm -rf {} + "\
+ .format(self.exp_dir, self.ec.exp_id)
+
return self.execute(cmd, with_lock = True)
def execute(self, command,
- sudo = False,
- env = None,
- tty = False,
- forward_x11 = False,
- retry = 3,
- connect_timeout = 30,
- strict_host_checking = False,
- persistent = True,
- blocking = True,
- with_lock = False
- ):
+ sudo = False,
+ env = None,
+ tty = False,
+ forward_x11 = False,
+ retry = 3,
+ connect_timeout = 30,
+ strict_host_checking = False,
+ persistent = True,
+ blocking = True,
+ with_lock = False
+ ):
""" Notice that this invocation will block until the
execution finishes. If this is not the desired behavior,
use 'run' instead."""
if self.localhost:
- (out, err), proc = execfuncs.lexec(command,
- user = self.get("username"), # still problem with localhost
- sudo = sudo,
- env = env)
+ (out, err), proc = execfuncs.lexec(
+ command,
+ user = self.get("username"), # still problem with localhost
+ sudo = sudo,
+ env = env)
else:
if with_lock:
# If the execute command is blocking, we don't want to keep
persistent = persistent,
blocking = blocking,
strict_host_checking = strict_host_checking
- )
+ )
else:
(out, err), proc = sshfuncs.rexec(
command,
persistent = persistent,
blocking = blocking,
strict_host_checking = strict_host_checking
- )
+ )
return (out, err), proc
tty = False,
strict_host_checking = False):
- self.debug("Running command '%s'" % command)
+ self.debug("Running command '{}'".format(command))
if self.localhost:
- (out, err), proc = execfuncs.lspawn(command, pidfile,
- home = home,
- create_home = create_home,
- stdin = stdin or '/dev/null',
- stdout = stdout or '/dev/null',
- stderr = stderr or '/dev/null',
- sudo = sudo)
+ (out, err), proc = execfuncs.lspawn(
+ command, pidfile,
+ home = home,
+ create_home = create_home,
+ stdin = stdin or '/dev/null',
+ stdout = stdout or '/dev/null',
+ stderr = stderr or '/dev/null',
+ sudo = sudo)
else:
with self._node_lock:
(out, err), proc = sshfuncs.rspawn(
server_key = self.get("serverKey"),
tty = tty,
strict_host_checking = strict_host_checking
- )
+ )
return (out, err), proc
identity = self.get("identity"),
server_key = self.get("serverKey"),
strict_host_checking = False
- )
+ )
return pidtuple
else:
with self._node_lock:
status = sshfuncs.rstatus(
- pid, ppid,
- host = self.get("hostname"),
- user = self.get("username"),
- port = self.get("port"),
- gwuser = self.get("gatewayUser"),
- gw = self.get("gateway"),
- agent = True,
- identity = self.get("identity"),
- server_key = self.get("serverKey"),
- strict_host_checking = False
- )
+ pid, ppid,
+ host = self.get("hostname"),
+ user = self.get("username"),
+ port = self.get("port"),
+ gwuser = self.get("gatewayUser"),
+ gw = self.get("gateway"),
+ agent = True,
+ identity = self.get("identity"),
+ server_key = self.get("serverKey"),
+ strict_host_checking = False
+ )
return status
identity = self.get("identity"),
server_key = self.get("serverKey"),
strict_host_checking = False
- )
+ )
return (out, err), proc
def copy(self, src, dst):
if self.localhost:
- (out, err), proc = execfuncs.lcopy(src, dst,
- recursive = True)
+ (out, err), proc = execfuncs.lcopy(
+ src, dst,
+ recursive = True)
else:
with self._node_lock:
(out, err), proc = sshfuncs.rcopy(
return (out, err), proc
def upload(self, src, dst, text = False, overwrite = True,
- raise_on_error = True):
+ raise_on_error = True):
""" Copy content to destination
src string with the content to copy. Can be:
if not self.localhost:
# Build destination as <user>@<server>:<path>
- dst = "%s@%s:%s" % (self.get("username"), self.get("hostname"), dst)
+ dst = "{}@{}:{}".format(self.get("username"), self.get("hostname"), dst)
((out, err), proc) = self.copy(src, dst)
os.remove(f.name)
if err:
- msg = " Failed to upload files - src: %s dst: %s" % (";".join(src), dst)
+ msg = " Failed to upload files - src: {} dst: {}".format(";".join(src), dst)
self.error(msg, out, err)
- msg = "%s out: %s err: %s" % (msg, out, err)
+ msg = "{} out: {} err: {}".format(msg, out, err)
if raise_on_error:
raise RuntimeError, msg
def download(self, src, dst, raise_on_error = True):
if not self.localhost:
# Build destination as <user>@<server>:<path>
- src = "%s@%s:%s" % (self.get("username"), self.get("hostname"), src)
+ src = "{}@{}:{}".format(self.get("username"), self.get("hostname"), src)
((out, err), proc) = self.copy(src, dst)
if err:
- msg = " Failed to download files - src: %s dst: %s" % (";".join(src), dst)
+ msg = " Failed to download files - src: {} dst: {}".format(";".join(src), dst)
self.error(msg, out, err)
if raise_on_error:
return command
- def install_packages(self, packages, home, run_home = None,
- raise_on_error = True):
+ def install_packages(self, packages, home,
+ run_home = None,
+ raise_on_error = True):
""" Install packages in the Linux host.
'home' is the directory to upload the package installation script.
run_home = run_home or home
(out, err), proc = self.run_and_wait(command, run_home,
- shfile = os.path.join(home, "instpkg.sh"),
- pidfile = "instpkg_pidfile",
- ecodefile = "instpkg_exitcode",
- stdout = "instpkg_stdout",
- stderr = "instpkg_stderr",
- overwrite = False,
- raise_on_error = raise_on_error)
+ shfile = os.path.join(home, "instpkg.sh"),
+ pidfile = "instpkg_pidfile",
+ ecodefile = "instpkg_exitcode",
+ stdout = "instpkg_stdout",
+ stderr = "instpkg_stderr",
+ overwrite = False,
+ raise_on_error = raise_on_error)
return (out, err), proc
def remove_packages(self, packages, home, run_home = None,
- raise_on_error = True):
+ raise_on_error = True):
""" Uninstall packages from the Linux host.
'home' is the directory to upload the package un-installation script.
run_home = run_home or home
(out, err), proc = self.run_and_wait(command, run_home,
- shfile = os.path.join(home, "rmpkg.sh"),
- pidfile = "rmpkg_pidfile",
- ecodefile = "rmpkg_exitcode",
- stdout = "rmpkg_stdout",
- stderr = "rmpkg_stderr",
- overwrite = False,
- raise_on_error = raise_on_error)
-
+ shfile = os.path.join(home, "rmpkg.sh"),
+ pidfile = "rmpkg_pidfile",
+ ecodefile = "rmpkg_exitcode",
+ stdout = "rmpkg_stdout",
+ stderr = "rmpkg_stderr",
+ overwrite = False,
+ raise_on_error = raise_on_error)
+
return (out, err), proc
def mkdir(self, paths, clean = False):
if isinstance(paths, str):
paths = [paths]
- cmd = " ; ".join(map(lambda path: "mkdir -p %s" % path, paths))
+ cmd = " ; ".join(["mkdir -p {}".format(path) for path in paths])
return self.execute(cmd, with_lock = True)
if isinstance(paths, str):
paths = [paths]
- cmd = " ; ".join(map(lambda path: "rm -rf %s" % path, paths))
+ cmd = " ; ".join(map(lambda path: "rm -rf {}".format(path), paths))
return self.execute(cmd, with_lock = True)
-
+
def run_and_wait(self, command, home,
- shfile="cmd.sh",
- env=None,
- overwrite=True,
- wait_run=True,
- pidfile="pidfile",
- ecodefile="exitcode",
- stdin=None,
- stdout="stdout",
- stderr="stderr",
- sudo=False,
- tty=False,
- raise_on_error=True):
+ shfile="cmd.sh",
+ env=None,
+ overwrite=True,
+ wait_run=True,
+ pidfile="pidfile",
+ ecodefile="exitcode",
+ stdin=None,
+ stdout="stdout",
+ stderr="stderr",
+ sudo=False,
+ tty=False,
+ raise_on_error=True):
"""
Uploads the 'command' to a bash script in the host.
Then runs the script detached in background in the host, and
shfile = os.path.join(home, shfile)
self.upload_command(command,
- shfile = shfile,
- ecodefile = ecodefile,
- env = env,
- overwrite = overwrite)
+ shfile = shfile,
+ ecodefile = ecodefile,
+ env = env,
+ overwrite = overwrite)
- command = "bash %s" % shfile
+ command = "bash {}".format(shfile)
# run command in background in remote host
(out, err), proc = self.run(command, home,
- pidfile = pidfile,
- stdin = stdin,
- stdout = stdout,
- stderr = stderr,
- sudo = sudo,
- tty = tty)
+ pidfile = pidfile,
+ stdin = stdin,
+ stdout = stdout,
+ stderr = stderr,
+ sudo = sudo,
+ tty = tty)
# check no errors occurred
if proc.poll():
- msg = " Failed to run command '%s' " % command
+ msg = " Failed to run command '{}' ".format(command)
self.error(msg, out, err)
if raise_on_error:
raise RuntimeError, msg
# Wait for pid file to be generated
pid, ppid = self.wait_pid(
- home = home,
- pidfile = pidfile,
- raise_on_error = raise_on_error)
+ home = home,
+ pidfile = pidfile,
+ raise_on_error = raise_on_error)
if wait_run:
# wait until command finishes to execute
self.wait_run(pid, ppid)
-
+
(eout, err), proc = self.check_errors(home,
- ecodefile = ecodefile,
- stderr = stderr)
+ ecodefile = ecodefile,
+ stderr = stderr)
# Out is what was written in the stderr file
if err:
- msg = " Failed to run command '%s' " % command
+ msg = " Failed to run command '{}' ".format(command)
self.error(msg, eout, err)
if raise_on_error:
return ExitCode.ERROR
def upload_command(self, command,
- shfile="cmd.sh",
- ecodefile="exitcode",
- overwrite=True,
- env=None):
+ shfile="cmd.sh",
+ ecodefile="exitcode",
+ overwrite=True,
+ env=None):
""" Saves the command as a bash script file in the remote host, and
forces to save the exit code of the command execution to the ecodefile
"""
if not (command.strip().endswith(";") or command.strip().endswith("&")):
command += ";"
-
+
# The exit code of the command will be stored in ecodefile
- command = " { %(command)s } ; echo $? > %(ecodefile)s ;" % {
- 'command': command,
- 'ecodefile': ecodefile,
- }
+ command = " {{ {command} }} ; echo $? > {ecodefile} ;"\
+ .format(command=command, ecodefile=ecodefile)
# Export environment
environ = self.format_environment(env)
env = re.sub(r'\s+', ' ', env.strip())
sep = ";" if inline else "\n"
- return sep.join(map(lambda e: " export %s" % e, env.split(" "))) + sep
+ return sep.join([" export {}".format(e) for e in env.split(" ")]) + sep
def check_errors(self, home,
- ecodefile = "exitcode",
- stderr = "stderr"):
+ ecodefile = "exitcode",
+ stderr = "stderr"):
""" Checks whether errors occurred while running a command.
It first checks the exit code for the command, and only if the
exit code is an error one it returns the error output.
ecode = self.exitcode(home, ecodefile)
if ecode in [ ExitCode.CORRUPTFILE, ExitCode.ERROR ]:
- err = "Error retrieving exit code status from file %s/%s" % (home, ecodefile)
+ err = "Error retrieving exit code status from file {}/{}".format(home, ecodefile)
elif ecode > 0 or ecode == ExitCode.FILENOTFOUND:
# The process returned an error code or didn't exist.
# Check standard error.
# (cat returns 1 for error "No such file or directory")
if ecode == ExitCode.FILENOTFOUND and proc.poll() == 1:
err = ""
-
+
return ("", err), proc
-
+
def wait_pid(self, home, pidfile = "pidfile", raise_on_error = False):
""" Waits until the pid file for the command is generated,
and returns the pid and ppid of the process """
time.sleep(delay)
delay = delay * 1.5
else:
- msg = " Failed to get pid for pidfile %s/%s " % (
- home, pidfile )
+ msg = " Failed to get pid for pidfile {}/{} ".format(home, pidfile )
self.error(msg)
-
+
if raise_on_error:
raise RuntimeError, msg
def check_output(self, home, filename):
""" Retrives content of file """
- (out, err), proc = self.execute("cat %s" %
- os.path.join(home, filename), retry = 1, with_lock = True)
+ (out, err), proc = self.execute(
+ "cat {}".format(os.path.join(home, filename)), retry = 1, with_lock = True)
return (out, err), proc
def is_alive(self):
# until the result is not empty string
try:
(out, err), proc = self.execute("echo 'ALIVE'",
- blocking = True,
- with_lock = True)
-
+ blocking = True,
+ with_lock = True)
+
if out.find("ALIVE") > -1:
return True
except:
trace = traceback.format_exc()
- msg = "Unresponsive host. Error reaching host: %s " % trace
+ msg = "Unresponsive host. Error reaching host: {} ".format(trace)
self.error(msg, out, err)
return False
def find_home(self):
- """ Retrieves host home directory
+ """
+ Retrieves host home directory
"""
# The underlying SSH layer will sometimes return an empty
# output (even if the command was executed without errors).
msg = "Impossible to retrieve HOME directory"
try:
(out, err), proc = self.execute("echo ${HOME}",
- blocking = True,
- with_lock = True)
-
+ blocking = True,
+ with_lock = True)
+
if out.strip() != "":
self._home_dir = out.strip()
except:
trace = traceback.format_exc()
- msg = "Impossible to retrieve HOME directory %s" % trace
+ msg = "Impossible to retrieve HOME directory {}".format(trace)
if not self._home_dir:
self.error(msg)
""" Removes files that already exist in the Linux host from src list
"""
# construct a dictionary with { dst: src }
- dests = dict(map(lambda s: (os.path.join(dst, os.path.basename(s)), s), src)) \
- if len(src) > 1 else dict({dst: src[0]})
+ dests = { os.path.join(dst, os.path.basename(s)) : s for s in src } \
+ if len(src) > 1 else {dst: src[0]}
command = []
for d in dests.keys():
- command.append(" [ -f %(dst)s ] && echo '%(dst)s' " % {'dst' : d} )
+ command.append(" [ -f {dst} ] && echo '{dst}' ".format(dst=d) )
command = ";".join(command)
(out, err), proc = self.execute(command, retry = 1, with_lock = True)
-
+
for d in dests.keys():
if out.find(d) > -1:
del dests[d]
git://git.onelab.eu / nepi.git / commitdiff