X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=nepi%2Fresources%2Flinux%2Fnode.py;h=168686c2f1e8a19fb550e0be5b5f3e06dee37932;hb=77008816b3a7de0a39ba169ffc0579135a058b98;hp=375ff3a62669f8e7f87116485995225dcce6df6b;hpb=941fd7f54114b824fa26563d389a7ae07426fe14;p=nepi.git diff --git a/nepi/resources/linux/node.py b/nepi/resources/linux/node.py index 375ff3a6..168686c2 100644 --- a/nepi/resources/linux/node.py +++ b/nepi/resources/linux/node.py @@ -25,6 +25,7 @@ from nepi.util.sshfuncs import ProcStatus import collections import os +import stat import random import re import tempfile @@ -149,81 +150,74 @@ class LinuxNode(ResourceManager): @classmethod def _register_attributes(cls): - cls._register_attribute(Attribute( - "hostname", "Hostname of the machine", - flags = Flags.Design)) - - cls._register_attribute(Attribute( - "username", "Local account username", - flags = Flags.Credential)) - - cls._register_attribute(Attribute( - "port", "SSH port", - flags = Flags.Design)) - - cls._register_attribute(Attribute( - "home", - "Experiment home directory to store all experiment related files", - flags = Flags.Design)) - - cls._register_attribute(Attribute( - "identity", "SSH identity file", - flags = Flags.Credential)) - - cls._register_attribute(Attribute( - "serverKey", "Server public key", - flags = Flags.Design)) - - cls._register_attribute(Attribute( - "cleanHome", - "Remove all nepi files and directories " - " from node home folder before starting experiment", - type = Types.Bool, - default = False, - flags = Flags.Design)) - - cls._register_attribute(Attribute( - "cleanExperiment", "Remove all files and directories " - " from a previous same experiment, before the new experiment starts", - type = Types.Bool, - default = False, - flags = Flags.Design)) - - cls._register_attribute(Attribute( - "cleanProcesses", - "Kill all running processes before starting experiment", - type = Types.Bool, - default = False, - flags = Flags.Design)) - - cls._register_attribute(Attribute( - "cleanProcessesAfter", - """Kill all running processes after starting experiment - This might be dangerous when using user root""", - type = Types.Bool, - default = True, - flags = Flags.Design)) - - cls._register_attribute(Attribute( - "tearDown", - "Bash script to be executed before releasing the resource", - flags = Flags.Design)) - - cls._register_attribute(Attribute( - "gatewayUser", - "Gateway account username", - flags = Flags.Design)) - - cls._register_attribute(Attribute( - "gateway", - "Hostname of the gateway machine", - flags = Flags.Design)) - - cls._register_attribute(Attribute( - "ip", - "Linux host public IP address. " - "Must not be modified by the user unless hostname is 'localhost'", + cls._register_attribute( + Attribute("hostname", + "Hostname of the machine", + flags = Flags.Design)) + cls._register_attribute( + Attribute("username", + "Local account username", + flags = Flags.Credential)) + cls._register_attribute( + Attribute("port", + "SSH port", + flags = Flags.Design)) + cls._register_attribute( + Attribute("home", + "Experiment home directory to store all experiment related files", + flags = Flags.Design)) + cls._register_attribute( + Attribute("identity", + "SSH identity file", + flags = Flags.Credential)) + cls._register_attribute( + Attribute("serverKey", + "Server public key", + flags = Flags.Design)) + cls._register_attribute( + Attribute("cleanHome", + "Remove all nepi files and directories " + " from node home folder before starting experiment", + type = Types.Bool, + default = False, flags = Flags.Design)) + cls._register_attribute( + Attribute("cleanExperiment", + "Remove all files and directories " + " from a previous same experiment, before the new experiment starts", + type = Types.Bool, + default = False, + flags = Flags.Design)) + cls._register_attribute( + Attribute("cleanProcesses", + "Kill all running processes before starting experiment", + type = Types.Bool, + default = False, + flags = Flags.Design)) + cls._register_attribute( + Attribute("cleanProcessesAfter", + "Kill all running processes after starting experiment" + "NOTE: This might be dangerous when using user root", + type = Types.Bool, + default = True, + flags = Flags.Design)) + cls._register_attribute( + Attribute("tearDown", + "Bash script to be executed before releasing the resource", + flags = Flags.Design)) + cls._register_attribute( + Attribute("gatewayUser", + "Gateway account username", + flags = Flags.Design)) + cls._register_attribute( + Attribute("gateway", + "Hostname of the gateway machine", + flags = Flags.Design)) + cls._register_attribute( + Attribute("ip", + "Linux host public IP address. " + "Must not be modified by the user unless hostname is 'localhost'", + flags = Flags.Design)) def __init__(self, ec, guid): super(LinuxNode, self).__init__(ec, guid) @@ -305,7 +299,7 @@ class LinuxNode(ResourceManager): if out.find("Debian") == 0: self._os = OSType.DEBIAN - elif out.find("Ubuntu") ==0: + elif out.find("Ubuntu") == 0: self._os = OSType.UBUNTU elif out.find("Fedora release") == 0: self._os = OSType.FEDORA @@ -341,7 +335,7 @@ class LinuxNode(ResourceManager): @property def use_deb(self): - return (self.os & (OSType.DEBIAN|OSType.UBUNTU)) + return (self.os & (OSType.DEBIAN | OSType.UBUNTU)) @property def use_rpm(self): @@ -512,9 +506,10 @@ class LinuxNode(ResourceManager): (out, err), proc = self.execute(cmd, retry = 1, with_lock = True) - def search_for_child(self, pid, pids, ppid, family=[]): + def search_for_child(self, pid, pids, ppid, family=None): """ Recursive function to search for child. List A contains the pids and list B the parents (ppid) """ + family = family if family is not None else [] family.append(pid) for key, value in enumerate(ppid): if value == pid: @@ -749,7 +744,7 @@ class LinuxNode(ResourceManager): return (out, err), proc def upload(self, src, dst, text = False, overwrite = True, - raise_on_error = True): + raise_on_error = True, executable = False): """ Copy content to destination src string with the content to copy. Can be: @@ -761,8 +756,9 @@ class LinuxNode(ResourceManager): dst string with destination path on the remote host (remote is always self.host) - text src is text input, it must be stored into a temp file before - uploading + when src is text input, it gets stored into a temp file before + uploading; in this case, and if executable is True, said temp file + is made executable, and thus uploaded file will be too """ # If source is a string input f = None @@ -771,9 +767,15 @@ class LinuxNode(ResourceManager): # create a temporal file with the content to upload # in python3 we need to open in binary mode if str is bytes mode = 'w' if isinstance(src, str) else 'wb' - f = tempfile.NamedTemporaryFile(mode=mode, delete=False) + f = tempfile.NamedTemporaryFile(mode = mode, delete = False) f.write(src) f.close() + if executable: + # do something like chmod u+x + mode = os.stat(f.name).st_mode + mode |= stat.S_IXUSR + os.chmod(f.name, mode) + src = f.name # If dst files should not be overwritten, check that the files do not @@ -914,18 +916,18 @@ class LinuxNode(ResourceManager): return self.execute(cmd, with_lock = True) def run_and_wait(self, command, home, - shfile="cmd.sh", - env=None, - overwrite=True, - wait_run=True, - pidfile="pidfile", - ecodefile="exitcode", - stdin=None, - stdout="stdout", - stderr="stderr", - sudo=False, - tty=False, - raise_on_error=True): + shfile = "cmd.sh", + env = None, + overwrite = True, + wait_run = True, + pidfile = "pidfile", + ecodefile = "exitcode", + stdin = None, + stdout = "stdout", + stderr = "stderr", + sudo = False, + tty = False, + raise_on_error = True): """ Uploads the 'command' to a bash script in the host. Then runs the script detached in background in the host, and @@ -1007,10 +1009,10 @@ class LinuxNode(ResourceManager): return ExitCode.ERROR def upload_command(self, command, - shfile="cmd.sh", - ecodefile="exitcode", - overwrite=True, - env=None): + shfile = "cmd.sh", + ecodefile = "exitcode", + overwrite = True, + env = None): """ Saves the command as a bash script file in the remote host, and forces to save the exit code of the command execution to the ecodefile """ @@ -1020,7 +1022,7 @@ class LinuxNode(ResourceManager): # The exit code of the command will be stored in ecodefile command = " {{ {command} }} ; echo $? > {ecodefile} ;"\ - .format(command=command, ecodefile=ecodefile) + .format(command = command, ecodefile = ecodefile) # Export environment environ = self.format_environment(env) @@ -1028,9 +1030,9 @@ class LinuxNode(ResourceManager): # Add environ to command command = environ + command - return self.upload(command, shfile, text=True, overwrite=overwrite) + return self.upload(command, shfile, text = True, overwrite = overwrite) - def format_environment(self, env, inline=False): + def format_environment(self, env, inline = False): """ Formats the environment variables for a command to be executed either as an inline command (i.e. export PYTHONPATH=src/..; export LALAL= ..;python script.py) or @@ -1183,13 +1185,16 @@ class LinuxNode(ResourceManager): command = [] for d in dests: - command.append(" [ -f {dst} ] && echo '{dst}' ".format(dst=d) ) + command.append(" [ -f {dst} ] && echo '{dst}' ".format(dst = d) ) command = ";".join(command) (out, err), proc = self.execute(command, retry = 1, with_lock = True) - for d in dests: + # avoid RuntimeError that would result from + # changing loop subject during iteration + keys = list(dests.keys()) + for d in keys: if out.find(d) > -1: del dests[d] @@ -1199,4 +1204,3 @@ class LinuxNode(ResourceManager): retcod = dests.values() if PY3: retcod = list(retcod) return retcod -