From: Thierry Parmentelat Date: Wed, 9 Mar 2016 15:10:51 +0000 (+0100) Subject: linux/{node,application} X-Git-Tag: nepi-6.0.9-pypi~3 X-Git-Url: http://git.onelab.eu/?p=nepi.git;a=commitdiff_plain;h=77008816b3a7de0a39ba169ffc0579135a058b98 linux/{node,application} only cosmetic and formatting use .format() instead of % should be harmless --- diff --git a/nepi/resources/linux/application.py b/nepi/resources/linux/application.py index 4b60f463..f9a37e16 100644 --- a/nepi/resources/linux/application.py +++ b/nepi/resources/linux/application.py @@ -175,7 +175,7 @@ class LinuxApplication(ResourceManager): self._pid = None self._ppid = None self._node = None - self._home = "app-%s" % self.guid + self._home = "app-{}".format(self.guid) # whether the command should run in foreground attached # to a terminal @@ -192,16 +192,16 @@ class LinuxApplication(ResourceManager): self._last_state_check = tnow() def log_message(self, msg): - return " guid %d - host %s - %s " % (self.guid, - self.node.get("hostname"), msg) + return " guid {} - host {} - {} "\ + .format(self.guid, self.node.get("hostname"), msg) @property def node(self): if not self._node: node = self.get_connected(LinuxNode.get_rtype()) if not node: - msg = "Application %s guid %d NOT connected to Node" % ( - self._rtype, self.guid) + msg = "Application {} guid {} NOT connected to Node"\ + .format(self._rtype, self.guid) raise RuntimeError(msg) self._node = node[0] @@ -226,7 +226,8 @@ class LinuxApplication(ResourceManager): @property def in_foreground(self): - """ Returns True if the command needs to be executed in foreground. + """ + Returns True if the command needs to be executed in foreground. This means that command will be executed using 'execute' instead of 'run' ('run' executes a command in background and detached from the terminal) @@ -241,15 +242,15 @@ class LinuxApplication(ResourceManager): return os.path.join(self.run_home, filename) def trace(self, name, attr = TraceAttr.ALL, block = 512, offset = 0): - self.info("Retrieving '%s' trace %s " % (name, attr)) + self.info("Retrieving '{}' trace {} ".format(name, attr)) path = self.trace_filepath(name) - command = "(test -f %s && echo 'success') || echo 'error'" % path + command = "(test -f {} && echo 'success') || echo 'error'".format(path) (out, err), proc = self.node.execute(command) if (err and proc.poll()) or out.find("error") != -1: - msg = " Couldn't find trace %s " % name + msg = " Couldn't find trace {} ".format(name) self.error(msg, out, err) return None @@ -260,21 +261,21 @@ class LinuxApplication(ResourceManager): (out, err), proc = self.node.check_output(self.run_home, name) if proc.poll(): - msg = " Couldn't read trace %s " % name + msg = " Couldn't read trace {} ".format(name) self.error(msg, out, err) return None return out if attr == TraceAttr.STREAM: - cmd = "dd if=%s bs=%d count=1 skip=%d" % (path, block, offset) + cmd = "dd if={} bs={} count=1 skip={}".format(path, block, offset) elif attr == TraceAttr.SIZE: - cmd = "stat -c%%s %s " % path + cmd = "stat -c {} ".format(path) (out, err), proc = self.node.execute(cmd) if proc.poll(): - msg = " Couldn't find trace %s " % name + msg = " Couldn't find trace {} ".format(name) self.error(msg, out, err) return None @@ -290,7 +291,7 @@ class LinuxApplication(ResourceManager): if self.node.get("username") == 'root': import pickle procs = dict() - ps_aux = "ps aux |awk '{print $2,$11}'" + ps_aux = "ps aux | awk '{print $2,$11}'" (out, err), proc = self.node.execute(ps_aux) if len(out) != 0: for line in out.strip().split("\n"): @@ -321,7 +322,8 @@ class LinuxApplication(ResourceManager): # build self.build, # Install - self.install] + self.install, + ] command = [] @@ -353,7 +355,7 @@ class LinuxApplication(ResourceManager): command = self.get("command") if command and not self.in_foreground: - self.info("Uploading command '%s'" % command) + self.info("Uploading command '{}'".format(command)) # replace application specific paths in the command command = self.replace_paths(command) @@ -364,9 +366,9 @@ class LinuxApplication(ResourceManager): shfile = os.path.join(self.app_home, "start.sh") self.node.upload_command(command, - shfile = shfile, - env = env, - overwrite = overwrite) + shfile = shfile, + env = env, + overwrite = overwrite) def execute_deploy_command(self, command, prefix="deploy"): if command: @@ -380,14 +382,14 @@ class LinuxApplication(ResourceManager): # Upload the command to a bash script and run it # in background ( but wait until the command has # finished to continue ) - shfile = os.path.join(self.app_home, "%s.sh" % prefix) + shfile = os.path.join(self.app_home, "{}.sh".format(prefix)) self.node.run_and_wait(command, self.run_home, - shfile = shfile, - overwrite = False, - pidfile = "%s_pidfile" % prefix, - ecodefile = "%s_exitcode" % prefix, - stdout = "%s_stdout" % prefix, - stderr = "%s_stderr" % prefix) + shfile = shfile, + overwrite = False, + pidfile = "{}_pidfile".format(prefix), + ecodefile = "{}_exitcode".format(prefix), + stdout = "{}_stdout".format(prefix), + stderr = "{}_stderr".format(prefix)) def upload_sources(self, sources = None, src_dir = None): if not sources: @@ -412,19 +414,20 @@ class LinuxApplication(ResourceManager): # remove the hhtp source from the sources list sources.remove(source) - command.append( " ( " - # Check if the source already exists - " ls %(src_dir)s/%(basename)s " - " || ( " - # If source doesn't exist, download it and check - # that it it downloaded ok - " wget -c --directory-prefix=%(src_dir)s %(source)s && " - " ls %(src_dir)s/%(basename)s " - " ) ) " % { - "basename": os.path.basename(source), - "source": source, - "src_dir": src_dir - }) + command.append( + " ( " + # Check if the source already exists + " ls {src_dir}/{basename} " + " || ( " + # If source doesn't exist, download it and check + # that it it downloaded ok + " wget -c --directory-prefix={src_dir} {source} && " + " ls {src_dir}/{basename} " + " ) ) ".format( + basename = os.path.basename(source), + source = source, + src_dir = src_dir + )) command = " && ".join(command) @@ -442,7 +445,7 @@ class LinuxApplication(ResourceManager): files = self.get("files") if files: - self.info("Uploading files %s " % files) + self.info("Uploading files {} ".format(files)) self.node.upload(files, self.node.share_dir, overwrite = False) def upload_libraries(self, libs = None): @@ -450,7 +453,7 @@ class LinuxApplication(ResourceManager): libs = self.get("libs") if libs: - self.info("Uploading libraries %s " % libaries) + self.info("Uploading libraries {} ".format(libs)) self.node.upload(libs, self.node.lib_dir, overwrite = False) def upload_binaries(self, bins = None): @@ -458,7 +461,7 @@ class LinuxApplication(ResourceManager): bins = self.get("bins") if bins: - self.info("Uploading binaries %s " % binaries) + self.info("Uploading binaries {} ".format(bins)) self.node.upload(bins, self.node.bin_dir, overwrite = False) def upload_code(self, code = None): @@ -467,7 +470,6 @@ class LinuxApplication(ResourceManager): if code: self.info("Uploading code") - dst = os.path.join(self.app_home, "code") self.node.upload(code, dst, overwrite = False, text = True, executable = True) @@ -489,10 +491,8 @@ class LinuxApplication(ResourceManager): self.node.upload(stdin, dst, overwrite = False, text = True) # create "stdin" symlink on ${APP_HOME} directory - command = "( cd %(app_home)s ; [ ! -f stdin ] && ln -s %(stdin)s stdin )" % ({ - "app_home": self.app_home, - "stdin": dst }) - + command = "( cd {app_home} ; [ ! -f stdin ] && ln -s {stdin} stdin )"\ + .format(app_home = self.app_home, stdin = dst) return command def install_dependencies(self, depends = None): @@ -500,7 +500,7 @@ class LinuxApplication(ResourceManager): depends = self.get("depends") if depends: - self.info("Installing dependencies %s" % depends) + self.info("Installing dependencies {}".format(depends)) return self.node.install_packages_command(depends) def build(self, build = None): @@ -527,11 +527,11 @@ class LinuxApplication(ResourceManager): # Wait until node is associated and deployed node = self.node if not node or node.state < ResourceState.READY: - self.debug("---- RESCHEDULING DEPLOY ---- node state %s " % self.node.state) + self.debug("---- RESCHEDULING DEPLOY ---- node state {} ".format(self.node.state)) self.ec.schedule(self.reschedule_delay, self.deploy) else: command = self.get("command") or "" - self.info("Deploying command '%s' " % command) + self.info("Deploying command '{}' ".format(command)) self.do_discover() self.do_provision() @@ -540,7 +540,7 @@ class LinuxApplication(ResourceManager): def do_start(self): command = self.get("command") - self.info("Starting command '%s'" % command) + self.info("Starting command '{}'".format(command)) if not command: # If no command was given (i.e. Application was used for dependency @@ -568,10 +568,10 @@ class LinuxApplication(ResourceManager): # We also set blocking = False, since we don't want the # thread to block until the execution finishes. (out, err), self._proc = self.execute_command(command, - env = env, - sudo = sudo, - forward_x11 = x11, - blocking = False) + env = env, + sudo = sudo, + forward_x11 = x11, + blocking = False) if self._proc.poll(): self.error(msg, out, err) @@ -592,15 +592,15 @@ class LinuxApplication(ResourceManager): # The command to run was previously uploaded to a bash script # during deployment, now we launch the remote script using 'run' # method from the node. - cmd = "bash %s" % os.path.join(self.app_home, "start.sh") + cmd = "bash {}".format(os.path.join(self.app_home, "start.sh")) (out, err), proc = self.node.run(cmd, self.run_home, - stdin = stdin, - stdout = stdout, - stderr = stderr, - sudo = sudo) + stdin = stdin, + stdout = stdout, + stderr = stderr, + sudo = sudo) # check if execution errors occurred - msg = " Failed to start command '%s' " % command + msg = " Failed to start command '{}' ".format(command) if proc.poll(): self.error(msg, out, err) @@ -615,11 +615,11 @@ class LinuxApplication(ResourceManager): # on the remote machine if not self.pid or not self.ppid: (out, err), proc = self.node.check_errors(self.run_home, - stderr = stderr) + stderr = stderr) # Out is what was written in the stderr file if err: - msg = " Failed to start command '%s' " % command + msg = " Failed to start command '{}' ".format(command) self.error(msg, out, err) raise RuntimeError(msg) @@ -630,7 +630,7 @@ class LinuxApplication(ResourceManager): if self.state == ResourceState.STARTED: - self.info("Stopping command '%s' " % command) + self.info("Stopping command '{}' ".format(command)) # If the command is running in foreground (it was launched using # the node 'execute' method), then we use the handler to the Popen @@ -643,12 +643,12 @@ class LinuxApplication(ResourceManager): # were retrieved if self.pid and self.ppid: (out, err), proc = self.node.kill(self.pid, self.ppid, - sudo = self._sudo_kill) + sudo = self._sudo_kill) """ # TODO: check if execution errors occurred if (proc and proc.poll()) or err: - msg = " Failed to STOP command '%s' " % self.get("command") + msg = " Failed to STOP command '{}' ".format(self.get("command")) self.error(msg, out, err) """ @@ -684,7 +684,7 @@ class LinuxApplication(ResourceManager): # retcode == 0 -> finished if retcode: out = "" - msg = " Failed to execute command '%s'" % self.get("command") + msg = " Failed to execute command '{}'".format(self.get("command")) err = self._proc.stderr.read() self.error(msg, out, err) self.do_fail() @@ -705,12 +705,12 @@ class LinuxApplication(ResourceManager): if status == ProcStatus.FINISHED: # If the program finished, check if execution # errors occurred - (out, err), proc = self.node.check_errors( - self.run_home) + (out, err), proc \ + = self.node.check_errors(self.run_home) if err: - msg = "Failed to execute command '%s'" % \ - self.get("command") + msg = "Failed to execute command '{}'"\ + .format(self.get("command")) self.error(msg, out, err) self.do_fail() else: @@ -721,52 +721,51 @@ class LinuxApplication(ResourceManager): return self._state def execute_command(self, command, - env=None, - sudo=False, - tty=False, - forward_x11=False, - blocking=False): + env = None, + sudo = False, + tty = False, + forward_x11 = False, + blocking = False): environ = "" if env: - environ = self.node.format_environment(env, inline=True) + environ = self.node.format_environment(env, inline = True) command = environ + command command = self.replace_paths(command) return self.node.execute(command, - sudo=sudo, - tty=tty, - forward_x11=forward_x11, - blocking=blocking) + sudo = sudo, + tty = tty, + forward_x11 = forward_x11, + blocking = blocking) - def replace_paths(self, command, node=None, app_home=None, run_home=None): + def replace_paths(self, command, node = None, app_home = None, run_home = None): """ Replace all special path tags with shell-escaped actual paths. """ if not node: - node=self.node + node = self.node if not app_home: - app_home=self.app_home + app_home = self.app_home if not run_home: run_home = self.run_home return ( command - .replace("${USR}", node.usr_dir) - .replace("${LIB}", node.lib_dir) - .replace("${BIN}", node.bin_dir) - .replace("${SRC}", node.src_dir) - .replace("${SHARE}", node.share_dir) - .replace("${EXP}", node.exp_dir) - .replace("${EXP_HOME}", node.exp_home) - .replace("${APP_HOME}", app_home) - .replace("${RUN_HOME}", run_home) - .replace("${NODE_HOME}", node.node_home) - .replace("${HOME}", node.home_dir) - ) + .replace("${USR}", node.usr_dir) + .replace("${LIB}", node.lib_dir) + .replace("${BIN}", node.bin_dir) + .replace("${SRC}", node.src_dir) + .replace("${SHARE}", node.share_dir) + .replace("${EXP}", node.exp_dir) + .replace("${EXP_HOME}", node.exp_home) + .replace("${APP_HOME}", app_home) + .replace("${RUN_HOME}", run_home) + .replace("${NODE_HOME}", node.node_home) + .replace("${HOME}", node.home_dir) + ) def valid_connection(self, guid): # TODO: Validate! return True - diff --git a/nepi/resources/linux/node.py b/nepi/resources/linux/node.py index 30438579..168686c2 100644 --- a/nepi/resources/linux/node.py +++ b/nepi/resources/linux/node.py @@ -299,7 +299,7 @@ class LinuxNode(ResourceManager): if out.find("Debian") == 0: self._os = OSType.DEBIAN - elif out.find("Ubuntu") ==0: + elif out.find("Ubuntu") == 0: self._os = OSType.UBUNTU elif out.find("Fedora release") == 0: self._os = OSType.FEDORA @@ -335,7 +335,7 @@ class LinuxNode(ResourceManager): @property def use_deb(self): - return (self.os & (OSType.DEBIAN|OSType.UBUNTU)) + return (self.os & (OSType.DEBIAN | OSType.UBUNTU)) @property def use_rpm(self): @@ -506,9 +506,10 @@ class LinuxNode(ResourceManager): (out, err), proc = self.execute(cmd, retry = 1, with_lock = True) - def search_for_child(self, pid, pids, ppid, family=[]): + def search_for_child(self, pid, pids, ppid, family=None): """ Recursive function to search for child. List A contains the pids and list B the parents (ppid) """ + family = family if family is not None else [] family.append(pid) for key, value in enumerate(ppid): if value == pid: @@ -766,7 +767,7 @@ class LinuxNode(ResourceManager): # create a temporal file with the content to upload # in python3 we need to open in binary mode if str is bytes mode = 'w' if isinstance(src, str) else 'wb' - f = tempfile.NamedTemporaryFile(mode=mode, delete=False) + f = tempfile.NamedTemporaryFile(mode = mode, delete = False) f.write(src) f.close() if executable: @@ -915,18 +916,18 @@ class LinuxNode(ResourceManager): return self.execute(cmd, with_lock = True) def run_and_wait(self, command, home, - shfile="cmd.sh", - env=None, - overwrite=True, - wait_run=True, - pidfile="pidfile", - ecodefile="exitcode", - stdin=None, - stdout="stdout", - stderr="stderr", - sudo=False, - tty=False, - raise_on_error=True): + shfile = "cmd.sh", + env = None, + overwrite = True, + wait_run = True, + pidfile = "pidfile", + ecodefile = "exitcode", + stdin = None, + stdout = "stdout", + stderr = "stderr", + sudo = False, + tty = False, + raise_on_error = True): """ Uploads the 'command' to a bash script in the host. Then runs the script detached in background in the host, and @@ -1008,10 +1009,10 @@ class LinuxNode(ResourceManager): return ExitCode.ERROR def upload_command(self, command, - shfile="cmd.sh", - ecodefile="exitcode", - overwrite=True, - env=None): + shfile = "cmd.sh", + ecodefile = "exitcode", + overwrite = True, + env = None): """ Saves the command as a bash script file in the remote host, and forces to save the exit code of the command execution to the ecodefile """ @@ -1021,7 +1022,7 @@ class LinuxNode(ResourceManager): # The exit code of the command will be stored in ecodefile command = " {{ {command} }} ; echo $? > {ecodefile} ;"\ - .format(command=command, ecodefile=ecodefile) + .format(command = command, ecodefile = ecodefile) # Export environment environ = self.format_environment(env) @@ -1029,9 +1030,9 @@ class LinuxNode(ResourceManager): # Add environ to command command = environ + command - return self.upload(command, shfile, text=True, overwrite=overwrite) + return self.upload(command, shfile, text = True, overwrite = overwrite) - def format_environment(self, env, inline=False): + def format_environment(self, env, inline = False): """ Formats the environment variables for a command to be executed either as an inline command (i.e. export PYTHONPATH=src/..; export LALAL= ..;python script.py) or @@ -1184,7 +1185,7 @@ class LinuxNode(ResourceManager): command = [] for d in dests: - command.append(" [ -f {dst} ] && echo '{dst}' ".format(dst=d) ) + command.append(" [ -f {dst} ] && echo '{dst}' ".format(dst = d) ) command = ";".join(command)