X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=src%2Fnepi%2Fresources%2Flinux%2Fapplication.py;h=302e9fd9507bf130c4d89e8c41d265c8f5c35f0d;hb=5857fa2d26b3aa135391cf936ccde907b9ff8c68;hp=c1de54327265b4b4351172f1f3e7e7bc53a59c3c;hpb=914bb8e91ae1ab342a2c4b5d9b33a7ef2b46d63f;p=nepi.git diff --git a/src/nepi/resources/linux/application.py b/src/nepi/resources/linux/application.py index c1de5432..302e9fd9 100644 --- a/src/nepi/resources/linux/application.py +++ b/src/nepi/resources/linux/application.py @@ -43,34 +43,34 @@ class LinuxApplication(ResourceManager): .. note:: - A LinuxApplication RM represents a process that can be executed in - a remote Linux host using SSH. + A LinuxApplication RM represents a process that can be executed in + a remote Linux host using SSH. - The LinuxApplication RM takes care of uploadin sources and any files - needed to run the experiment, to the remote host. - It also allows to provide source compilation (build) and installation - instructions, and takes care of automating the sources build and - installation tasks for the user. + The LinuxApplication RM takes care of uploadin sources and any files + needed to run the experiment, to the remote host. + It also allows to provide source compilation (build) and installation + instructions, and takes care of automating the sources build and + installation tasks for the user. - It is important to note that files uploaded to the remote host have - two possible scopes: single-experiment or multi-experiment. - Single experiment files are those that will not be re-used by other - experiments. Multi-experiment files are those that will. - Sources and shared files are always made available to all experiments. + It is important to note that files uploaded to the remote host have + two possible scopes: single-experiment or multi-experiment. + Single experiment files are those that will not be re-used by other + experiments. Multi-experiment files are those that will. + Sources and shared files are always made available to all experiments. - Directory structure: + Directory structure: - The directory structure used by LinuxApplication RM at the Linux - host is the following: + The directory structure used by LinuxApplication RM at the Linux + host is the following: - ${HOME}/nepi-usr --> Base directory for multi-experiment files + ${HOME}/.nepi/nepi-usr --> Base directory for multi-experiment files | ${LIB} |- /lib --> Base directory for libraries ${BIN} |- /bin --> Base directory for binary files ${SRC} |- /src --> Base directory for sources ${SHARE} |- /share --> Base directory for other files - ${HOME}/nepi-exp --> Base directory for single-experiment files + ${HOME}/.nepi/nepi-exp --> Base directory for single-experiment files | ${EXP_HOME} |- / --> Base directory for experiment exp-id | @@ -90,43 +90,43 @@ class LinuxApplication(ResourceManager): command = Attribute("command", "Command to execute at application start. " "Note that commands will be executed in the ${RUN_HOME} directory, " "make sure to take this into account when using relative paths. ", - flags = Flags.ExecReadOnly) + flags = Flags.Design) forward_x11 = Attribute("forwardX11", "Enables X11 forwarding for SSH connections", - flags = Flags.ExecReadOnly) + flags = Flags.Design) env = Attribute("env", "Environment variables string for command execution", - flags = Flags.ExecReadOnly) + flags = Flags.Design) sudo = Attribute("sudo", "Run with root privileges", - flags = Flags.ExecReadOnly) + flags = Flags.Design) depends = Attribute("depends", "Space-separated list of packages required to run the application", - flags = Flags.ExecReadOnly) + flags = Flags.Design) sources = Attribute("sources", - "Space-separated list of regular files to be uploaded to ${SRC} " + "semi-colon separated list of regular files to be uploaded to ${SRC} " "directory prior to building. Archives won't be expanded automatically. " "Sources are globally available for all experiments unless " "cleanHome is set to True (This will delete all sources). ", - flags = Flags.ExecReadOnly) + flags = Flags.Design) files = Attribute("files", "Space-separated list of regular miscellaneous files to be uploaded " "to ${SHARE} directory. " "Files are globally available for all experiments unless " "cleanHome is set to True (This will delete all files). ", - flags = Flags.ExecReadOnly) + flags = Flags.Design) libs = Attribute("libs", "Space-separated list of libraries (e.g. .so files) to be uploaded " "to ${LIB} directory. " "Libraries are globally available for all experiments unless " "cleanHome is set to True (This will delete all files). ", - flags = Flags.ExecReadOnly) + flags = Flags.Design) bins = Attribute("bins", "Space-separated list of binary files to be uploaded " "to ${BIN} directory. " "Binaries are globally available for all experiments unless " "cleanHome is set to True (This will delete all files). ", - flags = Flags.ExecReadOnly) + flags = Flags.Design) code = Attribute("code", "Plain text source code to be uploaded to the ${APP_HOME} directory. ", - flags = Flags.ExecReadOnly) + flags = Flags.Design) build = Attribute("build", "Build commands to execute after deploying the sources. " "Sources are uploaded to the ${SRC} directory and code " @@ -135,16 +135,16 @@ class LinuxApplication(ResourceManager): "./configure && make && make clean.\n" "Make sure to make the build commands return with a nonzero exit " "code on error.", - flags = Flags.ReadOnly) + flags = Flags.Design) install = Attribute("install", "Commands to transfer built files to their final destinations. " "Install commands are executed after build commands. ", - flags = Flags.ReadOnly) + flags = Flags.Design) stdin = Attribute("stdin", "Standard input for the 'command'", - flags = Flags.ExecReadOnly) + flags = Flags.Design) tear_down = Attribute("tearDown", "Command to be executed just before " "releasing the resource", - flags = Flags.ReadOnly) + flags = Flags.Design) cls._register_attribute(command) cls._register_attribute(forward_x11) @@ -163,8 +163,8 @@ class LinuxApplication(ResourceManager): @classmethod def _register_traces(cls): - stdout = Trace("stdout", "Standard output stream") - stderr = Trace("stderr", "Standard error stream") + stdout = Trace("stdout", "Standard output stream", enabled = True) + stderr = Trace("stderr", "Standard error stream", enabled = True) cls._register_trace(stdout) cls._register_trace(stderr) @@ -187,14 +187,14 @@ class LinuxApplication(ResourceManager): # timestamp of last state check of the application self._last_state_check = tnow() - + def log_message(self, msg): return " guid %d - host %s - %s " % (self.guid, self.node.get("hostname"), msg) @property def node(self): - node = self.get_connected(LinuxNode.rtype()) + node = self.get_connected(LinuxNode.get_rtype()) if node: return node[0] return None @@ -227,10 +227,13 @@ class LinuxApplication(ResourceManager): """ return self.get("forwardX11") or self._in_foreground + def trace_filepath(self, filename): + return os.path.join(self.run_home, filename) + def trace(self, name, attr = TraceAttr.ALL, block = 512, offset = 0): self.info("Retrieving '%s' trace %s " % (name, attr)) - path = os.path.join(self.run_home, name) + path = self.trace_filepath(name) command = "(test -f %s && echo 'success') || echo 'error'" % path (out, err), proc = self.node.execute(command) @@ -271,6 +274,18 @@ class LinuxApplication(ResourceManager): return out def do_provision(self): + # take a snapshot of the system if user is root + # to assure cleanProcess kill every nepi process + if self.node.get("username") == 'root': + import pickle + procs = dict() + ps_aux = "ps aux |awk '{print $2,$11}'" + (out, err), proc = self.node.execute(ps_aux) + for line in out.strip().split("\n"): + parts = line.strip().split(" ") + procs[parts[0]] = parts[1] + pickle.dump(procs, open("/tmp/save.proc", "wb")) + # create run dir for application self.node.mkdir(self.run_home) @@ -319,7 +334,7 @@ class LinuxApplication(ResourceManager): super(LinuxApplication, self).do_provision() - def upload_start_command(self): + def upload_start_command(self, overwrite = False): # Upload command to remote bash script # - only if command can be executed in background and detached command = self.get("command") @@ -339,7 +354,7 @@ class LinuxApplication(ResourceManager): self.node.upload_command(command, shfile = shfile, env = env, - overwrite = False) + overwrite = overwrite) def execute_deploy_command(self, command): if command: @@ -355,15 +370,18 @@ class LinuxApplication(ResourceManager): stdout = "deploy_stdout", stderr = "deploy_stderr") - def upload_sources(self): + def upload_sources(self, src_dir = None): sources = self.get("sources") command = "" + if not src_dir: + src_dir = self.node.src_dir + if sources: self.info("Uploading sources ") - sources = sources.split(' ') + sources = map(str.strip, sources.split(";")) # Separate sources that should be downloaded from # the web, from sources that should be uploaded from @@ -376,15 +394,16 @@ class LinuxApplication(ResourceManager): command.append( " ( " # Check if the source already exists - " ls ${SRC}/%(basename)s " + " ls %(src_dir)s/%(basename)s " " || ( " # If source doesn't exist, download it and check # that it it downloaded ok - " wget -c --directory-prefix=${SRC} %(source)s && " - " ls ${SRC}/%(basename)s " + " wget -c --directory-prefix=%(src_dir)s %(source)s && " + " ls %(src_dir)s/%(basename)s " " ) ) " % { "basename": os.path.basename(source), - "source": source + "source": source, + "src_dir": src_dir }) command = " && ".join(command) @@ -393,8 +412,8 @@ class LinuxApplication(ResourceManager): command = self.replace_paths(command) if sources: - sources = ' '.join(sources) - self.node.upload(sources, self.node.src_dir, overwrite = False) + sources = ';'.join(sources) + self.node.upload(sources, src_dir, overwrite = False) return command @@ -491,8 +510,8 @@ class LinuxApplication(ResourceManager): if not command: # If no command was given (i.e. Application was used for dependency - # installation), then the application is directly marked as FINISHED - super(LinuxApplication, self).do_finish() + # installation), then the application is directly marked as STOPPED + super(LinuxApplication, self).set_stopped() else: if self.in_foreground: self._run_in_foreground() @@ -507,10 +526,6 @@ class LinuxApplication(ResourceManager): x11 = self.get("forwardX11") env = self.get("env") - # For a command being executed in foreground, if there is stdin, - # it is expected to be text string not a file or pipe - stdin = self.get("stdin") or None - # Command will be launched in foreground and attached to the # terminal using the node 'execute' in non blocking mode. @@ -521,7 +536,6 @@ class LinuxApplication(ResourceManager): (out, err), self._proc = self.execute_command(command, env = env, sudo = sudo, - stdin = stdin, forward_x11 = x11, blocking = False) @@ -598,7 +612,7 @@ class LinuxApplication(ResourceManager): sudo = self._sudo_kill) # TODO: check if execution errors occurred - if proc.poll() or err: + if (proc and proc.poll()) or err: msg = " Failed to STOP command '%s' " % self.get("command") self.error(msg, out, err) @@ -636,7 +650,7 @@ class LinuxApplication(ResourceManager): self.do_fail() elif retcode == 0: - self.do_finish() + self.set_stopped() else: # We need to query the status of the command we launched in # background. In order to avoid overwhelming the remote host and @@ -660,7 +674,7 @@ class LinuxApplication(ResourceManager): self.error(msg, out, err) self.do_fail() else: - self.do_finish() + self.set_stopped() self._last_state_check = tnow() @@ -669,7 +683,6 @@ class LinuxApplication(ResourceManager): def execute_command(self, command, env = None, sudo = False, - stdin = None, forward_x11 = False, blocking = False): @@ -681,7 +694,6 @@ class LinuxApplication(ResourceManager): return self.node.execute(command, sudo = sudo, - stdin = stdin, forward_x11 = forward_x11, blocking = blocking)