1 # Utility library for spawning remote asynchronous tasks
2 from nepi.util import server
8 Special value that when given to remote_spawn in stderr causes stderr to
9 redirect to whatever stdout was redirected to.
14 Process is still running
24 Process hasn't started running yet (this should be very rare)
27 def remote_spawn(command, pidfile, stdout='/dev/null', stderr=STDOUT, stdin='/dev/null', home=None, create_home=False, sudo=False,
28 host = None, port = None, user = None, agent = None,
29 ident_key = None, server_key = None,
30 tty = False, hostip = None):
32 Spawn a remote command such that it will continue working asynchronously.
35 command: the command to run - it should be a single line.
37 pidfile: path of a (ideally unique to this task) pidfile for tracking the process.
39 stdout: path of a file to redirect standard output to - must be a string.
41 stderr: path of a file to redirect standard error to - string or the special STDOUT value
42 to redirect to the same file stdout was redirected to. Defaults to STDOUT.
43 stdin: path of a file with input to be piped into the command's standard input
45 home: path of a folder to use as working directory - should exist, unless you specify create_home
47 create_home: if True, the home folder will be created first with mkdir -p
49 sudo: whether the command needs to be executed as root
51 host/port/user/agent/ident_key: see nepi.util.server.popen_ssh_command
54 (stdout, stderr), process
56 Of the spawning process, which only captures errors at spawning time.
57 Usually only useful for diagnostics.
59 # Start process in a "daemonized" way, using nohup and heavy
60 # stdin/out redirection to avoid connection issues
66 daemon_command = '{ { %(command)s > %(stdout)s 2>%(stderr)s < %(stdin)s & } ; echo $! 1 > %(pidfile)s ; }' % {
68 'pidfile' : server.shell_escape(pidfile),
75 cmd = "%(create)s%(gohome)s rm -f %(pidfile)s ; %(sudo)s nohup bash -c %(command)s " % {
76 'command' : server.shell_escape(daemon_command),
78 'sudo' : 'sudo -S' if sudo else '',
80 'pidfile' : server.shell_escape(pidfile),
81 'gohome' : 'cd %s ; ' % (server.shell_escape(home),) if home else '',
82 'create' : 'mkdir -p %s ; ' % (server.shell_escape,) if create_home else '',
85 (out,err),proc = server.popen_ssh_command(
91 ident_key = ident_key,
92 server_key = server_key,
98 raise RuntimeError, "Failed to set up application on host %s: %s %s" % (host, out,err,)
100 return (out,err),proc
103 def remote_check_pid(pidfile,
104 host = None, port = None, user = None, agent = None,
105 ident_key = None, server_key = None, hostip = None):
107 Check the pidfile of a process spawned with remote_spawn.
110 pidfile: the pidfile passed to remote_span
112 host/port/user/agent/ident_key: see nepi.util.server.popen_ssh_command
116 A (pid, ppid) tuple useful for calling remote_status and remote_kill,
117 or None if the pidfile isn't valid yet (maybe the process is still starting).
120 (out,err),proc = server.popen_ssh_command(
121 "cat %(pidfile)s" % {
128 ident_key = ident_key,
129 server_key = server_key,
138 return map(int,out.strip().split(' ',1))
140 # Ignore, many ways to fail that don't matter that much
145 def remote_status(pid, ppid,
146 host = None, port = None, user = None, agent = None,
147 ident_key = None, server_key = None, hostip = None):
149 Check the status of a process spawned with remote_spawn.
152 pid/ppid: pid and parent-pid of the spawned process. See remote_check_pid
154 host/port/user/agent/ident_key: see nepi.util.server.popen_ssh_command
158 One of NOT_STARTED, RUNNING, FINISHED
161 (out,err),proc = server.popen_ssh_command(
162 "ps --pid %(pid)d -o pid | grep -c %(pid)d ; true" % {
170 ident_key = ident_key,
171 server_key = server_key,
181 status = bool(int(out.strip()))
184 logging.warn("Error checking remote status:\n%s%s\n", out, err)
185 # Ignore, many ways to fail that don't matter that much
187 return RUNNING if status else FINISHED
191 def remote_kill(pid, ppid, sudo = False,
192 host = None, port = None, user = None, agent = None,
193 ident_key = None, server_key = None, hostip = None,
196 Kill a process spawned with remote_spawn.
198 First tries a SIGTERM, and if the process does not end in 10 seconds,
202 pid/ppid: pid and parent-pid of the spawned process. See remote_check_pid
204 sudo: whether the command was run with sudo - careful killing like this.
206 host/port/user/agent/ident_key: see nepi.util.server.popen_ssh_command
210 Nothing, should have killed the process
214 subkill = "$(ps --ppid %(pid)d -o pid h)" % { 'pid' : pid }
218 SUBKILL="%(subkill)s" ;
219 %(sudo)s kill -- -%(pid)d $SUBKILL || /bin/true
220 %(sudo)s kill %(pid)d $SUBKILL || /bin/true
221 for x in 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 ; do
223 if [ `ps --pid %(pid)d -o pid | grep -c %(pid)d` == '0' ]; then
226 %(sudo)s kill -- -%(pid)d $SUBKILL || /bin/true
227 %(sudo)s kill %(pid)d $SUBKILL || /bin/true
231 if [ `ps --pid %(pid)d -o pid | grep -c %(pid)d` != '0' ]; then
232 %(sudo)s kill -9 -- -%(pid)d $SUBKILL || /bin/true
233 %(sudo)s kill -9 %(pid)d $SUBKILL || /bin/true
237 cmd = "( %s ) >/dev/null 2>/dev/null </dev/null &" % (cmd,)
239 (out,err),proc = server.popen_ssh_command(
243 'sudo' : 'sudo -S' if sudo else '',
250 ident_key = ident_key,
251 server_key = server_key,
255 # wait, don't leave zombies around