+def _retry_rexec(args,
+ log_msg,
+ stdout = subprocess.PIPE,
+ stdin = subprocess.PIPE,
+ stderr = subprocess.PIPE,
+ env = None,
+ retry = 3,
+ tmp_known_hosts = None,
+ blocking = True):
+
+ for x in xrange(retry):
+ # display command actually invoked when debug is turned on
+ message = " ".join( [ "'{}'".format(arg) for arg in args ] )
+ log("sshfuncs: invoking {}".format(message), logging.DEBUG)
+ # connects to the remote host and starts a remote connection
+ proc = subprocess.Popen(
+ args,
+ env = env,
+ stdout = stdout,
+ stdin = stdin,
+ stderr = stderr,
+ )
+ # attach tempfile object to the process, to make sure the file stays
+ # alive until the process is finished with it
+ proc._known_hosts = tmp_known_hosts
+
+ # The argument block == False forces to rexec to return immediately,
+ # without blocking
+ try:
+ err = out = " "
+ if blocking:
+ #(out, err) = proc.communicate()
+ # The method communicate was re implemented for performance issues
+ # when using python subprocess communicate method the ssh commands
+ # last one minute each
+ out, err = _communicate(proc, input=None)
+
+ elif stdout:
+ out = proc.stdout.read()
+ if proc.poll() and stderr:
+ err = proc.stderr.read()
+
+ log(log_msg, logging.DEBUG, out, err)
+
+ if proc.poll():
+ skip = False
+
+ if err.strip().startswith('ssh: ') or err.strip().startswith('mux_client_hello_exchange: '):
+ # SSH error, can safely retry
+ skip = True
+ elif retry:
+ # Probably timed out or plain failed but can retry
+ skip = True
+
+ if skip:
+ t = x*2
+ msg = "SLEEPING %d ... ATEMPT %d - command %s " % (
+ t, x, " ".join(args))
+ log(msg, logging.DEBUG)
+
+ time.sleep(t)
+ continue
+ break
+ except RuntimeError as e:
+ msg = " rexec EXCEPTION - TIMEOUT -> %s \n %s" % ( e.args, log_msg )
+ log(msg, logging.DEBUG, out, err)
+
+ if retry <= 0:
+ raise
+ retry -= 1
+
+ return ((out, err), proc)
+
git://git.onelab.eu / nepi.git / blobdiff