Supporting many concurrent LinuxApplications on same LinuxNode

[nepi.git] / src / neco / util / sshfuncs.py

diff --git a/src/neco/util/sshfuncs.py b/src/neco/util/sshfuncs.py
index e81c558..b5d8f0e 100644 (file)
--- a/src/neco/util/sshfuncs.py
+++ b/src/neco/util/sshfuncs.py
@@ -1,18 +1,29 @@
 import base64
 import errno
+import hashlib
+import logging
 import os
 import os.path
+import re
 import select
 import signal
 import socket
 import subprocess
 import time
-import traceback
-import re
 import tempfile
-import hashlib
 
-TRACE = os.environ.get("NEPI_TRACE", "false").lower() in ("true", "1", "on")
+
+logger = logging.getLogger("sshfuncs")
+
+def log(msg, level, out = None, err = None):
+    if out:
+        msg += " - OUT: %s " % out
+
+    if err:
+        msg += " - ERROR: %s " % err
+
+    logger.log(level, msg)
+
 
 if hasattr(os, "devnull"):
     DEV_NULL = os.devnull
@@ -176,7 +187,7 @@ def rexec(command, host, user,
         env = None,
         tty = False,
         timeout = None,
-        retry = 0,
+        retry = 3,
         err_on_timeout = True,
         connect_timeout = 30,
         persistent = True,
@@ -226,7 +237,7 @@ def rexec(command, host, user,
 
     args.append(command)
 
-    for x in xrange(retry or 3):
+    for x in xrange(retry):
         # connects to the remote host and starts a remote connection
         proc = subprocess.Popen(args, 
                 stdout = subprocess.PIPE,
@@ -239,21 +250,31 @@ def rexec(command, host, user,
     
         try:
             out, err = _communicate(proc, stdin, timeout, err_on_timeout)
-            if TRACE:
-                print "COMMAND host %s, command %s, out %s, error %s" % (host, " ".join(args), out, err)
+            msg = " rexec - host %s - command %s " % (host, " ".join(args))
+            log(msg, logging.DEBUG, out, err)
 
             if proc.poll():
+                skip = False
+
                 if err.strip().startswith('ssh: ') or err.strip().startswith('mux_client_hello_exchange: '):
                     # SSH error, can safely retry
-                    continue
+                    skip = True 
                 elif retry:
                     # Probably timed out or plain failed but can retry
+                    skip = True 
+                
+                if skip:
+                    t = x*2
+                    msg = "SLEEPING %d ... ATEMP %d - host %s - command %s " % ( 
+                            t, x, host, " ".join(args))
+                    log(msg, logging.DEBUG)
+
+                    time.sleep(t)
                     continue
             break
         except RuntimeError, e:
-            if TRACE:
-                print "EXCEPTION host %s, command %s, out %s, error %s, exception TIMEOUT ->  %s" % (
-                        host, " ".join(args), out, err, e.args)
+            msg = " rexec EXCEPTION - host %s - command %s - TIMEOUT -> %s" % (host, " ".join(args), e.args)
+            log(msg, logging.DEBUG, out, err)
 
             if retry <= 0:
                 raise
@@ -285,8 +306,8 @@ def rcopy(source, dest,
     in which case it is advised that the destination be a folder.
     """
     
-    if TRACE:
-        print "scp", source, dest
+    msg = " rcopy - scp %s %s " % (source, dest)
+    log(msg, logging.DEBUG)
     
     if isinstance(source, file) and source.tell() == 0:
         source = source.name