replace os.system with subprocess.call - add a timeout option in the mix

today for the first time ever a slice was not answering ssh but the ssh client would hang forever..

diff --git a/system/TestPlc.py b/system/TestPlc.py
index 8b5018e..c378af6 100644 (file)
--- a/system/TestPlc.py
+++ b/system/TestPlc.py
@@ -233,7 +233,7 @@ class TestPlc:
         # warning, we're now building 'sface' so let's be a bit more picky
         # full builds are expected to return with 0 here
         utils.header("Checking if build provides SFA package...")
-        retcod = os.system("curl --silent {}/ | grep -q sfa-".format(rpms_url)) == 0
+        retcod = utils.system("curl --silent {}/ | grep -q sfa-".format(rpms_url)) == 0
         encoded = 'yes' if retcod else 'no'
         with open(has_sfa_cache_filename,'w') as cache:
             cache.write(encoded)

diff --git a/system/TestSlice.py b/system/TestSlice.py
index a312657..a436ecd 100644 (file)
--- a/system/TestSlice.py
+++ b/system/TestSlice.py
@@ -28,7 +28,7 @@ class CompleterTaskSliceSsh (CompleterTask):
         site_spec, node_spec = self.test_plc.locate_hostname(self.hostname)
         test_ssh = TestSsh (self.hostname, key=self.private_key, username=self.slicename)
         full_command = test_ssh.actual_command(self.command)
-        retcod = utils.system (full_command, silent=silent)
+        retcod = utils.system (full_command, silent=silent, timeout=10)
         if self.dry_run:        return True
         if self.expected:       return retcod == 0
         else:                   return retcod != 0
@@ -222,7 +222,7 @@ class TestSlice:
         site_spec, node_spec = self.test_plc.locate_hostname(hostname)
         test_ssh = TestSsh (hostname, key=private_key, username=self.name())
         full_command = test_ssh.actual_command(command)
-        retcod = utils.system (full_command, silent=True)
+        retcod = utils.system (full_command, silent=True, timeout=10)
         if getattr(options, 'dry_run', None):
             return True
         if expected:
@@ -257,5 +257,5 @@ class TestSlice:
                 else:
                     print("Sliver rootfs {} still present - this is unexpected".format(rootfs))
                     utils.system(self.test_ssh.actual_command("ls -l {rootfs}; du -hs {rootfs}".format(**locals()),
-                                                              dry_run=self.dry_run))
+                                                              dry_run=self.dry_run, timeout=20))
         return [ CompleterTaskRootfs (nodename, qemuname) for (nodename,qemuname) in node_infos ]

diff --git a/system/utils.py b/system/utils.py
index 51205bd..55a990c 100644 (file)
--- a/system/utils.py
+++ b/system/utils.py
@@ -27,8 +27,9 @@ def pprint(message, spec, depth=2):
     PrettyPrinter(indent=8, depth=depth).pprint(spec)
 
 
-
-def system(command, background=False, silent=False, dry_run=None):
+# set a default timeout to 15 minutes - this should be plenty even for installations
+# call with timeout=None if the intention really is to wait until full completion
+def system(command, background=False, silent=False, dry_run=None, timeout=15*60):
     dry_run = dry_run if dry_run is not None else getattr(options, 'dry_run', False)
     if dry_run:
         print('dry_run:', command)
@@ -50,7 +51,11 @@ def system(command, background=False, silent=False, dry_run=None):
         sys.stdout.flush()
     if not silent:
         command = "set -x; " + command
-    return os.system(command)
+    try:
+        return subprocess.call(command, shell=True, timeout=timeout)
+    except subprocess.TimeoutExpired as e:
+        header("TIMEOUT when running command {}- {}".format(command, e))
+        return -1
 
 ### WARNING : this ALWAYS does its job, even in dry_run mode
 def output_of (command):