fix testmaster sensing : tests that break very early (like, when the IP pool is full...

[tests.git] / system / Substrate.py

diff --git a/system/Substrate.py b/system/Substrate.py
index b0d28f0..a4f8c7d 100644 (file)
--- a/system/Substrate.py
+++ b/system/Substrate.py
@@ -708,11 +708,11 @@ class QemuBox (Box):
     def sense(self, options):
         print 'qn',
         modules=self.backquote_ssh(['lsmod']).split('\n')
-        self._driver='*NO kqemu/kmv_intel MODULE LOADED*'
+        self._driver='*NO kqemu/kvm_intel MODULE LOADED*'
         for module in modules:
             if module.find('kqemu')==0:
                 self._driver='kqemu module loaded'
-            # kvm might be loaded without vkm_intel (we dont have AMD)
+            # kvm might be loaded without kvm_intel (we dont have AMD)
             elif module.find('kvm_intel')==0:
                 self._driver='kvm_intel module loaded'
         ########## find out running pids
@@ -730,13 +730,13 @@ class QemuBox (Box):
             header(">>%s<<"%line)
         ########## retrieve alive instances and map to build
         live_builds=[]
-        command=['grep','.','*/*/qemu.pid','/dev/null']
+        command=['grep','.','/vservers/*/*/qemu.pid','/dev/null']
         pid_lines=self.backquote_ssh(command,trash_err=True).split('\n')
         for pid_line in pid_lines:
             if not pid_line.strip(): continue
             # expect <build>/<nodename>/qemu.pid:<pid>pid
             try:
-                (buildname,nodename,tail)=pid_line.split('/')
+                (_,__,buildname,nodename,tail)=pid_line.split('/')
                 (_,pid)=tail.split(':')
                 q=self.qemu_instance_by_pid (pid)
                 if not q: continue
@@ -746,14 +746,14 @@ class QemuBox (Box):
         # retrieve timestamps
         if not live_builds: return
         command=   ['grep','.']
-        command += ['%s/*/timestamp'%b for b in live_builds]
+        command += ['/vservers/%s/*/timestamp'%b for b in live_builds]
         command += ['/dev/null']
         ts_lines=self.backquote_ssh(command,trash_err=True).split('\n')
         for ts_line in ts_lines:
             if not ts_line.strip(): continue
             # expect <build>/<nodename>/timestamp:<timestamp>
             try:
-                (buildname,nodename,tail)=ts_line.split('/')
+                (_,__,buildname,nodename,tail)=ts_line.split('/')
                 nodename=nodename.replace('qemu-','')
                 (_,timestamp)=tail.split(':')
                 timestamp=int(timestamp)
@@ -799,7 +799,8 @@ class TestInstance:
         else:                   msg += " !!!pids=%s!!!"%self.pids
         msg += " @%s"%self.pretty_timestamp()
         if self.broken_steps:
-            msg += " [BROKEN=" + " ".join( [ "%s@%s"%(s,i) for (i,s) in self.broken_steps ] ) + "]"
+            # sometimes we have an empty plcindex
+            msg += " [BROKEN=" + " ".join( [ "%s@%s"%(s,i) if i else s for (i,s) in self.broken_steps ] ) + "]"
         return msg
 
 class TestBox (Box):
@@ -847,6 +848,7 @@ class TestBox (Box):
 
     matcher_proc=re.compile (".*/proc/(?P<pid>[0-9]+)/cwd.*/root/(?P<buildname>[^/]+)$")
     matcher_grep=re.compile ("/root/(?P<buildname>[^/]+)/logs/trace.*:TRACE:\s*(?P<plcindex>[0-9]+).*step=(?P<step>\S+).*")
+    matcher_grep_missing=re.compile ("grep: /root/(?P<buildname>[^/]+)/logs/trace: No such file or directory")
     def sense (self, options):
         print 'tm',
         self.sense_uptime()
@@ -868,10 +870,19 @@ class TestBox (Box):
                 t=self.add_timestamp(buildname,timestamp)
             except:  print 'WARNING, could not parse ts line',ts_line
 
-        command=['bash','-c',"grep KO /root/*/logs/trace-* /dev/null" ]
+        # let's try to be robust here -- tests that fail very early like e.g.
+        # "Cannot make space for a PLC instance: vplc IP pool exhausted", that occurs as part of provision
+        # will result in a 'trace' symlink to an inexisting 'trace-<>.txt' because no step has gone through
+        # simple 'trace' sohuld exist though as it is created by run_log
+        command=['bash','-c',"grep KO /root/*/logs/trace /dev/null 2>&1" ]
         trace_lines=self.backquote_ssh (command).split('\n')
         for line in trace_lines:
             if not line.strip(): continue
+            m=TestBox.matcher_grep_missing.match(line)
+            if m:
+                buildname=m.group('buildname')
+                self.add_broken(buildname,'','NO STEP DONE')
+                continue
             m=TestBox.matcher_grep.match(line)
             if m: 
                 buildname=m.group('buildname')