k22-f8-32 builds also test in dual mode
[infrastructure.git] / scripts / manage-infrastructure.py
index f4fd9ad..3312d92 100755 (executable)
@@ -12,10 +12,12 @@ class BuildBoxes:
     testmaster = 'testmaster'
     build_boxes = [ "mirror", "liquid", "reed", "velvet", ]
     plc_boxes = [ "testplc" ]
+    # qemu32-5 is officially dead
     qemu_boxes = \
         [ "qemu64-%d"%i for i in range (1,4) ] + \
-        [ "qemu32-%d"%i for i in range (1,6) ]
+        [ "qemu32-%d"%i for i in range (1,5) ]
     test_boxes = plc_boxes + qemu_boxes
+    testmaster_boxes = [ testmaster ]
 
     def __init__ (self):
         # dummy defaults
@@ -133,6 +135,30 @@ class BuildBoxes:
                     command=['ps','-o','pid,command'] + [ pid for pid in pids.split("\n") if pid]
                     self.run_ssh(box,command,"Active build processes on %s (%s)"%(box,uptime),True)
 
+    # this one is more accurate as it locates processes in the vservers as well
+    # but it's so sloooowww
+    def handle_build_box_deep (self,box):
+        if not self.options.probe:
+            self.reboot(box)
+        else:
+            command=['uptime']
+            uptime=self.backquote_ssh(box,command,True).strip()
+
+            command=['vps','-e']
+            if self.options.dry_run:
+                self.run_ssh(box,command,None)
+            else:
+                # simulate grep vbuild
+                vps_lines=[ line for line in self.backquote_ssh(box,command,True).split("\n")
+                            if line.find('vbuild') >= 0]
+                pids=[ line.split()[0] for line in vps_lines ]
+                if not pids:
+                    self.header ('No build process on %s (%s)'%(box,uptime))
+                else:
+                    command=['vps','-o','pid,command'] + pids
+                    self.run_ssh(box,command,"Active build processes on %s (%s)"%(box,uptime),True)
+
+
     vplc_matcher = re.compile(".*(vplc[0-9]+$)")
     def vplcname (self, vservername):
         match = self.vplc_matcher.match(vservername)
@@ -185,7 +211,6 @@ class BuildBoxes:
         if match: return match.groups(0)
         else: return ""
 
-
     def handle_qemu_box (self,box):
         if not self.options.probe:
             self.reboot(box)
@@ -212,13 +237,45 @@ class BuildBoxes:
                         if not ps_line or ps_line.find('PID') >=0 : continue
                         print self.margin_outline(self.vnodename(ps_line)), ps_line
 
+    # the ouput of ps -o pid,command gives us <pid> bash <buildname>/run_log
+    def testmaster_buildname (self, ps_line):
+        chunks=ps_line.split()
+        path=chunks[2]
+        [buildname,command]=path.split('/')
+        return buildname
+
+    def handle_testmaster_box (self, box):
+        if not self.options.probe: 
+            pass
+        else:
+            command=['pgrep','run_log']
+            if self.options.dry_run:
+                self.run_ssh(box,command,None)
+            else:
+                pids=self.backquote_ssh(box,command)
+                if not pids:
+                    self.header ('No run_log process on %s'%box)
+                else:
+                    self.header ("Active run_log processes on %s"%(box))
+                    command=['ps','-o','pid,command'] + [ pid for pid in pids.split("\n") if pid]
+                    ps_lines = self.backquote_ssh (box,command).split("\n")
+                    for ps_line in ps_lines:
+                        if not ps_line or ps_line.find('PID') >=0 : continue
+                        print self.margin_outline(self.testmaster_buildname(ps_line)), ps_line
+        
+
     def handle_box(self,box,type):
         if box in self.qemu_boxes:
             if type=="qemu": self.handle_qemu_box(self.fqdn(box))
         elif box in self.plc_boxes:
             if type=="plc":  self.handle_plc_box(self.fqdn(box))
+        elif box in self.testmaster_boxes:
+            if type=='testmaster': self.handle_testmaster_box(self.fqdn(box))
         elif type=="build":
-            self.handle_build_box(self.fqdn(box))
+            if self.options.deep:
+                self.handle_build_box_deep(self.fqdn(box))
+            else:
+                self.handle_build_box(self.fqdn(box))
 
     def handle_disk (self,box):
         box=self.fqdn(box)
@@ -226,7 +283,7 @@ class BuildBoxes:
 
     def main (self):
         usage="""%prog [options] [hostname..(s)]
-Default is to act on test boxes only (with trackers clean)"""
+Default is to act on test boxes only"""
         parser = OptionParser (usage=usage)
         parser.add_option ("-n","--dry-run",action="store_true",dest="dry_run",default=False,
                            help="Dry run")
@@ -240,10 +297,14 @@ Default is to act on test boxes only (with trackers clean)"""
                            help="Acts on build and test boxes")
         parser.add_option ("-b","--build",action="store_true",dest="build_only",default=False,
                            help="Acts on build boxes only")
+        parser.add_option ("-e","--deep",action="store_true", dest="deep", default=False,
+                           help="on build boxes, shows vbuild processes in vservers as well; signif. slower")
         parser.add_option ("-q","--qemu",action="store_true",dest="qemu_only",default=False,
                            help="Only acts on the qemu boxes")
         parser.add_option ("-t","--trackers",action="store_true",dest="trackers_only",default=False,
                            help="Only wipes trackers")
+        parser.add_option ("-m","--master",action="store_true",dest="testmaster_only",default=False,
+                           help="Display the testmaster status")
         parser.add_option ("-d","--disk",action="store_true",dest="show_disk",default=False,
                            help="Only inspects disk status")
 
@@ -256,7 +317,7 @@ Default is to act on test boxes only (with trackers clean)"""
             self.do_tracker_plcs = False
             self.do_tracker_qemus = False
         elif self.options.all_boxes:
-            self.boxes=self.test_boxes + self.build_boxes
+            self.boxes=self.test_boxes + self.build_boxes + self.testmaster_boxes
             self.do_tracker_plcs = True
             self.do_tracker_qemus = True
         elif self.options.build_only:
@@ -271,6 +332,10 @@ Default is to act on test boxes only (with trackers clean)"""
             self.boxes=self.plc_boxes
             self.do_tracker_plcs = True
             self.do_tracker_qemus = False
+        elif self.options.testmaster_only:
+            self.boxes=self.testmaster_boxes
+            self.do_tracker_plcs = False
+            self.do_tracker_qemus = False
         elif self.options.trackers_only:
             self.boxes = []
             self.do_tracker_plcs = True
@@ -285,14 +350,16 @@ Default is to act on test boxes only (with trackers clean)"""
             for box in self.boxes: self.handle_disk(box)
             return
 
-        # ALL OTHERS
-        for box in self.boxes:  self.handle_box (box,"build")
         # PLCS
         if self.do_tracker_plcs:self.handle_tracker_plcs ()
         for box in self.boxes:  self.handle_box (box,"plc")
         # QEMU
         if self.do_tracker_qemus:self.handle_tracker_qemus ()
         for box in self.boxes:  self.handle_box (box,"qemu")
+        # ALL OTHERS
+        for box in self.boxes:  self.handle_box (box,"build")
+        # TESTMASTER
+        for box in self.boxes:  self.handle_box (box,"testmaster")
 
 if __name__ == "__main__":
     BuildBoxes().main()