X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=system%2FSubstrate.py;h=938757953d6d1a41139912e3bae46401aef326a6;hb=803c72244fe8e93805f83e660964b742ae7c6b26;hp=f57172d7bdec7899835ef3ecc8d450652798d84d;hpb=6504d863b51f856956a39c10ff935cce81c04418;p=tests.git

diff --git a/system/Substrate.py b/system/Substrate.py
index f57172d..9387579 100644
--- a/system/Substrate.py
+++ b/system/Substrate.py
@@ -4,6 +4,8 @@
 #
 # #################### history
 #
+# see also Substrate.readme
+#
 # This is a complete rewrite of TestResources/Tracker/Pool
 # we don't use trackers anymore and just probe/sense the running 
 # boxes to figure out where we are
@@ -63,11 +65,41 @@ def header (message,banner=True):
     print message
     sys.stdout.flush()
 
-def timestamp_sort(o1,o2): 
-    if not o1.timestamp:        return -1
-    elif not o2.timestamp:      return 1
-    else:                       return o2.timestamp-o1.timestamp
+def timestamp_sort(o1,o2): return o1.timestamp-o2.timestamp
+
+def short_hostname (hostname):
+    return hostname.split('.')[0]
 
+####################
+# the place were other test instances tell about their not-yet-started
+# instances, that go undetected through sensing
+class Starting:
+
+    location='/root/starting'
+    def __init__ (self):
+        self.tuples=[]
+
+    def load (self):
+        try:    self.tuples=[line.strip().split('@') 
+                             for line in file(Starting.location).readlines()]
+        except: self.tuples=[]
+
+    def vnames (self) : 
+        self.load()
+        return [ x for (x,_) in self.tuples ]
+
+    def add (self, vname, bname):
+        if not vname in self.vnames():
+            file(Starting.location,'a').write("%s@%s\n"%(vname,bname))
+            
+    def delete_vname (self, vname):
+        self.load()
+        if vname in self.vnames():
+            f=file(Starting.location,'w')
+            for (v,b) in self.tuples: 
+                if v != vname: f.write("%s@%s\n"%(v,b))
+            f.close()
+    
 ####################
 # pool class
 # allows to pick an available IP among a pool
@@ -96,6 +128,14 @@ class PoolItem:
 
     def line(self):
         return "Pooled %s (%s) -> %s"%(self.hostname,self.userdata, self.status)
+
+    def char (self):
+        if   self.status==None:       return '?'
+        elif self.status=='busy':     return '+'
+        elif self.status=='free':     return '-'
+        elif self.status=='mine':     return 'M'
+        elif self.status=='starting': return 'S'
+
     def get_ip(self):
         if self.ip: return self.ip
         ip=socket.gethostbyname(self.hostname)
@@ -104,46 +144,97 @@ class PoolItem:
 
 class Pool:
 
-    def __init__ (self, tuples,message):
-        self.pool= [ PoolItem (h,u) for (h,u) in tuples ] 
+    def __init__ (self, tuples,message, substrate):
+        self.pool_items= [ PoolItem (hostname,userdata) for (hostname,userdata) in tuples ] 
         self.message=message
-
-    def sense (self):
-        print 'Checking IP pool',self.message,
-        for item in self.pool:
-            if item.status is not None: 
-                continue
-            if self.check_ping (item.hostname): 
-                item.status='busy'
-            else:
-                item.status='free'
-        print 'Done'
+        # where to send notifications upon load_starting
+        self.substrate=substrate
 
     def list (self):
-        for i in self.pool: print i.line()
+        for i in self.pool_items: print i.line()
 
-    def retrieve_userdata (self, hostname):
-        for i in self.pool: 
-            if i.hostname==hostname: return i.userdata
-        return None
+    def line (self):
+        line=self.message
+        for i in self.pool_items: line += ' ' + i.char()
+        return line
+
+    def _item (self, hostname):
+        for i in self.pool_items: 
+            if i.hostname==hostname: return i
+        raise Exception ("Could not locate hostname %s in pool %s"%(hostname,self.message))
+
+    def retrieve_userdata (self, hostname): 
+        return self._item(hostname).userdata
 
     def get_ip (self, hostname):
-        # use cached if in pool
-        for i in self.pool: 
-            if i.hostname==hostname: return i.get_ip()
-        # otherwise just ask dns again
-        return socket.gethostbyname(hostname)
+        try:    return self._item(hostname).get_ip()
+        except: return socket.gethostbyname(hostname)
+        
+    def set_mine (self, hostname):
+        try:
+            self._item(hostname).status='mine'
+        except:
+            print 'WARNING: host %s not found in IP pool %s'%(hostname,self.message)
 
     def next_free (self):
-        for i in self.pool:
-            if i.status in ['busy','mine','starting' ]: continue
-            i.status='mine'
-            return (i.hostname,i.userdata)
-        raise Exception,"No IP address available in pool %s"%self.message
+        for i in self.pool_items:
+            if i.status == 'free':
+                i.status='mine'
+                return (i.hostname,i.userdata)
+        return None
 
-# OS-dependent ping option (support for macos, for convenience)
+    ####################
+    # we have a starting instance of our own
+    def add_starting (self, vname, bname):
+        Starting().add(vname,bname)
+        for i in self.pool_items:
+            if i.hostname==vname: i.status='mine'
+
+    # load the starting instances from the common file
+    # remember that might be ours
+    # return the list of (vname,bname) that are not ours
+    def load_starting (self):
+        starting=Starting()
+        starting.load()
+        new_tuples=[]
+        for (v,b) in starting.tuples:
+            for i in self.pool_items:
+                if i.hostname==v and i.status=='free':
+                    i.status='starting'
+                    new_tuples.append( (v,b,) )
+        return new_tuples
+
+    def release_my_starting (self):
+        for i in self.pool_items:
+            if i.status=='mine':
+                Starting().delete_vname (i.hostname)
+                i.status=None
+
+
+    ##########
+    def _sense (self):
+        for item in self.pool_items:
+            if item.status is not None: 
+                print item.char(),
+                continue
+            if self.check_ping (item.hostname): 
+                item.status='busy'
+                print '*',
+            else:
+                item.status='free'
+                print '.',
+    
+    def sense (self):
+        print 'Sensing IP pool',self.message,
+        self._sense()
+        print 'Done'
+        for (vname,bname) in self.load_starting():
+            self.substrate.add_starting_dummy (bname, vname)
+        print 'After starting: IP pool'
+        print self.line()
+    # OS-dependent ping option (support for macos, for convenience)
     ping_timeout_option = None
-# checks whether a given hostname/ip responds to ping
+    # returns True when a given hostname/ip responds to ping
     def check_ping (self,hostname):
         if not Pool.ping_timeout_option:
             (status,osname) = commands.getstatusoutput("uname -s")
@@ -156,52 +247,27 @@ class Pool:
 
         command="ping -c 1 %s 1 %s"%(Pool.ping_timeout_option,hostname)
         (status,output) = commands.getstatusoutput(command)
-        if status==0:   print '+',
-        else:           print '-',
         return status == 0
 
-    # the place were other test instances tell about their not-yet-started
-    # instances, that go undetected through sensing
-    starting='/root/starting'
-    def add_starting (self, name):
-        try:    items=[line.strip() for line in file(Pool.starting).readlines()]
-        except: items=[]
-        if not name in items:
-            file(Pool.starting,'a').write(name+'\n')
-        for i in self.pool:
-            if i.hostname==name: i.status='mine'
-            
-    def load_starting (self):
-        try:    items=[line.strip() for line in file(Pool.starting).readlines()]
-        except: items=[]
-        for item in items:
-            for i in self.pool:
-                if i.hostname==item: i.status='starting'
-
-    def release_my_fakes (self):
-        for i in self.pool:
-            if i.status=='mine': 
-                self.del_starting(i.hostname)
-                i.status=None
-
-    def del_starting (self, name):
-        try:    items=[line.strip() for line in file(Pool.starting).readlines()]
-        except: items=[]
-        if name in items:
-            f=file(Pool.starting,'w')
-            for item in items: 
-                if item != name: f.write(item+'\n')
-            f.close()
-    
 ####################
 class Box:
     def __init__ (self,hostname):
         self.hostname=hostname
-    def simple_hostname (self):
-        return self.hostname.split('.')[0]
+        self._probed=None
+    def shortname (self):
+        return short_hostname(self.hostname)
     def test_ssh (self): return TestSsh(self.hostname,username='root',unknown_host=False)
-    def reboot (self):
-        self.test_ssh().run("shutdown -r now",message="Rebooting %s"%self.hostname)
+    def reboot (self, options):
+        self.test_ssh().run("shutdown -r now",message="Rebooting %s"%self.hostname,
+                            dry_run=options.dry_run)
+
+    def uptime(self):
+        if hasattr(self,'_uptime') and self._uptime: return self._uptime
+        return '*undef* uptime'
+    def sense_uptime (self):
+        command=['uptime']
+        self._uptime=self.backquote_ssh(command,trash_err=True).strip()
+        if not self._uptime: self._uptime='unreachable'
 
     def run(self,argv,message=None,trash_err=False,dry_run=False):
         if dry_run:
@@ -215,28 +281,35 @@ class Box:
             else:
                 return subprocess.call(argv,stderr=file('/dev/null','w'))
                 
-    def run_ssh (self, argv, message, trash_err=False):
+    def run_ssh (self, argv, message, trash_err=False, dry_run=False):
         ssh_argv = self.test_ssh().actual_argv(argv)
-        result=self.run (ssh_argv, message, trash_err)
+        result=self.run (ssh_argv, message, trash_err, dry_run=dry_run)
         if result!=0:
             print "WARNING: failed to run %s on %s"%(" ".join(argv),self.hostname)
         return result
 
     def backquote (self, argv, trash_err=False):
+        # print 'running backquote',argv
         if not trash_err:
-            return subprocess.Popen(argv,stdout=subprocess.PIPE).communicate()[0]
+            result= subprocess.Popen(argv,stdout=subprocess.PIPE).communicate()[0]
         else:
-            return subprocess.Popen(argv,stdout=subprocess.PIPE,stderr=file('/dev/null','w')).communicate()[0]
+            result= subprocess.Popen(argv,stdout=subprocess.PIPE,stderr=file('/dev/null','w')).communicate()[0]
+        return result
 
-    def backquote_ssh (self, argv, trash_err=False):
+    def probe (self):
+        if self._probed is not None: return self._probed
         # first probe the ssh link
         probe_argv=self.test_ssh().actual_argv(['hostname'])
-        hostname=self.backquote ( probe_argv, trash_err=True )
-        if not hostname:
-            print "root@%s unreachable"%self.hostname
-            return ''
-        else:
-            return self.backquote( self.test_ssh().actual_argv(argv), trash_err)
+        self._probed=self.backquote ( probe_argv, trash_err=True )
+        if not self._probed: print "root@%s unreachable"%self.hostname
+        return self._probed
+
+    # use argv=['bash','-c',"the command line"]
+    # if you have any shell-expanded arguments like *
+    # and if there's any chance the command is adressed to the local host
+    def backquote_ssh (self, argv, trash_err=False):
+        if not self.probe(): return ''
+        return self.backquote( self.test_ssh().actual_argv(argv), trash_err)
 
 ############################################################
 class BuildInstance:
@@ -271,28 +344,29 @@ class BuildBox (Box):
             for b in self.build_instances: 
                 header (b.line(),banner=False)
 
-    def uptime(self):
-        if hasattr(self,'_uptime') and self._uptime: return self._uptime
-        return '*undef* uptime'
+    def reboot (self, options):
+        if not options.soft:
+            self.reboot(options)
+        else:
+            command=['pkill','vbuild']
+            self.run_ssh(command,"Terminating vbuild processes",dry_run=options.dry_run)
 
     # inspect box and find currently running builds
     matcher=re.compile("\s*(?P<pid>[0-9]+).*-[bo]\s+(?P<buildname>[^\s]+)(\s|\Z)")
-    def sense(self,reboot=False,verbose=True):
-        if reboot:
-            self.reboot(box)
-            return
+    def sense(self, options):
         print 'b',
-        command=['uptime']
-        self._uptime=self.backquote_ssh(command,trash_err=True).strip()
-        if not self._uptime: self._uptime='unreachable'
-        pids=self.backquote_ssh(['pgrep','build'],trash_err=True)
+        self.sense_uptime()
+        pids=self.backquote_ssh(['pgrep','vbuild'],trash_err=True)
         if not pids: return
         command=['ps','-o','pid,command'] + [ pid for pid in pids.split("\n") if pid]
         ps_lines=self.backquote_ssh (command).split('\n')
         for line in ps_lines:
             if not line.strip() or line.find('PID')>=0: continue
             m=BuildBox.matcher.match(line)
-            if m: self.add_build (m.group('buildname'),m.group('pid'))
+            if m: 
+                date=time.strftime('%Y-%m-%d',time.localtime(time.time()))
+                buildname=m.group('buildname').replace('@DATE@',date)
+                self.add_build (buildname,m.group('pid'))
             else: header('command %r returned line that failed to match'%command)
 
 ############################################################
@@ -302,17 +376,24 @@ class PlcInstance:
         self.ctxid=ctxid
         self.plc_box=plcbox
         # unknown yet
-        self.timestamp=None
+        self.timestamp=0
 
     def set_timestamp (self,timestamp): self.timestamp=timestamp
     def set_now (self): self.timestamp=int(time.time())
     def pretty_timestamp (self): return time.strftime("%Y-%m-%d:%H-%M",time.localtime(self.timestamp))
 
+    def vplcname (self):
+        return self.vservername.split('-')[-1]
+    def buildname (self):
+        return self.vservername.rsplit('-',2)[0]
+
     def line (self):
-        msg="== %s == (ctx=%s)"%(self.vservername,self.ctxid)
+        msg="== %s =="%(self.vplcname())
+        msg += " [=%s]"%self.vservername
+        if self.ctxid==0:  msg+=" not (yet?) running"
+        else:              msg+=" (ctx=%s)"%self.ctxid     
         if self.timestamp: msg += " @ %s"%self.pretty_timestamp()
         else:              msg += " *unknown timestamp*"
-        if self.ctxid==0: msg+=" not (yet?) running"
         return msg
 
     def kill (self):
@@ -352,6 +433,7 @@ class PlcBox (Box):
             header ('No vserver running on %s'%(self.line()))
         else:
             header ("Active plc VMs on %s"%self.line())
+            self.plc_instances.sort(timestamp_sort)
             for p in self.plc_instances: 
                 header (p.line(),banner=False)
 
@@ -367,17 +449,14 @@ class PlcBox (Box):
             if p.vservername==vservername: return p
         return None
 
-    def sense (self, reboot=False, soft=False):
-        if reboot:
-            # remove mark for all running servers to avoid resurrection
-            stop_command=['rm','-rf','/etc/vservers/*/apps/init/mark']
-            self.run_ssh(stop_command,"Removing all vserver marks on %s"%self.hostname)
-            if not soft:
-                self.reboot()
-                return
-            else:
-                self.run_ssh(['service','util-vserver','stop'],"Stopping all running vservers")
-            return
+    def reboot (self, options):
+        if not options.soft:
+            self.reboot(options)
+        else:
+            self.run_ssh(['service','util-vserver','stop'],"Stopping all running vservers",
+                         dry_run=options.dry_run)
+
+    def sense (self, options):
         print 'p',
         self._uname=self.backquote_ssh(['uname','-r']).strip()
         # try to find fullname (vserver_stat truncates to a ridiculously short name)
@@ -399,29 +478,32 @@ class PlcBox (Box):
             if not vserver_line: continue
             context=vserver_line.split()[0]
             if context=="CTX": continue
-            longname=ctx_dict[context]
-            self.add_vserver(longname,context)
-#            print self.margin_outline(self.vplcname(longname)),"%(vserver_line)s [=%(longname)s]"%locals()
+            try:
+                longname=ctx_dict[context]
+                self.add_vserver(longname,context)
+            except:
+                print 'WARNING: found ctx %s in vserver_stat but was unable to figure a corresp. vserver'%context
 
-        # scan timestamps
-        running_ctx_ids = [ i.ctxid for i in self.plc_instances ]
+        # scan timestamps 
+        running_vsnames = [ i.vservername for i in self.plc_instances ]
         command=   ['grep','.']
-        command += ['/vservers/%s/timestamp'%b for b in running_ctx_ids]
+        command += ['/vservers/%s.timestamp'%vs for vs in running_vsnames]
         command += ['/dev/null']
         ts_lines=self.backquote_ssh(command,trash_err=True).split('\n')
         for ts_line in ts_lines:
             if not ts_line.strip(): continue
-            # expect /vservers/<vservername>/timestamp:<timestamp>
+            # expect /vservers/<vservername>.timestamp:<timestamp>
             try:
-                (_,__,vservername,tail)=ts_line.split('/')
-                (_,timestamp)=tail.split(':')
+                (ts_file,timestamp)=ts_line.split(':')
+                ts_file=os.path.basename(ts_file)
+                (vservername,_)=os.path.splitext(ts_file)
                 timestamp=int(timestamp)
-                q=self.plc_instance_by_vservername(vservername)
-                if not q: 
-                    print 'WARNING unattached plc instance',ts_line
-                    print 'was expeting to find',vservername,'in',[i.vservername for i in self.plc_instances]
+                p=self.plc_instance_by_vservername(vservername)
+                if not p: 
+                    print 'WARNING zombie plc',self.hostname,ts_line
+                    print '... was expecting',vservername,'in',[i.vservername for i in self.plc_instances]
                     continue
-                q.set_timestamp(timestamp)
+                p.set_timestamp(timestamp)
             except:  print 'WARNING, could not parse ts line',ts_line
         
 
@@ -435,7 +517,7 @@ class QemuInstance:
         self.qemu_box=qemubox
         # not known yet
         self.buildname=None
-        self.timestamp=None
+        self.timestamp=0
         
     def set_buildname (self,buildname): self.buildname=buildname
     def set_timestamp (self,timestamp): self.timestamp=timestamp
@@ -443,17 +525,18 @@ class QemuInstance:
     def pretty_timestamp (self): return time.strftime("%Y-%m-%d:%H-%M",time.localtime(self.timestamp))
     
     def line (self):
-        msg = "== %s == (pid=%s)"%(self.nodename,self.pid)
-        if self.buildname: msg += " <--> %s"%self.buildname
-        else:              msg += " *unknown build*"
+        msg = "== %s =="%(short_hostname(self.nodename))
+        msg += " [=%s]"%self.buildname
+        if self.pid:       msg += " (pid=%s)"%self.pid
+        else:              msg += " not (yet?) running"
         if self.timestamp: msg += " @ %s"%self.pretty_timestamp()
         else:              msg += " *unknown timestamp*"
-        if self.pid:       msg += "pid=%s"%self.pid
-        else:              msg += " not (yet?) running"
         return msg
     
     def kill(self):
-        if self.pid==0: print "cannot kill qemu %s with pid==0"%self.nodename
+        if self.pid==0: 
+            print "cannot kill qemu %s with pid==0"%self.nodename
+            return
         msg="Killing qemu %s with pid=%s on box %s"%(self.nodename,self.pid,self.qemu_box.hostname)
         self.qemu_box.run_ssh(['kill',"%s"%self.pid],msg)
         self.qemu_box.forget(self)
@@ -491,6 +574,7 @@ class QemuBox (Box):
             header ('No qemu process on %s'%(self.line()))
         else:
             header ("Active qemu processes on %s"%(self.line()))
+            self.qemu_instances.sort(timestamp_sort)
             for q in self.qemu_instances: 
                 header (q.line(),banner=False)
 
@@ -512,14 +596,15 @@ class QemuBox (Box):
                 return q
         return None
 
+    def reboot (self, options):
+        if not options.soft:
+            self.reboot(options)
+        else:
+            self.run_ssh(['pkill','qemu'],"Killing qemu instances",
+                         dry_run=options.dry_run)
+
     matcher=re.compile("\s*(?P<pid>[0-9]+).*-cdrom\s+(?P<nodename>[^\s]+)\.iso")
-    def sense(self, reboot=False, soft=False):
-        if reboot:
-            if not soft:
-                self.reboot()
-            else:
-                self.run_ssh(box,['pkill','qemu'],"Killing qemu instances")
-            return
+    def sense(self, options):
         print 'q',
         modules=self.backquote_ssh(['lsmod']).split('\n')
         self._driver='*NO kqemu/kmv_intel MODULE LOADED*'
@@ -555,6 +640,7 @@ class QemuBox (Box):
                 live_builds.append(buildname)
             except: print 'WARNING, could not parse pid line',pid_line
         # retrieve timestamps
+        if not live_builds: return
         command=   ['grep','.']
         command += ['%s/*/timestamp'%b for b in live_builds]
         command += ['/dev/null']
@@ -569,113 +655,310 @@ class QemuBox (Box):
                 timestamp=int(timestamp)
                 q=self.qemu_instance_by_nodename_buildname(nodename,buildname)
                 if not q: 
-                    print 'WARNING unattached qemu instance',ts_line,nodename,buildname
+                    print 'WARNING zombie qemu',self.hostname,ts_line
+                    print '... was expecting (',short_hostname(nodename),buildname,') in',\
+                        [ (short_hostname(i.nodename),i.buildname) for i in self.qemu_instances ]
                     continue
                 q.set_timestamp(timestamp)
             except:  print 'WARNING, could not parse ts line',ts_line
 
+####################
+class TestInstance:
+    def __init__ (self, buildname, pid=0):
+        self.pids=[]
+        if pid!=0: self.pid.append(pid)
+        self.buildname=buildname
+        # latest trace line
+        self.trace=''
+        # has a KO test
+        self.broken_steps=[]
+        self.timestamp = 0
+
+    def set_timestamp (self,timestamp): self.timestamp=timestamp
+    def set_now (self): self.timestamp=int(time.time())
+    def pretty_timestamp (self): return time.strftime("%Y-%m-%d:%H-%M",time.localtime(self.timestamp))
+
+
+    def add_pid (self,pid):
+        self.pids.append(pid)
+    def set_broken (self,plcindex, step): 
+        self.broken_steps.append ( (plcindex, step,) )
+
+    def line (self):
+        double='=='
+        if self.pids: double='*'+double[1]
+        if self.broken_steps: double=double[0]+'B'
+        msg = " %s %s =="%(double,self.buildname)
+        if not self.pids:       pass
+        elif len(self.pids)==1: msg += " (pid=%s)"%self.pids[0]
+        else:                   msg += " !!!pids=%s!!!"%self.pids
+        msg += " @%s"%self.pretty_timestamp()
+        if self.broken_steps:
+            msg += "\n BROKEN IN STEPS"
+            for (i,s) in self.broken_steps: msg += " %s@%s"%(s,i)
+        return msg
+
+class TestBox (Box):
+    def __init__ (self,hostname):
+        Box.__init__(self,hostname)
+        self.starting_ips=[]
+        self.test_instances=[]
+
+    def reboot (self, options):
+        # can't reboot a vserver VM
+        self.run_ssh (['pkill','run_log'],"Terminating current runs",
+                      dry_run=options.dry_run)
+        self.run_ssh (['rm','-f',Starting.location],"Cleaning %s"%Starting.location,
+                      dry_run=options.dry_run)
+
+    def get_test (self, buildname):
+        for i in self.test_instances:
+            if i.buildname==buildname: return i
+
+    # we scan ALL remaining test results, even the ones not running
+    def add_timestamp (self, buildname, timestamp):
+        i=self.get_test(buildname)
+        if i:   
+            i.set_timestamp(timestamp)
+        else:   
+            i=TestInstance(buildname,0)
+            i.set_timestamp(timestamp)
+            self.test_instances.append(i)
+
+    def add_running_test (self, pid, buildname):
+        i=self.get_test(buildname)
+        if not i:
+            self.test_instances.append (TestInstance (buildname,pid))
+            return
+        if i.pids:
+            print "WARNING: 2 concurrent tests run on same build %s"%buildname
+        i.add_pid (pid)
+
+    def add_broken (self, buildname, plcindex, step):
+        i=self.get_test(buildname)
+        if not i:
+            i=TestInstance(buildname)
+            self.test_instances.append(i)
+        i.set_broken(plcindex, step)
+
+    matcher_proc=re.compile (".*/proc/(?P<pid>[0-9]+)/cwd.*/root/(?P<buildname>[^/]+)$")
+    matcher_grep=re.compile ("/root/(?P<buildname>[^/]+)/logs/trace.*:TRACE:\s*(?P<plcindex>[0-9]+).*step=(?P<step>\S+).*")
+    def sense (self, options):
+        print 't',
+        self.sense_uptime()
+        self.starting_ips=[x for x in self.backquote_ssh(['cat',Starting.location], trash_err=True).strip().split('\n') if x]
+
+        # scan timestamps on all tests
+        # this is likely to not invoke ssh so we need to be a bit smarter to get * expanded
+        # xxx would make sense above too
+        command=['bash','-c',"grep . /root/*/timestamp /dev/null"]
+        ts_lines=self.backquote_ssh(command,trash_err=True).split('\n')
+        for ts_line in ts_lines:
+            if not ts_line.strip(): continue
+            # expect /root/<buildname>/timestamp:<timestamp>
+            try:
+                (ts_file,timestamp)=ts_line.split(':')
+                ts_file=os.path.dirname(ts_file)
+                buildname=os.path.basename(ts_file)
+                timestamp=int(timestamp)
+                t=self.add_timestamp(buildname,timestamp)
+            except:  print 'WARNING, could not parse ts line',ts_line
+
+        command=['bash','-c',"grep KO /root/*/logs/trace* /dev/null" ]
+        trace_lines=self.backquote_ssh (command).split('\n')
+        for line in trace_lines:
+            if not line.strip(): continue
+            m=TestBox.matcher_grep.match(line)
+            if m: 
+                buildname=m.group('buildname')
+                plcindex=m.group('plcindex')
+                step=m.group('step')
+                self.add_broken(buildname,plcindex, step)
+            else: header("command %r returned line that failed to match\n%s"%(command,line))
+
+        pids = self.backquote_ssh (['pgrep','run_log'],trash_err=True)
+        if not pids: return
+        command=['ls','-ld'] + ["/proc/%s/cwd"%pid for pid in pids.split("\n") if pid]
+        ps_lines=self.backquote_ssh (command).split('\n')
+        for line in ps_lines:
+            if not line.strip(): continue
+            m=TestBox.matcher_proc.match(line)
+            if m: 
+                pid=m.group('pid')
+                buildname=m.group('buildname')
+                self.add_running_test(pid, buildname)
+            else: header("command %r returned line that failed to match\n%s"%(command,line))
+        
+        
+    def line (self):
+        return "%s (%s)"%(self.hostname,self.uptime())
+
+    def list (self):
+        if not self.test_instances:
+            header ("No known tests on %s"%self.line())
+        else:
+            header ("Known tests on %s"%self.line())
+            self.test_instances.sort(timestamp_sort)
+            for i in self.test_instances: print i.line()
+        if self.starting_ips:
+            header ("Starting IP addresses on %s"%self.line())
+            self.starting_ips.sort()
+            for starting in self.starting_ips: print starting
+
 ############################################################
 class Options: pass
 
 class Substrate:
 
-    def test (self): 
-        self.sense()
-
     def __init__ (self):
         self.options=Options()
         self.options.dry_run=False
         self.options.verbose=False
-        self.options.probe=True
-        self.options.soft=True
+        self.options.reboot=False
+        self.options.soft=False
+        self.test_box = TestBox (self.test_box_spec())
         self.build_boxes = [ BuildBox(h) for h in self.build_boxes_spec() ]
         self.plc_boxes = [ PlcBox (h,m) for (h,m) in self.plc_boxes_spec ()]
         self.qemu_boxes = [ QemuBox (h,m) for (h,m) in self.qemu_boxes_spec ()]
-        self.all_boxes = self.build_boxes + self.plc_boxes + self.qemu_boxes
+        self.default_boxes = self.plc_boxes + self.qemu_boxes
+        self.all_boxes = self.build_boxes + [ self.test_box ] + self.plc_boxes + self.qemu_boxes
         self._sensed=False
 
-        self.vplc_pool = Pool (self.vplc_ips(),"for vplcs")
-        self.vnode_pool = Pool (self.vnode_ips(),"for vnodes")
+        self.vplc_pool = Pool (self.vplc_ips(),"for vplcs",self)
+        self.vnode_pool = Pool (self.vnode_ips(),"for vnodes",self)
 
-#    def build_box_names (self):
-#        return [ h for h in self.build_boxes_spec() ]
-#    def plc_boxes (self):
-#        return [ h for (h,m) in self.plc_boxes_spec() ]
-#    def qemu_boxes (self):
-#        return [ h for (h,m) in self.qemu_boxes_spec() ]
+    def fqdn (self, hostname):
+        if hostname.find('.')<0: return "%s.%s"%(hostname,self.domain())
+        return hostname
 
+    # return True if actual sensing takes place
     def sense (self,force=False):
-        if self._sensed and not force: return
+        if self._sensed and not force: return False
         print 'Sensing local substrate...',
-        for b in self.all_boxes: b.sense()
+        for b in self.default_boxes: b.sense(self.options)
         print 'Done'
         self._sensed=True
+        return True
+
+    def list (self):
+        for b in self.default_boxes:
+            b.list()
+
+    def add_dummy_plc (self, plc_boxname, plcname):
+        for pb in self.plc_boxes:
+            if pb.hostname==plc_boxname:
+                pb.add_dummy(plcname)
+                return True
+    def add_dummy_qemu (self, qemu_boxname, qemuname):
+        for qb in self.qemu_boxes:
+            if qb.hostname==qemu_boxname:
+                qb.add_dummy(qemuname)
+                return True
+
+    def add_starting_dummy (self, bname, vname):
+        return self.add_dummy_plc (bname, vname) or self.add_dummy_qemu (bname, vname)
 
     ########## 
     def provision (self,plcs,options):
         try:
-            self.sense()
-            self.list_all()
             # attach each plc to a plc box and an IP address
             plcs = [ self.provision_plc (plc,options) for plc in plcs ]
             # attach each node/qemu to a qemu box with an IP address
             plcs = [ self.provision_qemus (plc,options) for plc in plcs ]
             # update the SFA spec accordingly
             plcs = [ self.localize_sfa_rspec(plc,options) for plc in plcs ]
+            self.list()
             return plcs
         except Exception, e:
             print '* Could not provision this test on current substrate','--',e,'--','exiting'
             traceback.print_exc()
             sys.exit(1)
 
+    # it is expected that a couple of options like ips_bplc and ips_vplc 
+    # are set or unset together
+    @staticmethod
+    def check_options (x,y):
+        if not x and not y: return True
+        return len(x)==len(y)
+
     # find an available plc box (or make space)
     # and a free IP address (using options if present)
     def provision_plc (self, plc, options):
-        #### we need to find one plc box that still has a slot
-        plc_box=None
-        max_free=0
-        # use the box that has max free spots for load balancing
-        for pb in self.plc_boxes:
-            free=pb.free_spots()
-            if free>max_free:
-                plc_box=pb
-                max_free=free
-        # everything is already used
-        if not plc_box:
-            # find the oldest of all our instances
-            all_plc_instances=reduce(lambda x, y: x+y, 
-                                     [ pb.plc_instances for pb in self.plc_boxes ],
-                                     [])
-            all_plc_instances.sort(timestamp_sort)
-            plc_instance_to_kill=all_plc_instances[0]
-            plc_box=plc_instance_to_kill.plc_box
-            plc_instance_to_kill.kill()
-            print 'killed oldest = %s on %s'%(plc_instance_to_kill.line(),
-                                             plc_instance_to_kill.plc_box.hostname)
-
-        utils.header( 'plc %s -> box %s'%(plc['name'],plc_box.line()))
-        plc_box.add_dummy(plc['name'])
-        #### OK we have a box to run in, let's find an IP address
-        # look in options
+        
+        assert Substrate.check_options (options.ips_bplc, options.ips_vplc)
+
+        #### let's find an IP address for that plc
+        # look in options 
         if options.ips_vplc:
+            # this is a rerun
+            # we don't check anything here, 
+            # it is the caller's responsability to cleanup and make sure this makes sense
+            plc_boxname = options.ips_bplc.pop()
             vplc_hostname=options.ips_vplc.pop()
         else:
-            self.vplc_pool.load_starting()
+            if self.sense(): self.list()
+            plc_boxname=None
+            vplc_hostname=None
+            # try to find an available IP 
             self.vplc_pool.sense()
-            (vplc_hostname,unused)=self.vplc_pool.next_free()
+            couple=self.vplc_pool.next_free()
+            if couple:
+                (vplc_hostname,unused)=couple
+            #### we need to find one plc box that still has a slot
+            max_free=0
+            # use the box that has max free spots for load balancing
+            for pb in self.plc_boxes:
+                free=pb.free_spots()
+                if free>max_free:
+                    plc_boxname=pb.hostname
+                    max_free=free
+            # if there's no available slot in the plc_boxes, or we need a free IP address
+            # make space by killing the oldest running instance
+            if not plc_boxname or not vplc_hostname:
+                # find the oldest of all our instances
+                all_plc_instances=reduce(lambda x, y: x+y, 
+                                         [ pb.plc_instances for pb in self.plc_boxes ],
+                                         [])
+                all_plc_instances.sort(timestamp_sort)
+                try:
+                    plc_instance_to_kill=all_plc_instances[0]
+                except:
+                    msg=""
+                    if not plc_boxname: msg += " PLC boxes are full"
+                    if not vplc_hostname: msg += " vplc IP pool exhausted" 
+                    raise Exception,"Could not make space for a PLC instance:"+msg
+                freed_plc_boxname=plc_instance_to_kill.plc_box.hostname
+                freed_vplc_hostname=plc_instance_to_kill.vplcname()
+                message='killing oldest plc instance = %s on %s'%(plc_instance_to_kill.line(),
+                                                                  freed_plc_boxname)
+                plc_instance_to_kill.kill()
+                # use this new plcbox if that was the problem
+                if not plc_boxname:
+                    plc_boxname=freed_plc_boxname
+                # ditto for the IP address
+                if not vplc_hostname:
+                    vplc_hostname=freed_vplc_hostname
+                    # record in pool as mine
+                    self.vplc_pool.set_mine(vplc_hostname)
+
+        # 
+        self.add_dummy_plc(plc_boxname,plc['name'])
         vplc_ip = self.vplc_pool.get_ip(vplc_hostname)
-        self.vplc_pool.add_starting(vplc_hostname)
+        self.vplc_pool.add_starting(vplc_hostname, plc_boxname)
 
         #### compute a helpful vserver name
         # remove domain in hostname
-        vplc_simple = vplc_hostname.split('.')[0]
-        vservername = "%s-%d-%s" % (options.buildname,plc['index'],vplc_simple)
-        plc_name = "%s_%s"%(plc['name'],vplc_simple)
+        vplc_short = short_hostname(vplc_hostname)
+        vservername = "%s-%d-%s" % (options.buildname,plc['index'],vplc_short)
+        plc_name = "%s_%s"%(plc['name'],vplc_short)
+
+        utils.header( 'PROVISION plc %s in box %s at IP %s as %s'%\
+                          (plc['name'],plc_boxname,vplc_hostname,vservername))
 
         #### apply in the plc_spec
         # # informative
         # label=options.personality.replace("linux","")
-        mapper = {'plc': [ ('*' , {'hostname':plc_box.hostname,
+        mapper = {'plc': [ ('*' , {'host_box':plc_boxname,
                                    # 'name':'%s-'+label,
                                    'name': plc_name,
                                    'vservername':vservername,
@@ -689,63 +972,84 @@ class Substrate:
                                    } ) ]
                   }
 
-        utils.header("Attaching %s on IP %s in vserver %s"%(plc['name'],vplc_hostname,vservername))
+
         # mappers only work on a list of plcs
         return TestMapper([plc],options).map(mapper)[0]
 
     ##########
     def provision_qemus (self, plc, options):
+
+        assert Substrate.check_options (options.ips_bnode, options.ips_vnode)
+
         test_mapper = TestMapper ([plc], options)
         nodenames = test_mapper.node_names()
         maps=[]
         for nodename in nodenames:
-            #### similarly we want to find a qemu box that can host us
-            qemu_box=None
-            max_free=0
-            # use the box that has max free spots for load balancing
-            for qb in self.qemu_boxes:
-                free=qb.free_spots()
-            if free>max_free:
-                qemu_box=qb
-                max_free=free
-            # everything is already used
-            if not qemu_box:
-                # find the oldest of all our instances
-                all_qemu_instances=reduce(lambda x, y: x+y, 
-                                         [ qb.qemu_instances for qb in self.qemu_boxes ],
-                                         [])
-                all_qemu_instances.sort(timestamp_sort)
-                qemu_instance_to_kill=all_qemu_instances[0]
-                qemu_box=qemu_instance_to_kill.qemu_box
-                qemu_instance_to_kill.kill()
-                print 'killed oldest = %s on %s'%(qemu_instance_to_kill.line(),
-                                                 qemu_instance_to_kill.qemu_box.hostname)
-
-            utils.header( 'node %s -> qemu box %s'%(nodename,qemu_box.line()))
-            qemu_box.add_dummy(nodename)
-            #### OK we have a box to run in, let's find an IP address
-            # look in options
+
             if options.ips_vnode:
+                # as above, it's a rerun, take it for granted
+                qemu_boxname=options.ips_bnode.pop()
                 vnode_hostname=options.ips_vnode.pop()
-                mac=self.vnode_pool.retrieve_userdata(vnode_hostname)
             else:
-                self.vnode_pool.load_starting()
+                if self.sense(): self.list()
+                qemu_boxname=None
+                vnode_hostname=None
+                # try to find an available IP 
                 self.vnode_pool.sense()
-                (vnode_hostname,mac)=self.vnode_pool.next_free()
+                couple=self.vnode_pool.next_free()
+                if couple:
+                    (vnode_hostname,unused)=couple
+                # find a physical box
+                max_free=0
+                # use the box that has max free spots for load balancing
+                for qb in self.qemu_boxes:
+                    free=qb.free_spots()
+                    if free>max_free:
+                        qemu_boxname=qb.hostname
+                        max_free=free
+                # if we miss the box or the IP, kill the oldest instance
+                if not qemu_boxname or not vnode_hostname:
+                # find the oldest of all our instances
+                    all_qemu_instances=reduce(lambda x, y: x+y, 
+                                              [ qb.qemu_instances for qb in self.qemu_boxes ],
+                                              [])
+                    all_qemu_instances.sort(timestamp_sort)
+                    try:
+                        qemu_instance_to_kill=all_qemu_instances[0]
+                    except:
+                        msg=""
+                        if not qemu_boxname: msg += " QEMU boxes are full"
+                        if not vnode_hostname: msg += " vnode IP pool exhausted" 
+                        raise Exception,"Could not make space for a QEMU instance:"+msg
+                    freed_qemu_boxname=qemu_instance_to_kill.qemu_box.hostname
+                    freed_vnode_hostname=short_hostname(qemu_instance_to_kill.nodename)
+                    # kill it
+                    message='killing oldest qemu node = %s on %s'%(qemu_instance_to_kill.line(),
+                                                                   freed_qemu_boxname)
+                    qemu_instance_to_kill.kill()
+                    # use these freed resources where needed
+                    if not qemu_boxname:
+                        qemu_boxname=freed_qemu_boxname
+                    if not vnode_hostname:
+                        vnode_hostname=freed_vnode_hostname
+                        self.vnode_pool.set_mine(vnode_hostname)
+
+            self.add_dummy_qemu (qemu_boxname,vnode_hostname)
+            mac=self.vnode_pool.retrieve_userdata(vnode_hostname)
             ip=self.vnode_pool.get_ip (vnode_hostname)
-            self.vnode_pool.add_starting(vnode_hostname)
+            self.vnode_pool.add_starting(vnode_hostname,qemu_boxname)
 
-            if vnode_hostname.find('.')<0:
-                vnode_hostname += "."+self.domain()
-            nodemap={'host_box':qemu_box.hostname,
-                     'node_fields:hostname':vnode_hostname,
+            vnode_fqdn = self.fqdn(vnode_hostname)
+            nodemap={'host_box':qemu_boxname,
+                     'node_fields:hostname':vnode_fqdn,
                      'interface_fields:ip':ip, 
                      'interface_fields:mac':mac,
                      }
             nodemap.update(self.network_settings())
             maps.append ( (nodename, nodemap) )
 
-            utils.header("Attaching %s on IP %s MAC %s"%(plc['name'],vnode_hostname,mac))
+            utils.header("PROVISION node %s in box %s at IP %s with MAC %s"%\
+                             (nodename,qemu_boxname,vnode_hostname,mac))
 
         return test_mapper.map({'node':maps})[0]
 
@@ -756,41 +1060,74 @@ class Substrate:
         plc['sfa']['SFA_SM_HOST'] = plc['PLC_DB_HOST']
         plc['sfa']['SFA_PLC_DB_HOST'] = plc['PLC_DB_HOST']
         plc['sfa']['SFA_PLC_URL'] = 'https://' + plc['PLC_API_HOST'] + ':443/PLCAPI/' 
-        for site in plc['sites']:
-            for node in site['nodes']:
-                plc['sfa']['sfa_slice_rspec']['part4'] = node['node_fields']['hostname']
 	return plc
 
     #################### release:
     def release (self,options):
-        self.vplc_pool.release_my_fakes()
-        self.vnode_pool.release_my_fakes()
+        self.vplc_pool.release_my_starting()
+        self.vnode_pool.release_my_starting()
         pass
 
     #################### show results for interactive mode
-    def list_all (self):
-        self.sense()
-        for b in self.all_boxes: b.list()
-
-    def get_box (self,box):
-        for b in self.build_boxes + self.plc_boxes + self.qemu_boxes:
-            if b.simple_hostname()==box:
+    def get_box (self,boxname):
+        for b in self.build_boxes + self.plc_boxes + self.qemu_boxes + [self.test_box] :
+            if b.shortname()==boxname:
                 return b
-        print "Could not find box %s"%box
+        print "Could not find box %s"%boxname
         return None
 
-    def list_box(self,box):
-        b=self.get_box(box)
-        if not b: return
-        b.sense()
-        b.list()
-
+    def list_boxes(self,box_or_names):
+        print 'Sensing',
+        for box in box_or_names:
+            if not isinstance(box,Box): box=self.get_box(box)
+            if not box: continue
+            box.sense(self.options)
+        print 'Done'
+        for box in box_or_names:
+            if not isinstance(box,Box): box=self.get_box(box)
+            if not box: continue
+            box.list()
+
+    def reboot_boxes(self,box_or_names):
+        for box in box_or_names:
+            if not isinstance(box,Box): box=self.get_box(box)
+            if not box: continue
+            box.reboot(self.options)
+
+    ####################
     # can be run as a utility to manage the local infrastructure
     def main (self):
         parser=OptionParser()
-        (options,args)=parser.parse_args()
-        if not args:
-            self.list_all()
-        else:
-            for box in args:
-                self.list_box(box)
+        parser.add_option ('-r',"--reboot",action='store_true',dest='reboot',default=False,
+                           help='reboot mode (use shutdown -r)')
+        parser.add_option ('-s',"--soft",action='store_true',dest='soft',default=False,
+                           help='soft mode for reboot (vserver stop or kill qemus)')
+        parser.add_option ('-t',"--testbox",action='store_true',dest='testbox',default=False,
+                           help='add test box') 
+        parser.add_option ('-b',"--build",action='store_true',dest='builds',default=False,
+                           help='add build boxes')
+        parser.add_option ('-p',"--plc",action='store_true',dest='plcs',default=False,
+                           help='add plc boxes')
+        parser.add_option ('-q',"--qemu",action='store_true',dest='qemus',default=False,
+                           help='add qemu boxes') 
+        parser.add_option ('-a',"--all",action='store_true',dest='all',default=False,
+                           help='address all known  boxes, like -b -t -p -q')
+        parser.add_option ('-v',"--verbose",action='store_true',dest='verbose',default=False,
+                           help='verbose mode')
+        parser.add_option ('-n',"--dry_run",action='store_true',dest='dry_run',default=False,
+                           help='dry run mode')
+        (self.options,args)=parser.parse_args()
+
+        boxes=args
+        if self.options.testbox: boxes += [self.test_box]
+        if self.options.builds: boxes += self.build_boxes
+        if self.options.plcs: boxes += self.plc_boxes
+        if self.options.qemus: boxes += self.qemu_boxes
+        if self.options.all: boxes += self.all_boxes
+        
+        # default scope is -b -p -q
+        if not boxes:
+            boxes = self.build_boxes + self.plc_boxes + self.qemu_boxes
+
+        if self.options.reboot: self.reboot_boxes (boxes)
+        else:                   self.list_boxes (boxes)