X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=system%2FSubstrate.py;h=1d42cb7d3154f6b8e4592c6be4c9bcf4fee9232c;hb=0ecbbc54d1bef5ddc4b1bc0523fe1ed1b4b4bbf0;hp=8da463cc552c56af812fa4e43c97d9951fcd3203;hpb=bac889bca8102aec62ddf28fc75c3eafb4a40dc0;p=tests.git

diff --git a/system/Substrate.py b/system/Substrate.py
index 8da463c..1d42cb7 100644
--- a/system/Substrate.py
+++ b/system/Substrate.py
@@ -4,6 +4,8 @@
 #
 # #################### history
 #
+# see also Substrate.readme
+#
 # This is a complete rewrite of TestResources/Tracker/Pool
 # we don't use trackers anymore and just probe/sense the running 
 # boxes to figure out where we are
@@ -65,6 +67,39 @@ def header (message,banner=True):
 
 def timestamp_sort(o1,o2): return o1.timestamp-o2.timestamp
 
+def short_hostname (hostname):
+    return hostname.split('.')[0]
+
+####################
+# the place were other test instances tell about their not-yet-started
+# instances, that go undetected through sensing
+class Starting:
+
+    location='/root/starting'
+    def __init__ (self):
+        self.tuples=[]
+
+    def load (self):
+        try:    self.tuples=[line.strip().split('@') 
+                             for line in file(Starting.location).readlines()]
+        except: self.tuples=[]
+
+    def vnames (self) : 
+        self.load()
+        return [ x for (x,_) in self.tuples ]
+
+    def add (self, vname, bname):
+        if not vname in self.vnames():
+            file(Starting.location,'a').write("%s@%s\n"%(vname,bname))
+            
+    def delete_vname (self, vname):
+        self.load()
+        if vname in self.vnames():
+            f=file(Starting.location,'w')
+            for (v,b) in self.tuples: 
+                if v != vname: f.write("%s@%s\n"%(v,b))
+            f.close()
+    
 ####################
 # pool class
 # allows to pick an available IP among a pool
@@ -96,8 +131,8 @@ class PoolItem:
 
     def char (self):
         if   self.status==None:       return '?'
-        elif self.status=='busy':     return '*'
-        elif self.status=='free':     return '.'
+        elif self.status=='busy':     return '+'
+        elif self.status=='free':     return '-'
         elif self.status=='mine':     return 'M'
         elif self.status=='starting': return 'S'
 
@@ -109,20 +144,22 @@ class PoolItem:
 
 class Pool:
 
-    def __init__ (self, tuples,message):
-        self.pool= [ PoolItem (h,u) for (h,u) in tuples ] 
+    def __init__ (self, tuples,message, substrate):
+        self.pool_items= [ PoolItem (hostname,userdata) for (hostname,userdata) in tuples ] 
         self.message=message
+        # where to send notifications upon load_starting
+        self.substrate=substrate
 
     def list (self):
-        for i in self.pool: print i.line()
+        for i in self.pool_items: print i.line()
 
     def line (self):
         line=self.message
-        for i in self.pool: line += ' ' + i.char()
+        for i in self.pool_items: line += ' ' + i.char()
         return line
 
     def _item (self, hostname):
-        for i in self.pool: 
+        for i in self.pool_items: 
             if i.hostname==hostname: return i
         raise Exception ("Could not locate hostname %s in pool %s"%(hostname,self.message))
 
@@ -140,64 +177,61 @@ class Pool:
             print 'WARNING: host %s not found in IP pool %s'%(hostname,self.message)
 
     def next_free (self):
-        for i in self.pool:
+        for i in self.pool_items:
             if i.status == 'free':
                 i.status='mine'
                 return (i.hostname,i.userdata)
         return None
 
-    # the place were other test instances tell about their not-yet-started
-    # instances, that go undetected through sensing
-    starting='/root/starting'
-    def add_starting (self, name):
-        try:    items=[line.strip() for line in file(Pool.starting).readlines()]
-        except: items=[]
-        if not name in items:
-            file(Pool.starting,'a').write(name+'\n')
-        for i in self.pool:
-            if i.hostname==name: i.status='mine'
-            
-    # we load this after actual sensing; 
+    ####################
+    # we have a starting instance of our own
+    def add_starting (self, vname, bname):
+        Starting().add(vname,bname)
+        for i in self.pool_items:
+            if i.hostname==vname: i.status='mine'
+
+    # load the starting instances from the common file
+    # remember that might be ours
+    # return the list of (vname,bname) that are not ours
     def load_starting (self):
-        try:    items=[line.strip() for line in file(Pool.starting).readlines()]
-        except: items=[]
-        for i in self.pool:
-            if i.hostname in items:
-                if i.status=='free' : i.status='starting'
+        starting=Starting()
+        starting.load()
+        new_tuples=[]
+        for (v,b) in starting.tuples:
+            for i in self.pool_items:
+                if i.hostname==v and i.status=='free':
+                    i.status='starting'
+                    new_tuples.append( (v,b,) )
+        return new_tuples
 
     def release_my_starting (self):
-        for i in self.pool:
-            if i.status=='mine': 
-                self.del_starting(i.hostname)
+        for i in self.pool_items:
+            if i.status=='mine':
+                Starting().delete_vname (i.hostname)
                 i.status=None
 
-    def del_starting (self, name):
-        try:    items=[line.strip() for line in file(Pool.starting).readlines()]
-        except: items=[]
-        if name in items:
-            f=file(Pool.starting,'w')
-            for item in items: 
-                if item != name: f.write(item+'\n')
-            f.close()
-    
+
     ##########
     def _sense (self):
-        for item in self.pool:
+        for item in self.pool_items:
             if item.status is not None: 
+                print item.char(),
                 continue
             if self.check_ping (item.hostname): 
                 item.status='busy'
+                print '*',
             else:
                 item.status='free'
+                print '.',
     
     def sense (self):
         print 'Sensing IP pool',self.message,
         self._sense()
         print 'Done'
-        self.load_starting()
+        for (vname,bname) in self.load_starting():
+            self.substrate.add_starting_dummy (bname, vname)
         print 'After starting: IP pool'
         print self.line()
-
     # OS-dependent ping option (support for macos, for convenience)
     ping_timeout_option = None
     # returns True when a given hostname/ip responds to ping
@@ -213,19 +247,27 @@ class Pool:
 
         command="ping -c 1 %s 1 %s"%(Pool.ping_timeout_option,hostname)
         (status,output) = commands.getstatusoutput(command)
-        if status==0:   print '*',
-        else:           print '.',
         return status == 0
 
 ####################
 class Box:
     def __init__ (self,hostname):
         self.hostname=hostname
-    def short_hostname (self):
-        return self.hostname.split('.')[0]
+        self._probed=None
+    def shortname (self):
+        return short_hostname(self.hostname)
     def test_ssh (self): return TestSsh(self.hostname,username='root',unknown_host=False)
-    def reboot (self):
-        self.test_ssh().run("shutdown -r now",message="Rebooting %s"%self.hostname)
+    def reboot (self, options):
+        self.test_ssh().run("shutdown -r now",message="Rebooting %s"%self.hostname,
+                            dry_run=options.dry_run)
+
+    def uptime(self):
+        if hasattr(self,'_uptime') and self._uptime: return self._uptime
+        return '*undef* uptime'
+    def sense_uptime (self):
+        command=['uptime']
+        self._uptime=self.backquote_ssh(command,trash_err=True).strip()
+        if not self._uptime: self._uptime='unreachable'
 
     def run(self,argv,message=None,trash_err=False,dry_run=False):
         if dry_run:
@@ -239,29 +281,35 @@ class Box:
             else:
                 return subprocess.call(argv,stderr=file('/dev/null','w'))
                 
-    def run_ssh (self, argv, message, trash_err=False):
+    def run_ssh (self, argv, message, trash_err=False, dry_run=False):
         ssh_argv = self.test_ssh().actual_argv(argv)
-        result=self.run (ssh_argv, message, trash_err)
+        result=self.run (ssh_argv, message, trash_err, dry_run=dry_run)
         if result!=0:
             print "WARNING: failed to run %s on %s"%(" ".join(argv),self.hostname)
         return result
 
     def backquote (self, argv, trash_err=False):
+        # print 'running backquote',argv
         if not trash_err:
             result= subprocess.Popen(argv,stdout=subprocess.PIPE).communicate()[0]
         else:
             result= subprocess.Popen(argv,stdout=subprocess.PIPE,stderr=file('/dev/null','w')).communicate()[0]
         return result
 
-    def backquote_ssh (self, argv, trash_err=False):
+    def probe (self):
+        if self._probed is not None: return self._probed
         # first probe the ssh link
         probe_argv=self.test_ssh().actual_argv(['hostname'])
-        hostname=self.backquote ( probe_argv, trash_err=True )
-        if not hostname:
-            print "root@%s unreachable"%self.hostname
-            return ''
-        else:
-            return self.backquote( self.test_ssh().actual_argv(argv), trash_err)
+        self._probed=self.backquote ( probe_argv, trash_err=True )
+        if not self._probed: print "root@%s unreachable"%self.hostname
+        return self._probed
+
+    # use argv=['bash','-c',"the command line"]
+    # if you have any shell-expanded arguments like *
+    # and if there's any chance the command is adressed to the local host
+    def backquote_ssh (self, argv, trash_err=False):
+        if not self.probe(): return ''
+        return self.backquote( self.test_ssh().actual_argv(argv), trash_err)
 
 ############################################################
 class BuildInstance:
@@ -296,21 +344,19 @@ class BuildBox (Box):
             for b in self.build_instances: 
                 header (b.line(),banner=False)
 
-    def uptime(self):
-        if hasattr(self,'_uptime') and self._uptime: return self._uptime
-        return '*undef* uptime'
+    def reboot (self, options):
+        if not options.soft:
+            self.reboot(options)
+        else:
+            command=['pkill','vbuild']
+            self.run_ssh(command,"Terminating vbuild processes",dry_run=options.dry_run)
 
     # inspect box and find currently running builds
     matcher=re.compile("\s*(?P<pid>[0-9]+).*-[bo]\s+(?P<buildname>[^\s]+)(\s|\Z)")
-    def sense(self,reboot=False,verbose=True):
-        if reboot:
-            self.reboot(box)
-            return
+    def sense(self, options):
         print 'b',
-        command=['uptime']
-        self._uptime=self.backquote_ssh(command,trash_err=True).strip()
-        if not self._uptime: self._uptime='unreachable'
-        pids=self.backquote_ssh(['pgrep','build'],trash_err=True)
+        self.sense_uptime()
+        pids=self.backquote_ssh(['pgrep','vbuild'],trash_err=True)
         if not pids: return
         command=['ps','-o','pid,command'] + [ pid for pid in pids.split("\n") if pid]
         ps_lines=self.backquote_ssh (command).split('\n')
@@ -338,12 +384,16 @@ class PlcInstance:
 
     def vplcname (self):
         return self.vservername.split('-')[-1]
+    def buildname (self):
+        return self.vservername.rsplit('-',2)[0]
 
     def line (self):
-        msg="== %s == (ctx=%s)"%(self.vservername,self.ctxid)
+        msg="== %s =="%(self.vplcname())
+        msg += " [=%s]"%self.vservername
+        if self.ctxid==0:  msg+=" not (yet?) running"
+        else:              msg+=" (ctx=%s)"%self.ctxid     
         if self.timestamp: msg += " @ %s"%self.pretty_timestamp()
         else:              msg += " *unknown timestamp*"
-        if self.ctxid==0:  msg+=" not (yet?) running"
         return msg
 
     def kill (self):
@@ -383,6 +433,7 @@ class PlcBox (Box):
             header ('No vserver running on %s'%(self.line()))
         else:
             header ("Active plc VMs on %s"%self.line())
+            self.plc_instances.sort(timestamp_sort)
             for p in self.plc_instances: 
                 header (p.line(),banner=False)
 
@@ -398,17 +449,14 @@ class PlcBox (Box):
             if p.vservername==vservername: return p
         return None
 
-    def sense (self, reboot=False, soft=False):
-        if reboot:
-            # remove mark for all running servers to avoid resurrection
-            stop_command=['rm','-rf','/etc/vservers/*/apps/init/mark']
-            self.run_ssh(stop_command,"Removing all vserver marks on %s"%self.hostname)
-            if not soft:
-                self.reboot()
-                return
-            else:
-                self.run_ssh(['service','util-vserver','stop'],"Stopping all running vservers")
-            return
+    def reboot (self, options):
+        if not options.soft:
+            self.reboot(options)
+        else:
+            self.run_ssh(['service','util-vserver','stop'],"Stopping all running vservers",
+                         dry_run=options.dry_run)
+
+    def sense (self, options):
         print 'p',
         self._uname=self.backquote_ssh(['uname','-r']).strip()
         # try to find fullname (vserver_stat truncates to a ridiculously short name)
@@ -430,27 +478,30 @@ class PlcBox (Box):
             if not vserver_line: continue
             context=vserver_line.split()[0]
             if context=="CTX": continue
-            longname=ctx_dict[context]
-            self.add_vserver(longname,context)
-#            print self.margin_outline(self.vplcname(longname)),"%(vserver_line)s [=%(longname)s]"%locals()
+            try:
+                longname=ctx_dict[context]
+                self.add_vserver(longname,context)
+            except:
+                print 'WARNING: found ctx %s in vserver_stat but was unable to figure a corresp. vserver'%context
 
         # scan timestamps 
         running_vsnames = [ i.vservername for i in self.plc_instances ]
         command=   ['grep','.']
-        command += ['/vservers/%s/timestamp'%vs for vs in running_vsnames]
+        command += ['/vservers/%s.timestamp'%vs for vs in running_vsnames]
         command += ['/dev/null']
         ts_lines=self.backquote_ssh(command,trash_err=True).split('\n')
         for ts_line in ts_lines:
             if not ts_line.strip(): continue
-            # expect /vservers/<vservername>/timestamp:<timestamp>
+            # expect /vservers/<vservername>.timestamp:<timestamp>
             try:
-                (_,__,vservername,tail)=ts_line.split('/')
-                (_,timestamp)=tail.split(':')
+                (ts_file,timestamp)=ts_line.split(':')
+                ts_file=os.path.basename(ts_file)
+                (vservername,_)=os.path.splitext(ts_file)
                 timestamp=int(timestamp)
                 p=self.plc_instance_by_vservername(vservername)
                 if not p: 
-                    print 'WARNING unattached plc instance',ts_line
-                    print 'was expecting to find',vservername,'in',[i.vservername for i in self.plc_instances]
+                    print 'WARNING zombie plc',self.hostname,ts_line
+                    print '... was expecting',vservername,'in',[i.vservername for i in self.plc_instances]
                     continue
                 p.set_timestamp(timestamp)
             except:  print 'WARNING, could not parse ts line',ts_line
@@ -474,13 +525,12 @@ class QemuInstance:
     def pretty_timestamp (self): return time.strftime("%Y-%m-%d:%H-%M",time.localtime(self.timestamp))
     
     def line (self):
-        msg = "== %s == (pid=%s)"%(self.nodename,self.pid)
-        if self.buildname: msg += " <--> %s"%self.buildname
-        else:              msg += " *unknown build*"
+        msg = "== %s =="%(short_hostname(self.nodename))
+        msg += " [=%s]"%self.buildname
+        if self.pid:       msg += " (pid=%s)"%self.pid
+        else:              msg += " not (yet?) running"
         if self.timestamp: msg += " @ %s"%self.pretty_timestamp()
         else:              msg += " *unknown timestamp*"
-        if self.pid:       msg += " pid=%s"%self.pid
-        else:              msg += " not (yet?) running"
         return msg
     
     def kill(self):
@@ -524,6 +574,7 @@ class QemuBox (Box):
             header ('No qemu process on %s'%(self.line()))
         else:
             header ("Active qemu processes on %s"%(self.line()))
+            self.qemu_instances.sort(timestamp_sort)
             for q in self.qemu_instances: 
                 header (q.line(),banner=False)
 
@@ -545,14 +596,15 @@ class QemuBox (Box):
                 return q
         return None
 
+    def reboot (self, options):
+        if not options.soft:
+            self.reboot(options)
+        else:
+            self.run_ssh(['pkill','qemu'],"Killing qemu instances",
+                         dry_run=options.dry_run)
+
     matcher=re.compile("\s*(?P<pid>[0-9]+).*-cdrom\s+(?P<nodename>[^\s]+)\.iso")
-    def sense(self, reboot=False, soft=False):
-        if reboot:
-            if not soft:
-                self.reboot()
-            else:
-                self.run_ssh(box,['pkill','qemu'],"Killing qemu instances")
-            return
+    def sense(self, options):
         print 'q',
         modules=self.backquote_ssh(['lsmod']).split('\n')
         self._driver='*NO kqemu/kmv_intel MODULE LOADED*'
@@ -588,6 +640,7 @@ class QemuBox (Box):
                 live_builds.append(buildname)
             except: print 'WARNING, could not parse pid line',pid_line
         # retrieve timestamps
+        if not live_builds: return
         command=   ['grep','.']
         command += ['%s/*/timestamp'%b for b in live_builds]
         command += ['/dev/null']
@@ -602,11 +655,156 @@ class QemuBox (Box):
                 timestamp=int(timestamp)
                 q=self.qemu_instance_by_nodename_buildname(nodename,buildname)
                 if not q: 
-                    print 'WARNING unattached qemu instance',ts_line,nodename,buildname
+                    print 'WARNING zombie qemu',self.hostname,ts_line
+                    print '... was expecting (',short_hostname(nodename),buildname,') in',\
+                        [ (short_hostname(i.nodename),i.buildname) for i in self.qemu_instances ]
                     continue
                 q.set_timestamp(timestamp)
             except:  print 'WARNING, could not parse ts line',ts_line
 
+####################
+class TestInstance:
+    def __init__ (self, buildname, pid=0):
+        self.pids=[]
+        if pid!=0: self.pid.append(pid)
+        self.buildname=buildname
+        # latest trace line
+        self.trace=''
+        # has a KO test
+        self.broken_steps=[]
+        self.timestamp = 0
+
+    def set_timestamp (self,timestamp): self.timestamp=timestamp
+    def set_now (self): self.timestamp=int(time.time())
+    def pretty_timestamp (self): return time.strftime("%Y-%m-%d:%H-%M",time.localtime(self.timestamp))
+
+
+    def add_pid (self,pid):
+        self.pids.append(pid)
+    def set_broken (self,plcindex, step): 
+        self.broken_steps.append ( (plcindex, step,) )
+
+    def line (self):
+        double='=='
+        if self.pids: double='*'+double[1]
+        if self.broken_steps: double=double[0]+'B'
+        msg = " %s %s =="%(double,self.buildname)
+        if not self.pids:       pass
+        elif len(self.pids)==1: msg += " (pid=%s)"%self.pids[0]
+        else:                   msg += " !!!pids=%s!!!"%self.pids
+        msg += " @%s"%self.pretty_timestamp()
+        if self.broken_steps:
+            msg += "\n BROKEN IN STEPS"
+            for (i,s) in self.broken_steps: msg += " %s@%s"%(s,i)
+        return msg
+
+class TestBox (Box):
+    def __init__ (self,hostname):
+        Box.__init__(self,hostname)
+        self.starting_ips=[]
+        self.test_instances=[]
+
+    def reboot (self, options):
+        # can't reboot a vserver VM
+        self.run_ssh (['pkill','run_log'],"Terminating current runs",
+                      dry_run=options.dry_run)
+        self.run_ssh (['rm','-f',Starting.location],"Cleaning %s"%Starting.location,
+                      dry_run=options.dry_run)
+
+    def get_test (self, buildname):
+        for i in self.test_instances:
+            if i.buildname==buildname: return i
+
+    # we scan ALL remaining test results, even the ones not running
+    def add_timestamp (self, buildname, timestamp):
+        i=self.get_test(buildname)
+        if i:   
+            i.set_timestamp(timestamp)
+        else:   
+            i=TestInstance(buildname,0)
+            i.set_timestamp(timestamp)
+            self.test_instances.append(i)
+
+    def add_running_test (self, pid, buildname):
+        i=self.get_test(buildname)
+        if not i:
+            self.test_instances.append (TestInstance (buildname,pid))
+            return
+        if i.pids:
+            print "WARNING: 2 concurrent tests run on same build %s"%buildname
+        i.add_pid (pid)
+
+    def add_broken (self, buildname, plcindex, step):
+        i=self.get_test(buildname)
+        if not i:
+            i=TestInstance(buildname)
+            self.test_instances.append(i)
+        i.set_broken(plcindex, step)
+
+    matcher_proc=re.compile (".*/proc/(?P<pid>[0-9]+)/cwd.*/root/(?P<buildname>[^/]+)$")
+    matcher_grep=re.compile ("/root/(?P<buildname>[^/]+)/logs/trace.*:TRACE:\s*(?P<plcindex>[0-9]+).*step=(?P<step>\S+).*")
+    def sense (self, options):
+        print 't',
+        self.sense_uptime()
+        self.starting_ips=[x for x in self.backquote_ssh(['cat',Starting.location], trash_err=True).strip().split('\n') if x]
+
+        # scan timestamps on all tests
+        # this is likely to not invoke ssh so we need to be a bit smarter to get * expanded
+        # xxx would make sense above too
+        command=['bash','-c',"grep . /root/*/timestamp /dev/null"]
+        ts_lines=self.backquote_ssh(command,trash_err=True).split('\n')
+        for ts_line in ts_lines:
+            if not ts_line.strip(): continue
+            # expect /root/<buildname>/timestamp:<timestamp>
+            try:
+                (ts_file,timestamp)=ts_line.split(':')
+                ts_file=os.path.dirname(ts_file)
+                buildname=os.path.basename(ts_file)
+                timestamp=int(timestamp)
+                t=self.add_timestamp(buildname,timestamp)
+            except:  print 'WARNING, could not parse ts line',ts_line
+
+        command=['bash','-c',"grep KO /root/*/logs/trace* /dev/null" ]
+        trace_lines=self.backquote_ssh (command).split('\n')
+        for line in trace_lines:
+            if not line.strip(): continue
+            m=TestBox.matcher_grep.match(line)
+            if m: 
+                buildname=m.group('buildname')
+                plcindex=m.group('plcindex')
+                step=m.group('step')
+                self.add_broken(buildname,plcindex, step)
+            else: header("command %r returned line that failed to match\n%s"%(command,line))
+
+        pids = self.backquote_ssh (['pgrep','run_log'],trash_err=True)
+        if not pids: return
+        command=['ls','-ld'] + ["/proc/%s/cwd"%pid for pid in pids.split("\n") if pid]
+        ps_lines=self.backquote_ssh (command).split('\n')
+        for line in ps_lines:
+            if not line.strip(): continue
+            m=TestBox.matcher_proc.match(line)
+            if m: 
+                pid=m.group('pid')
+                buildname=m.group('buildname')
+                self.add_running_test(pid, buildname)
+            else: header("command %r returned line that failed to match\n%s"%(command,line))
+        
+        
+    def line (self):
+        return "%s (%s)"%(self.hostname,self.uptime())
+
+    def list (self):
+        if not self.test_instances:
+            header ("No known tests on %s"%self.line())
+        else:
+            header ("Known tests on %s"%self.line())
+            self.test_instances.sort(timestamp_sort)
+            for i in self.test_instances: print i.line()
+        if self.starting_ips:
+            header ("Starting IP addresses on %s"%self.line())
+            self.starting_ips.sort()
+            for starting in self.starting_ips: print starting
+
 ############################################################
 class Options: pass
 
@@ -616,39 +814,49 @@ class Substrate:
         self.options=Options()
         self.options.dry_run=False
         self.options.verbose=False
-        self.options.probe=True
-        self.options.soft=True
+        self.options.reboot=False
+        self.options.soft=False
+        self.test_box = TestBox (self.test_box_spec())
         self.build_boxes = [ BuildBox(h) for h in self.build_boxes_spec() ]
         self.plc_boxes = [ PlcBox (h,m) for (h,m) in self.plc_boxes_spec ()]
         self.qemu_boxes = [ QemuBox (h,m) for (h,m) in self.qemu_boxes_spec ()]
-        self.all_boxes = self.build_boxes + self.plc_boxes + self.qemu_boxes
+        self.default_boxes = self.plc_boxes + self.qemu_boxes
+        self.all_boxes = self.build_boxes + [ self.test_box ] + self.plc_boxes + self.qemu_boxes
         self._sensed=False
 
-        self.vplc_pool = Pool (self.vplc_ips(),"for vplcs")
-        self.vnode_pool = Pool (self.vnode_ips(),"for vnodes")
+        self.vplc_pool = Pool (self.vplc_ips(),"for vplcs",self)
+        self.vnode_pool = Pool (self.vnode_ips(),"for vnodes",self)
 
     def fqdn (self, hostname):
         if hostname.find('.')<0: return "%s.%s"%(hostname,self.domain())
-    def short_hostname (self, hostname):
-        if hostname.find('.')>=0: return hostname.split('.')[0]
+        return hostname
 
     # return True if actual sensing takes place
     def sense (self,force=False):
         if self._sensed and not force: return False
         print 'Sensing local substrate...',
-        for b in self.all_boxes: b.sense()
+        for b in self.default_boxes: b.sense(self.options)
         print 'Done'
         self._sensed=True
         return True
 
+    def list (self):
+        for b in self.default_boxes:
+            b.list()
+
     def add_dummy_plc (self, plc_boxname, plcname):
         for pb in self.plc_boxes:
             if pb.hostname==plc_boxname:
                 pb.add_dummy(plcname)
+                return True
     def add_dummy_qemu (self, qemu_boxname, qemuname):
         for qb in self.qemu_boxes:
             if qb.hostname==qemu_boxname:
                 qb.add_dummy(qemuname)
+                return True
+
+    def add_starting_dummy (self, bname, vname):
+        return self.add_dummy_plc (bname, vname) or self.add_dummy_qemu (bname, vname)
 
     ########## 
     def provision (self,plcs,options):
@@ -659,6 +867,7 @@ class Substrate:
             plcs = [ self.provision_qemus (plc,options) for plc in plcs ]
             # update the SFA spec accordingly
             plcs = [ self.localize_sfa_rspec(plc,options) for plc in plcs ]
+            self.list()
             return plcs
         except Exception, e:
             print '* Could not provision this test on current substrate','--',e,'--','exiting'
@@ -687,7 +896,7 @@ class Substrate:
             plc_boxname = options.ips_bplc.pop()
             vplc_hostname=options.ips_vplc.pop()
         else:
-            if self.sense(): self.list_all()
+            if self.sense(): self.list()
             plc_boxname=None
             vplc_hostname=None
             # try to find an available IP 
@@ -735,11 +944,11 @@ class Substrate:
         # 
         self.add_dummy_plc(plc_boxname,plc['name'])
         vplc_ip = self.vplc_pool.get_ip(vplc_hostname)
-        self.vplc_pool.add_starting(vplc_hostname)
+        self.vplc_pool.add_starting(vplc_hostname, plc_boxname)
 
         #### compute a helpful vserver name
         # remove domain in hostname
-        vplc_short = self.short_hostname(vplc_hostname)
+        vplc_short = short_hostname(vplc_hostname)
         vservername = "%s-%d-%s" % (options.buildname,plc['index'],vplc_short)
         plc_name = "%s_%s"%(plc['name'],vplc_short)
 
@@ -782,7 +991,7 @@ class Substrate:
                 qemu_boxname=options.ips_bnode.pop()
                 vnode_hostname=options.ips_vnode.pop()
             else:
-                if self.sense(): self.list_all()
+                if self.sense(): self.list()
                 qemu_boxname=None
                 vnode_hostname=None
                 # try to find an available IP 
@@ -813,7 +1022,7 @@ class Substrate:
                         if not vnode_hostname: msg += " vnode IP pool exhausted" 
                         raise Exception,"Could not make space for a QEMU instance:"+msg
                     freed_qemu_boxname=qemu_instance_to_kill.qemu_box.hostname
-                    freed_vnode_hostname=self.short_hostname(qemu_instance_to_kill.nodename)
+                    freed_vnode_hostname=short_hostname(qemu_instance_to_kill.nodename)
                     # kill it
                     message='killing oldest qemu node = %s on %s'%(qemu_instance_to_kill.line(),
                                                                    freed_qemu_boxname)
@@ -825,10 +1034,10 @@ class Substrate:
                         vnode_hostname=freed_vnode_hostname
                         self.vnode_pool.set_mine(vnode_hostname)
 
-            self.add_dummy_qemu (qemu_boxname,nodename)
+            self.add_dummy_qemu (qemu_boxname,vnode_hostname)
             mac=self.vnode_pool.retrieve_userdata(vnode_hostname)
             ip=self.vnode_pool.get_ip (vnode_hostname)
-            self.vnode_pool.add_starting(vnode_hostname)
+            self.vnode_pool.add_starting(vnode_hostname,qemu_boxname)
 
             vnode_fqdn = self.fqdn(vnode_hostname)
             nodemap={'host_box':qemu_boxname,
@@ -849,11 +1058,8 @@ class Substrate:
         plc['sfa']['SFA_REGISTRY_HOST'] = plc['PLC_DB_HOST']
         plc['sfa']['SFA_AGGREGATE_HOST'] = plc['PLC_DB_HOST']
         plc['sfa']['SFA_SM_HOST'] = plc['PLC_DB_HOST']
-        plc['sfa']['SFA_PLC_DB_HOST'] = plc['PLC_DB_HOST']
+        plc['sfa']['SFA_DB_HOST'] = plc['PLC_DB_HOST']
         plc['sfa']['SFA_PLC_URL'] = 'https://' + plc['PLC_API_HOST'] + ':443/PLCAPI/' 
-        for site in plc['sites']:
-            for node in site['nodes']:
-                plc['sfa']['sfa_slice_rspec']['part4'] = node['node_fields']['hostname']
 	return plc
 
     #################### release:
@@ -863,33 +1069,65 @@ class Substrate:
         pass
 
     #################### show results for interactive mode
-    def list_all (self):
-        self.sense()
-        for b in self.all_boxes: b.list()
-
-    def get_box (self,box):
-        for b in self.build_boxes + self.plc_boxes + self.qemu_boxes:
-            if b.short_hostname()==box:
+    def get_box (self,boxname):
+        for b in self.build_boxes + self.plc_boxes + self.qemu_boxes + [self.test_box] :
+            if b.shortname()==boxname:
                 return b
-        print "Could not find box %s"%box
+        print "Could not find box %s"%boxname
         return None
 
-    def list_box(self,box):
-        b=self.get_box(box)
-        if not b: return
-        b.sense()
-        b.list()
-
+    def list_boxes(self,box_or_names):
+        print 'Sensing',
+        for box in box_or_names:
+            if not isinstance(box,Box): box=self.get_box(box)
+            if not box: continue
+            box.sense(self.options)
+        print 'Done'
+        for box in box_or_names:
+            if not isinstance(box,Box): box=self.get_box(box)
+            if not box: continue
+            box.list()
+
+    def reboot_boxes(self,box_or_names):
+        for box in box_or_names:
+            if not isinstance(box,Box): box=self.get_box(box)
+            if not box: continue
+            box.reboot(self.options)
+
+    ####################
     # can be run as a utility to manage the local infrastructure
     def main (self):
         parser=OptionParser()
+        parser.add_option ('-r',"--reboot",action='store_true',dest='reboot',default=False,
+                           help='reboot mode (use shutdown -r)')
+        parser.add_option ('-s',"--soft",action='store_true',dest='soft',default=False,
+                           help='soft mode for reboot (vserver stop or kill qemus)')
+        parser.add_option ('-t',"--testbox",action='store_true',dest='testbox',default=False,
+                           help='add test box') 
+        parser.add_option ('-b',"--build",action='store_true',dest='builds',default=False,
+                           help='add build boxes')
+        parser.add_option ('-p',"--plc",action='store_true',dest='plcs',default=False,
+                           help='add plc boxes')
+        parser.add_option ('-q',"--qemu",action='store_true',dest='qemus',default=False,
+                           help='add qemu boxes') 
+        parser.add_option ('-a',"--all",action='store_true',dest='all',default=False,
+                           help='address all known  boxes, like -b -t -p -q')
         parser.add_option ('-v',"--verbose",action='store_true',dest='verbose',default=False,
                            help='verbose mode')
-        (options,args)=parser.parse_args()
-        if options.verbose:
-            self.options.verbose=True
-        if not args:
-            self.list_all()
-        else:
-            for box in args:
-                self.list_box(box)
+        parser.add_option ('-n',"--dry_run",action='store_true',dest='dry_run',default=False,
+                           help='dry run mode')
+        (self.options,args)=parser.parse_args()
+
+        boxes=args
+        if self.options.testbox: boxes += [self.test_box]
+        if self.options.builds: boxes += self.build_boxes
+        if self.options.plcs: boxes += self.plc_boxes
+        if self.options.qemus: boxes += self.qemu_boxes
+        if self.options.all: boxes += self.all_boxes
+        
+        # default scope is -b -p -q
+        if not boxes:
+            boxes = self.build_boxes + self.plc_boxes + self.qemu_boxes
+
+        if self.options.reboot: self.reboot_boxes (boxes)
+        else:                   self.list_boxes (boxes)