bonds: use rung <=> run -G that hurts my pinky

[tests.git] / system / Substrate.py
diff --git a/system/Substrate.py b/system/Substrate.py

index cbc6f71..a6d02e2 100644 (file)
--- a/system/Substrate.py
+++ b/system/Substrate.py
@@ -1,6 +1,6 @@
  #
  # Thierry Parmentelat <thierry.parmentelat@inria.fr>
  #
  # Thierry Parmentelat <thierry.parmentelat@inria.fr>
-# Copyright (C) 2010 INRIA 
+# Copyright (C) 2010-2015 INRIA 
  #
  # #################### history
  #
  #
  # #################### history
  #
@@ -51,24 +51,25 @@ import time
  import re
  import traceback
  import subprocess
  import re
  import traceback
  import subprocess
-import commands
  import socket
  from optparse import OptionParser
  
  import utils
  from TestSsh import TestSsh
  from TestMapper import TestMapper
  import socket
  from optparse import OptionParser
  
  import utils
  from TestSsh import TestSsh
  from TestMapper import TestMapper
+from functools import reduce
  
  # too painful to propagate this cleanly
  
  # too painful to propagate this cleanly
-verbose=None
+verbose = None
  
  
-def header (message,banner=True):
+def header (message, banner=True):
      if not message: return
      if not message: return
-    if banner: print "===============",
-    print message
+    if banner:
+        print("===============", end=' ')
+    print(message)
      sys.stdout.flush()
  
      sys.stdout.flush()
  
-def timestamp_sort(o1,o2): return o1.timestamp-o2.timestamp
+def timestamp_key(o): return o.timestamp
  
  def short_hostname (hostname):
      return hostname.split('.')[0]
  
  def short_hostname (hostname):
      return hostname.split('.')[0]
@@ -78,30 +79,37 @@ def short_hostname (hostname):
  # instances, that go undetected through sensing
  class Starting:
  
  # instances, that go undetected through sensing
  class Starting:
  
-    location='/root/starting'
+    location = '/root/starting'
+    
      def __init__ (self):
          self.tuples=[]
  
      def __init__ (self):
          self.tuples=[]
  
+    def __repr__(self):
+        return '<Starting>'
+
      def load (self):
      def load (self):
-        try:    self.tuples=[line.strip().split('@') 
-                             for line in file(Starting.location).readlines()]
-        except: self.tuples=[]
+        try:
+            with open(Starting.location) as starting:
+                self.tuples = [line.strip().split('@') for line in starting.readlines()]
+        except:
+            self.tuples = []
  
      def vnames (self) : 
          self.load()
  
      def vnames (self) : 
          self.load()
-        return [ x for (x,_) in self.tuples ]
+        return [ x for (x, _) in self.tuples ]
  
      def add (self, vname, bname):
          if not vname in self.vnames():
  
      def add (self, vname, bname):
          if not vname in self.vnames():
-            file(Starting.location,'a').write("%s@%s\n"%(vname,bname))
+            with open(Starting.location, 'a') as out:
+                out.write("{}@{}\n".format(vname, bname))
              
      def delete_vname (self, vname):
          self.load()
          if vname in self.vnames():
              
      def delete_vname (self, vname):
          self.load()
          if vname in self.vnames():
-            f=file(Starting.location,'w')
-            for (v,b) in self.tuples: 
-                if v != vname: f.write("%s@%s\n"%(v,b))
-            f.close()
+            with open(Starting.location, 'w') as f:
+                for (v, b) in self.tuples: 
+                    if v != vname:
+                        f.write("{}@{}\n".format(v, b))
      
  ####################
  # pool class
      
  ####################
  # pool class
@@ -121,157 +129,178 @@ class Starting:
  # that is, even if ip2 is not busy/pingable when the second next_free() is issued
  
  class PoolItem:
  # that is, even if ip2 is not busy/pingable when the second next_free() is issued
  
  class PoolItem:
-    def __init__ (self,hostname,userdata):
-        self.hostname=hostname
-        self.userdata=userdata
+    def __init__ (self, hostname, userdata):
+        self.hostname = hostname
+        self.userdata = userdata
          # slot holds 'busy' or 'free' or 'mine' or 'starting' or None
          # 'mine' is for our own stuff, 'starting' from the concurrent tests
          # slot holds 'busy' or 'free' or 'mine' or 'starting' or None
          # 'mine' is for our own stuff, 'starting' from the concurrent tests
-        self.status=None
-        self.ip=None
+        self.status = None
+        self.ip = None
+
+    def __repr__(self):
+        return "<PoolItem {} {}>".format(self.hostname, self.userdata)
  
      def line(self):
  
      def line(self):
-        return "Pooled %s (%s) -> %s"%(self.hostname,self.userdata, self.status)
+        return "Pooled {} ({}) -> {}".format(self.hostname, self.userdata, self.status)
  
      def char (self):
  
      def char (self):
-        if   self.status==None:       return '?'
-        elif self.status=='busy':     return '+'
-        elif self.status=='free':     return '-'
-        elif self.status=='mine':     return 'M'
-        elif self.status=='starting': return 'S'
+        if   self.status == None:       return '?'
+        elif self.status == 'busy':     return '+'
+        elif self.status == 'free':     return '-'
+        elif self.status == 'mine':     return 'M'
+        elif self.status == 'starting': return 'S'
  
      def get_ip(self):
  
      def get_ip(self):
-        if self.ip: return self.ip
-        ip=socket.gethostbyname(self.hostname)
-        self.ip=ip
-        return ip
+        if self.ip:
+            return self.ip
+        self.ip = socket.gethostbyname(self.hostname)
+        return self.ip
  
  class Pool:
  
  
  class Pool:
  
-    def __init__ (self, tuples,message, substrate):
-        self.pool_items= [ PoolItem (hostname,userdata) for (hostname,userdata) in tuples ] 
-        self.message=message
+    def __init__ (self, tuples, message, substrate):
+        self.pool_items = [ PoolItem (hostname, userdata) for (hostname, userdata) in tuples ] 
+        self.message = message
          # where to send notifications upon load_starting
          # where to send notifications upon load_starting
-        self.substrate=substrate
+        self.substrate = substrate
+
+    def __repr__(self):
+        return "<Pool {} : {} .. {}>".format(self.message, self.pool_items[0], self.pool_items[-1])
  
      def list (self, verbose=False):
  
      def list (self, verbose=False):
-        for i in self.pool_items: print i.line()
+        print(self)
+        for i in self.pool_items: print(i.line())
  
      def line (self):
  
      def line (self):
-        line=self.message
+        line = self.message
          for i in self.pool_items: line += ' ' + i.char()
          return line
  
      def _item (self, hostname):
          for i in self.pool_items: 
          for i in self.pool_items: line += ' ' + i.char()
          return line
  
      def _item (self, hostname):
          for i in self.pool_items: 
-            if i.hostname==hostname: return i
-        raise Exception ("Could not locate hostname %s in pool %s"%(hostname,self.message))
+            if i.hostname == hostname: return i
+        raise Exception ("Could not locate hostname {} in pool {}".format(hostname, self.message))
  
      def retrieve_userdata (self, hostname): 
          return self._item(hostname).userdata
  
      def get_ip (self, hostname):
  
      def retrieve_userdata (self, hostname): 
          return self._item(hostname).userdata
  
      def get_ip (self, hostname):
-        try:    return self._item(hostname).get_ip()
-        except: return socket.gethostbyname(hostname)
+        try:
+            return self._item(hostname).get_ip()
+        except:
+            return socket.gethostbyname(hostname)
          
      def set_mine (self, hostname):
          try:
              self._item(hostname).status='mine'
          except:
          
      def set_mine (self, hostname):
          try:
              self._item(hostname).status='mine'
          except:
-            print 'WARNING: host %s not found in IP pool %s'%(hostname,self.message)
+            print('WARNING: host {} not found in IP pool {}'.format(hostname, self.message))
  
      def next_free (self):
          for i in self.pool_items:
              if i.status == 'free':
  
      def next_free (self):
          for i in self.pool_items:
              if i.status == 'free':
-                i.status='mine'
-                return (i.hostname,i.userdata)
+                i.status = 'mine'
+                return (i.hostname, i.userdata)
          return None
  
      ####################
      # we have a starting instance of our own
      def add_starting (self, vname, bname):
          return None
  
      ####################
      # we have a starting instance of our own
      def add_starting (self, vname, bname):
-        Starting().add(vname,bname)
+        Starting().add(vname, bname)
          for i in self.pool_items:
          for i in self.pool_items:
-            if i.hostname==vname: i.status='mine'
+            if i.hostname == vname:
+                i.status = 'mine'
  
      # load the starting instances from the common file
      # remember that might be ours
      # return the list of (vname,bname) that are not ours
      def load_starting (self):
  
      # load the starting instances from the common file
      # remember that might be ours
      # return the list of (vname,bname) that are not ours
      def load_starting (self):
-        starting=Starting()
+        starting = Starting()
          starting.load()
          starting.load()
-        new_tuples=[]
-        for (v,b) in starting.tuples:
+        new_tuples = []
+        for (v, b) in starting.tuples:
              for i in self.pool_items:
              for i in self.pool_items:
-                if i.hostname==v and i.status=='free':
-                    i.status='starting'
-                    new_tuples.append( (v,b,) )
+                if i.hostname == v and i.status == 'free':
+                    i.status = 'starting'
+                    new_tuples.append( (v, b,) )
          return new_tuples
  
      def release_my_starting (self):
          for i in self.pool_items:
          return new_tuples
  
      def release_my_starting (self):
          for i in self.pool_items:
-            if i.status=='mine':
-                Starting().delete_vname (i.hostname)
-                i.status=None
+            if i.status == 'mine':
+                Starting().delete_vname(i.hostname)
+                i.status = None
  
  
      ##########
      def _sense (self):
          for item in self.pool_items:
              if item.status is not None: 
  
  
      ##########
      def _sense (self):
          for item in self.pool_items:
              if item.status is not None: 
-                print item.char(),
+                print(item.char(), end=' ')
                  continue
              if self.check_ping (item.hostname): 
                  continue
              if self.check_ping (item.hostname): 
-                item.status='busy'
-                print '*',
+                item.status = 'busy'
+                print('*', end=' ')
              else:
              else:
-                item.status='free'
-                print '.',
+                item.status = 'free'
+                print('.', end=' ')
+            sys.stdout.flush()
      
      def sense (self):
      
      def sense (self):
-        print 'Sensing IP pool',self.message,
+        print('Sensing IP pool', self.message, end=' ')
+        sys.stdout.flush()
          self._sense()
          self._sense()
-        print 'Done'
-        for (vname,bname) in self.load_starting():
-            self.substrate.add_starting_dummy (bname, vname)
-        print 'After starting: IP pool'
-        print self.line()
+        print('Done')
+        for vname, bname in self.load_starting():
+            self.substrate.add_starting_dummy(bname, vname)
+        print("After having loaded 'starting': IP pool")
+        print(self.line())
      # OS-dependent ping option (support for macos, for convenience)
      ping_timeout_option = None
      # returns True when a given hostname/ip responds to ping
      # OS-dependent ping option (support for macos, for convenience)
      ping_timeout_option = None
      # returns True when a given hostname/ip responds to ping
-    def check_ping (self,hostname):
+    def check_ping (self, hostname):
+        if '.' not in hostname:
+            hostname = self.substrate.fqdn(hostname)
          if not Pool.ping_timeout_option:
          if not Pool.ping_timeout_option:
-            (status,osname) = commands.getstatusoutput("uname -s")
+            (status, osname) = subprocess.getstatusoutput("uname -s")
              if status != 0:
              if status != 0:
-                raise Exception, "TestPool: Cannot figure your OS name"
+                raise Exception("TestPool: Cannot figure your OS name")
              if osname == "Linux":
              if osname == "Linux":
-                Pool.ping_timeout_option="-w"
+                Pool.ping_timeout_option = "-w"
              elif osname == "Darwin":
              elif osname == "Darwin":
-                Pool.ping_timeout_option="-t"
+                Pool.ping_timeout_option = "-t"
  
  
-        command="ping -c 1 %s 1 %s"%(Pool.ping_timeout_option,hostname)
-        (status,output) = commands.getstatusoutput(command)
+        command = "ping -c 1 {} 1 {}".format(Pool.ping_timeout_option, hostname)
+        (status, output) = subprocess.getstatusoutput(command)
          return status == 0
  
  ####################
  class Box:
          return status == 0
  
  ####################
  class Box:
-    def __init__ (self,hostname):
-        self.hostname=hostname
-        self._probed=None
+    def __init__ (self, hostname):
+        self.hostname = hostname
+        self._probed = None
+    def __repr__(self):
+        return "<Box {}>".format(self.hostname)
      def shortname (self):
          return short_hostname(self.hostname)
      def shortname (self):
          return short_hostname(self.hostname)
-    def test_ssh (self): return TestSsh(self.hostname,username='root',unknown_host=False)
+    def test_ssh (self):
+        return TestSsh(self.hostname, username='root', unknown_host=False)
      def reboot (self, options):
      def reboot (self, options):
-        self.test_ssh().run("shutdown -r now",message="Rebooting %s"%self.hostname,
+        self.test_ssh().run("shutdown -r now",
+                            message="Rebooting {}".format(self.hostname),
                              dry_run=options.dry_run)
  
                              dry_run=options.dry_run)
  
-    def hostname_fedora (self,virt=None):
-        result = "%s {"%self.hostname
-        if virt: result += "%s-"%virt
-        result += "%s"%self.fedora()
+    def hostname_fedora (self, virt=None):
+        # this truly is an opening bracket
+        result = "{}".format(self.hostname) + " {"
+        if virt:
+            result += "{}-".format(virt)
+        result += "{} {}".format(self.fedora(), self.memory())
          # too painful to propagate this cleanly
          global verbose
          if verbose:
          # too painful to propagate this cleanly
          global verbose
          if verbose:
-            result += "-%s" % self.uname()
+            result += "-{}".format(self.uname())
+        # and the matching closing bracket
          result += "}"
          return result
  
          result += "}"
          return result
  
@@ -290,32 +319,38 @@ class Box:
          composite_command += [ "uname", "-r"]
          composite_command += [ ";" , "echo", Box.separator , ";" ]
          composite_command += [ "cat" , "/etc/fedora-release" ]
          composite_command += [ "uname", "-r"]
          composite_command += [ ";" , "echo", Box.separator , ";" ]
          composite_command += [ "cat" , "/etc/fedora-release" ]
+        composite_command += [ ";" , "echo", Box.separator , ";" ]
+        composite_command += [ "grep", "MemTotal", "/proc/meminfo" ]
  
          # due to colons and all, this is going wrong on the local box (typically testmaster)
          # I am reluctant to change TestSsh as it might break all over the place, so
          if self.test_ssh().is_local():
              probe_argv = [ "bash", "-c", " ".join (composite_command) ]
          else:
  
          # due to colons and all, this is going wrong on the local box (typically testmaster)
          # I am reluctant to change TestSsh as it might break all over the place, so
          if self.test_ssh().is_local():
              probe_argv = [ "bash", "-c", " ".join (composite_command) ]
          else:
-            probe_argv=self.test_ssh().actual_argv(composite_command)
-        composite=self.backquote ( probe_argv, trash_err=True )
-        self._hostname = self._uptime = self._uname = self._fedora = "** Unknown **"
+            probe_argv = self.test_ssh().actual_argv(composite_command)
+        composite = self.backquote ( probe_argv, trash_err=True )
+        self._hostname = self._uptime = self._uname = self._fedora = self._memory = "** Unknown **"
          if not composite: 
          if not composite: 
-            print "root@%s unreachable"%self.hostname
-            self._probed=''
+            print("root@{} unreachable".format(self.hostname))
+            self._probed = ''
          else:
              try:
                  pieces = composite.split(Box.separator)
                  pieces = [ x.strip() for x in pieces ]
          else:
              try:
                  pieces = composite.split(Box.separator)
                  pieces = [ x.strip() for x in pieces ]
-                [self._hostname, self._uptime, self._uname, self._fedora] = pieces
+                # get raw data
+                [hostname, uptime, uname, fedora, memory] = pieces
                  # customize
                  # customize
-                self._uptime = ', '.join([ x.strip() for x in self._uptime.split(',')[2:]])
-                self._fedora = self._fedora.replace("Fedora release ","f").split(" ")[0]
-            except:
+                self._hostname = hostname
+                self._uptime = ', '.join([ x.strip() for x in uptime.split(',')[2:]]).replace("load average", "load")
+                self._uname = uname
+                self._fedora = fedora.replace("Fedora release ","f").split(" ")[0]
+                # translate into Mb
+                self._memory = int(memory.split()[1])/(1024)
+            except Exception as e:
                  import traceback
                  import traceback
-                print 'BEG issue with pieces',pieces
+                print('BEG issue with pieces')
                  traceback.print_exc()
                  traceback.print_exc()
-                print 'END issue with pieces',pieces
-            self._probed=self._hostname
+            self._probed = self._hostname
          return self._probed
  
      # use argv=['bash','-c',"the command line"]
          return self._probed
  
      # use argv=['bash','-c',"the command line"]
@@ -331,76 +366,89 @@ class Box:
          self.probe()
          if hasattr(self,'_fedora') and self._fedora: return self._fedora
          return '*unprobed* fedora'
          self.probe()
          if hasattr(self,'_fedora') and self._fedora: return self._fedora
          return '*unprobed* fedora'
+    def memory(self):
+        self.probe()
+        if hasattr(self,'_memory') and self._memory: return "{} Mb".format(self._memory)
+        return '*unprobed* memory'
  
  
-    def run(self,argv,message=None,trash_err=False,dry_run=False):
+    def run(self, argv, message=None, trash_err=False, dry_run=False):
          if dry_run:
          if dry_run:
-            print 'DRY_RUN:',
-            print " ".join(argv)
+            print('DRY_RUN:', end=' ')
+            print(" ".join(argv))
              return 0
          else:
              header(message)
              if not trash_err:
                  return subprocess.call(argv)
              else:
              return 0
          else:
              header(message)
              if not trash_err:
                  return subprocess.call(argv)
              else:
-                return subprocess.call(argv,stderr=file('/dev/null','w'))
+                with open('/dev/null', 'w') as null:
+                    return subprocess.call(argv, stderr=null)
                  
      def run_ssh (self, argv, message, trash_err=False, dry_run=False):
          ssh_argv = self.test_ssh().actual_argv(argv)
                  
      def run_ssh (self, argv, message, trash_err=False, dry_run=False):
          ssh_argv = self.test_ssh().actual_argv(argv)
-        result=self.run (ssh_argv, message, trash_err, dry_run=dry_run)
-        if result!=0:
-            print "WARNING: failed to run %s on %s"%(" ".join(argv),self.hostname)
+        result = self.run (ssh_argv, message, trash_err, dry_run=dry_run)
+        if result != 0:
+            print("WARNING: failed to run {} on {}".format(" ".join(argv), self.hostname))
          return result
  
      def backquote (self, argv, trash_err=False):
          return result
  
      def backquote (self, argv, trash_err=False):
-        # print 'running backquote',argv
+        # in python3 we need to set universal_newlines=True
          if not trash_err:
          if not trash_err:
-            result= subprocess.Popen(argv,stdout=subprocess.PIPE).communicate()[0]
+            out_err = subprocess.Popen(argv, stdout=subprocess.PIPE,
+                                       universal_newlines=True).communicate()
          else:
          else:
-            result= subprocess.Popen(argv,stdout=subprocess.PIPE,stderr=file('/dev/null','w')).communicate()[0]
-        return result
+            with open('/dev/null', 'w') as null:
+                out_err = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=null,
+                                           universal_newlines=True).communicate()
+        # only interested in stdout here
+        return out_err[0]
  
      # if you have any shell-expanded arguments like *
      # and if there's any chance the command is adressed to the local host
      def backquote_ssh (self, argv, trash_err=False):
          if not self.probe(): return ''
  
      # if you have any shell-expanded arguments like *
      # and if there's any chance the command is adressed to the local host
      def backquote_ssh (self, argv, trash_err=False):
          if not self.probe(): return ''
-        return self.backquote( self.test_ssh().actual_argv(argv), trash_err)
+        return self.backquote(self.test_ssh().actual_argv(argv), trash_err)
  
  ############################################################
  class BuildInstance:
      def __init__ (self, buildname, pid, buildbox):
  
  ############################################################
  class BuildInstance:
      def __init__ (self, buildname, pid, buildbox):
-        self.buildname=buildname
-        self.buildbox=buildbox
-        self.pids=[pid]
+        self.buildname = buildname
+        self.buildbox = buildbox
+        self.pids = [pid]
+    def __repr__(self):
+        return "<BuildInstance {} in {}>".format(self.buildname, self.buildbox)
  
      def add_pid(self,pid):
          self.pids.append(pid)
  
      def line (self):
  
      def add_pid(self,pid):
          self.pids.append(pid)
  
      def line (self):
-        return "== %s == (pids=%r)"%(self.buildname,self.pids)
+        return "== {} == (pids={})".format(self.buildname, self.pids)
  
  class BuildBox (Box):
  
  class BuildBox (Box):
-    def __init__ (self,hostname):
-        Box.__init__(self,hostname)
-        self.build_instances=[]
+    def __init__ (self, hostname):
+        Box.__init__(self, hostname)
+        self.build_instances = []
+    def __repr__(self):
+        return "<BuildBox {}>".format(self.hostname)
  
  
-    def add_build (self,buildname,pid):
+    def add_build(self, buildname, pid):
          for build in self.build_instances:
          for build in self.build_instances:
-            if build.buildname==buildname: 
+            if build.buildname == buildname: 
                  build.add_pid(pid)
                  return
          self.build_instances.append(BuildInstance(buildname, pid, self))
  
      def list(self, verbose=False):
          if not self.build_instances: 
                  build.add_pid(pid)
                  return
          self.build_instances.append(BuildInstance(buildname, pid, self))
  
      def list(self, verbose=False):
          if not self.build_instances: 
-            header ('No build process on %s (%s)'%(self.hostname_fedora(),self.uptime()))
+            header ('No build process on {} ({})'.format(self.hostname_fedora(), self.uptime()))
          else:
          else:
-            header ("Builds on %s (%s)"%(self.hostname_fedora(),self.uptime()))
+            header ("Builds on {} ({})".format(self.hostname_fedora(), self.uptime()))
              for b in self.build_instances: 
              for b in self.build_instances: 
-                header (b.line(),banner=False)
+                header (b.line(), banner=False)
  
      def reboot (self, options):
          if not options.soft:
  
      def reboot (self, options):
          if not options.soft:
-            Box.reboot(self,options)
+            Box.reboot(self, options)
          else:
              self.soft_reboot (options)
  
          else:
              self.soft_reboot (options)
  
@@ -410,48 +458,56 @@ build_matcher_initvm=re.compile("\s*(?P<pid>[0-9]+).*initvm.*\s+(?P<buildname>[^
  class BuildLxcBox (BuildBox):
      def soft_reboot (self, options):
              command=['pkill','lbuild']
  class BuildLxcBox (BuildBox):
      def soft_reboot (self, options):
              command=['pkill','lbuild']
-            self.run_ssh(command,"Terminating vbuild processes",dry_run=options.dry_run)
+            self.run_ssh(command, "Terminating vbuild processes", dry_run=options.dry_run)
  
      # inspect box and find currently running builds
      def sense(self, options):
  
      # inspect box and find currently running builds
      def sense(self, options):
-        print 'xb',
-        pids=self.backquote_ssh(['pgrep','lbuild'],trash_err=True)
+        print('xb', end=' ')
+        sys.stdout.flush()
+        pids = self.backquote_ssh(['pgrep','lbuild'], trash_err=True)
          if not pids: return
          if not pids: return
-        command=['ps','-o','pid,command'] + [ pid for pid in pids.split("\n") if pid]
-        ps_lines=self.backquote_ssh (command).split('\n')
+        command = ['ps', '-o', 'pid,command'] + [ pid for pid in pids.split("\n") if pid]
+        ps_lines = self.backquote_ssh(command).split('\n')
          for line in ps_lines:
          for line in ps_lines:
-            if not line.strip() or line.find('PID')>=0: continue
-            m=build_matcher.match(line)
+            if not line.strip() or line.find('PID') >= 0: continue
+            m = build_matcher.match(line)
              if m: 
              if m: 
-                date=time.strftime('%Y-%m-%d',time.localtime(time.time()))
-                buildname=m.group('buildname').replace('@DATE@',date)
-                self.add_build (buildname,m.group('pid'))
+                date = time.strftime('%Y-%m-%d', time.localtime(time.time()))
+                buildname = m.group('buildname').replace('@DATE@', date)
+                self.add_build(buildname, m.group('pid'))
                  continue
                  continue
-            m=build_matcher_initvm.match(line)
+            m = build_matcher_initvm.match(line)
              if m: 
                  # buildname is expansed here
              if m: 
                  # buildname is expansed here
-                self.add_build (buildname,m.group('pid'))
+                self.add_build(buildname, m.group('pid'))
                  continue
                  continue
-            header('BuildLxcBox.sense: command %r returned line that failed to match'%command)
-            header(">>%s<<"%line)
+            header('BuildLxcBox.sense: command {} returned line that failed to match'.format(command))
+            header(">>{}<<".format(line))
      
  ############################################################
  class PlcInstance:
      def __init__ (self, plcbox):
      
  ############################################################
  class PlcInstance:
      def __init__ (self, plcbox):
-        self.plc_box=plcbox
+        self.plc_box = plcbox
          # unknown yet
          # unknown yet
-        self.timestamp=0
+        self.timestamp = 0
+    def __repr__(self):
+        return "<PlcInstance {}>".format(self.plc_box)
          
          
-    def set_timestamp (self,timestamp): self.timestamp=timestamp
-    def set_now (self): self.timestamp=int(time.time())
-    def pretty_timestamp (self): return time.strftime("%Y-%m-%d:%H-%M",time.localtime(self.timestamp))
+    def set_timestamp (self,timestamp):
+        self.timestamp = timestamp
+    def set_now (self):
+        self.timestamp = int(time.time())
+    def pretty_timestamp (self):
+        return time.strftime("%Y-%m-%d:%H-%M", time.localtime(self.timestamp))
  
  class PlcLxcInstance (PlcInstance):
      # does lxc have a context id of any kind ?
      def __init__ (self, plcbox, lxcname, pid):
          PlcInstance.__init__(self, plcbox)
          self.lxcname = lxcname
  
  class PlcLxcInstance (PlcInstance):
      # does lxc have a context id of any kind ?
      def __init__ (self, plcbox, lxcname, pid):
          PlcInstance.__init__(self, plcbox)
          self.lxcname = lxcname
-       self.pid = pid
+        self.pid = pid
+    def __repr__(self):
+        return "<PlcLxcInstance {}>".format(self.lxcname)
  
      def vplcname (self):
          return self.lxcname.split('-')[-1]
  
      def vplcname (self):
          return self.lxcname.split('-')[-1]
@@ -459,34 +515,36 @@ class PlcLxcInstance (PlcInstance):
          return self.lxcname.rsplit('-',2)[0]
  
      def line (self):
          return self.lxcname.rsplit('-',2)[0]
  
      def line (self):
-        msg="== %s =="%(self.vplcname())
-        msg += " [=%s]"%self.lxcname
+        msg="== {} ==".format(self.vplcname())
+        msg += " [={}]".format(self.lxcname)
          if self.pid==-1:  msg+=" not (yet?) running"
          if self.pid==-1:  msg+=" not (yet?) running"
-        else:              msg+=" (pid=%s)"%self.pid
-        if self.timestamp: msg += " @ %s"%self.pretty_timestamp()
+        else:              msg+=" (pid={})".format(self.pid)
+        if self.timestamp: msg += " @ {}".format(self.pretty_timestamp())
          else:              msg += " *unknown timestamp*"
          return msg
  
      def kill (self):
          else:              msg += " *unknown timestamp*"
          return msg
  
      def kill (self):
-        command="rsync lxc-driver.sh  %s:/root"%self.plc_box.hostname
-       commands.getstatusoutput(command)
-       msg="lxc container stopping %s on %s"%(self.lxcname,self.plc_box.hostname)
-       self.plc_box.run_ssh(['/root/lxc-driver.sh','-c','stop_lxc','-n',self.lxcname],msg)
+        command="rsync lxc-driver.sh  {}:/root".format(self.plc_box.hostname)
+        subprocess.getstatusoutput(command)
+        msg="lxc container stopping {} on {}".format(self.lxcname, self.plc_box.hostname)
+        self.plc_box.run_ssh(['/root/lxc-driver.sh', '-c', 'stop_lxc', '-n', self.lxcname], msg)
          self.plc_box.forget(self)
  
  ##########
  class PlcBox (Box):
      def __init__ (self, hostname, max_plcs):
          self.plc_box.forget(self)
  
  ##########
  class PlcBox (Box):
      def __init__ (self, hostname, max_plcs):
-        Box.__init__(self,hostname)
-        self.plc_instances=[]
-        self.max_plcs=max_plcs
+        Box.__init__(self, hostname)
+        self.plc_instances = []
+        self.max_plcs = max_plcs
+    def __repr__(self):
+        return "<PlcBox {}>".format(self.hostname)
  
      def free_slots (self):
          return self.max_plcs - len(self.plc_instances)
  
      # fill one slot even though this one is not started yet
      def add_dummy (self, plcname):
  
      def free_slots (self):
          return self.max_plcs - len(self.plc_instances)
  
      # fill one slot even though this one is not started yet
      def add_dummy (self, plcname):
-        dummy=PlcLxcInstance(self,'dummy_'+plcname,0)
+        dummy=PlcLxcInstance(self, 'dummy_'+plcname, 0)
          dummy.set_now()
          self.plc_instances.append(dummy)
  
          dummy.set_now()
          self.plc_instances.append(dummy)
  
@@ -495,261 +553,303 @@ class PlcBox (Box):
  
      def reboot (self, options):
          if not options.soft:
  
      def reboot (self, options):
          if not options.soft:
-            Box.reboot(self,options)
+            Box.reboot(self, options)
          else:
              self.soft_reboot (options)
  
      def list(self, verbose=False):
          if not self.plc_instances: 
          else:
              self.soft_reboot (options)
  
      def list(self, verbose=False):
          if not self.plc_instances: 
-            header ('No plc running on %s'%(self.line()))
+            header ('No plc running on {}'.format(self.line()))
          else:
          else:
-            header ("Active plc VMs on %s"%self.line())
-            self.plc_instances.sort(timestamp_sort)
+            header ("Active plc VMs on {}".format(self.line()))
+            self.plc_instances.sort(key=timestamp_key)
              for p in self.plc_instances: 
              for p in self.plc_instances: 
-                header (p.line(),banner=False)
+                header (p.line(), banner=False)
  
  ## we do not this at INRIA any more
  class PlcLxcBox (PlcBox):
  
  
  ## we do not this at INRIA any more
  class PlcLxcBox (PlcBox):
  
-    def add_lxc (self,lxcname,pid):
+    def add_lxc (self, lxcname, pid):
          for plc in self.plc_instances:
          for plc in self.plc_instances:
-            if plc.lxcname==lxcname:
-                header("WARNING, duplicate myplc %s running on %s"%\
-                           (lxcname,self.hostname),banner=False)
+            if plc.lxcname == lxcname:
+                header("WARNING, duplicate myplc {} running on {}"\
+                       .format(lxcname, self.hostname), banner=False)
                  return
                  return
-        self.plc_instances.append(PlcLxcInstance(self,lxcname,pid))    
+        self.plc_instances.append(PlcLxcInstance(self, lxcname, pid))    
  
  
      # a line describing the box
      def line(self): 
  
  
      # a line describing the box
      def line(self): 
-        return "%s [max=%d,free=%d] (%s)"%(self.hostname_fedora(virt="lxc"), 
-                                           self.max_plcs,self.free_slots(),
-                                           self.uptime(),
-                                           )
+        return "{} [max={},free={}] ({})".format(self.hostname_fedora(virt="lxc"), 
+                                                 self.max_plcs, self.free_slots(),
+                                                 self.uptime())
      
      
-    def plc_instance_by_lxcname (self, lxcname):
+    def plc_instance_by_lxcname(self, lxcname):
          for p in self.plc_instances:
          for p in self.plc_instances:
-            if p.lxcname==lxcname: return p
+            if p.lxcname == lxcname:
+                return p
          return None
      
      # essentially shutdown all running containers
          return None
      
      # essentially shutdown all running containers
-    def soft_reboot (self, options):
-        command="rsync lxc-driver.sh  %s:/root"%self.hostname
-        commands.getstatusoutput(command)
-       self.run_ssh(['/root/lxc-driver.sh','-c','stop_all'],"Stopping all running lxc containers on %s"%(self.hostname,),
-                     dry_run=options.dry_run)
+    def soft_reboot(self, options):
+        command="rsync lxc-driver.sh  {}:/root".format(self.hostname)
+        subprocess.getstatusoutput(command)
+        self.run_ssh( ['/root/lxc-driver.sh','-c','stop_all'],
+                      "Stopping all running lxc containers on {}".format(self.hostname),
+                      dry_run=options.dry_run)
  
  
      # sense is expected to fill self.plc_instances with PlcLxcInstance's 
      # to describe the currently running VM's
  
  
      # sense is expected to fill self.plc_instances with PlcLxcInstance's 
      # to describe the currently running VM's
-    def sense (self, options):
-        print "xp",
-       command="rsync lxc-driver.sh  %s:/root"%self.hostname
-        commands.getstatusoutput(command)
-       command=['/root/lxc-driver.sh','-c','sense_all']
+    def sense(self, options):
+        print("xp", end=' ')
+        sys.stdout.flush()
+        command = "rsync lxc-driver.sh  {}:/root".format(self.hostname)
+        subprocess.getstatusoutput(command)
+        command = ['/root/lxc-driver.sh', '-c', 'sense_all']
          lxc_stat = self.backquote_ssh (command)
          lxc_stat = self.backquote_ssh (command)
-       for lxc_line in lxc_stat.split("\n"):
-            if not lxc_line: continue
-            lxcname=lxc_line.split(";")[0]
-           pid=lxc_line.split(";")[1]
-           timestamp=lxc_line.split(";")[2]
+        for lxc_line in lxc_stat.split("\n"):
+            if not lxc_line:
+                continue
+            # we mix build and plc VMs
+            if 'vplc' not in lxc_line:
+                continue
+            lxcname = lxc_line.split(";")[0]
+            pid = lxc_line.split(";")[1]
+            timestamp = lxc_line.split(";")[2]
              self.add_lxc(lxcname,pid)
              self.add_lxc(lxcname,pid)
-            try: timestamp=int(timestamp)
-            except: timestamp=0
-            p=self.plc_instance_by_lxcname(lxcname)
+            try: timestamp = int(timestamp)
+            except: timestamp = 0
+            p = self.plc_instance_by_lxcname(lxcname)
              if not p:
              if not p:
-                print 'WARNING zombie plc',self.hostname,lxcname
-                print '... was expecting',lxcname,'in',[i.lxcname for i in self.plc_instances]
+                print('WARNING zombie plc',self.hostname,lxcname)
+                print('... was expecting',lxcname,'in',[i.lxcname for i in self.plc_instances])
                  continue
              p.set_timestamp(timestamp)
  
  ############################################################
  class QemuInstance: 
                  continue
              p.set_timestamp(timestamp)
  
  ############################################################
  class QemuInstance: 
-    def __init__ (self, nodename, pid, qemubox):
-        self.nodename=nodename
-        self.pid=pid
-        self.qemu_box=qemubox
+    def __init__(self, nodename, pid, qemubox):
+        self.nodename = nodename
+        self.pid = pid
+        self.qemu_box = qemubox
          # not known yet
          # not known yet
-        self.buildname=None
-        self.timestamp=0
+        self.buildname = None
+        self.timestamp = 0
+    def __repr__(self):
+        return "<QemuInstance {}>".format(self.nodename)
          
          
-    def set_buildname (self,buildname): self.buildname=buildname
-    def set_timestamp (self,timestamp): self.timestamp=timestamp
-    def set_now (self): self.timestamp=int(time.time())
-    def pretty_timestamp (self): return time.strftime("%Y-%m-%d:%H-%M",time.localtime(self.timestamp))
+    def set_buildname (self, buildname):
+        self.buildname = buildname
+    def set_timestamp (self, timestamp):
+        self.timestamp = timestamp
+    def set_now (self):
+        self.timestamp = int(time.time())
+    def pretty_timestamp (self):
+        return time.strftime("%Y-%m-%d:%H-%M", time.localtime(self.timestamp))
      
      def line (self):
      
      def line (self):
-        msg = "== %s =="%(short_hostname(self.nodename))
-        msg += " [=%s]"%self.buildname
-        if self.pid:       msg += " (pid=%s)"%self.pid
+        msg = "== {} ==".format(short_hostname(self.nodename))
+        msg += " [={}]".format(self.buildname)
+        if self.pid:       msg += " (pid={})".format(self.pid)
          else:              msg += " not (yet?) running"
          else:              msg += " not (yet?) running"
-        if self.timestamp: msg += " @ %s"%self.pretty_timestamp()
+        if self.timestamp: msg += " @ {}".format(self.pretty_timestamp())
          else:              msg += " *unknown timestamp*"
          return msg
      
      def kill(self):
          else:              msg += " *unknown timestamp*"
          return msg
      
      def kill(self):
-        if self.pid==0: 
-            print "cannot kill qemu %s with pid==0"%self.nodename
+        if self.pid == 0: 
+            print("cannot kill qemu {} with pid==0".format(self.nodename))
              return
              return
-        msg="Killing qemu %s with pid=%s on box %s"%(self.nodename,self.pid,self.qemu_box.hostname)
-        self.qemu_box.run_ssh(['kill',"%s"%self.pid],msg)
+        msg = "Killing qemu {} with pid={} on box {}".format(self.nodename, self.pid, self.qemu_box.hostname)
+        self.qemu_box.run_ssh(['kill', "{}".format(self.pid)], msg)
          self.qemu_box.forget(self)
  
  
  class QemuBox (Box):
      def __init__ (self, hostname, max_qemus):
          self.qemu_box.forget(self)
  
  
  class QemuBox (Box):
      def __init__ (self, hostname, max_qemus):
-        Box.__init__(self,hostname)
-        self.qemu_instances=[]
-        self.max_qemus=max_qemus
+        Box.__init__(self, hostname)
+        self.qemu_instances = []
+        self.max_qemus = max_qemus
+    def __repr__(self):
+        return "<QemuBox {}>".format(self.hostname)
  
  
-    def add_node (self,nodename,pid):
+    def add_node(self, nodename, pid):
          for qemu in self.qemu_instances:
          for qemu in self.qemu_instances:
-            if qemu.nodename==nodename: 
-                header("WARNING, duplicate qemu %s running on %s"%\
-                           (nodename,self.hostname), banner=False)
+            if qemu.nodename == nodename: 
+                header("WARNING, duplicate qemu {} running on {}"\
+                       .format(nodename,self.hostname), banner=False)
                  return
                  return
-        self.qemu_instances.append(QemuInstance(nodename,pid,self))
+        self.qemu_instances.append(QemuInstance(nodename, pid, self))
+
+    def node_names (self):
+        return [ qi.nodename for qi in self.qemu_instances ]
  
      def forget (self, qemu_instance):
          self.qemu_instances.remove(qemu_instance)
  
      # fill one slot even though this one is not started yet
  
      def forget (self, qemu_instance):
          self.qemu_instances.remove(qemu_instance)
  
      # fill one slot even though this one is not started yet
-    def add_dummy (self, nodename):
-        dummy=QemuInstance('dummy_'+nodename,0,self)
+    def add_dummy(self, nodename):
+        dummy=QemuInstance('dummy_'+nodename, 0, self)
          dummy.set_now()
          self.qemu_instances.append(dummy)
  
      def line (self):
          dummy.set_now()
          self.qemu_instances.append(dummy)
  
      def line (self):
-        return "%s [max=%d,free=%d] (%s) %s"%(
-            self.hostname_fedora(virt="qemu"), self.max_qemus,self.free_slots(),
-            self.uptime(),self.driver())
+        return "{} [max={},free={}] ({}) {}"\
+            .format(self.hostname_fedora(virt="qemu"),
+                    self.max_qemus, self.free_slots(),
+                    self.uptime(), self.driver())
  
      def list(self, verbose=False):
          if not self.qemu_instances: 
  
      def list(self, verbose=False):
          if not self.qemu_instances: 
-            header ('No qemu on %s'%(self.line()))
+            header ('No qemu on {}'.format(self.line()))
          else:
          else:
-            header ("Qemus on %s"%(self.line()))
-            self.qemu_instances.sort(timestamp_sort)
+            header ("Qemus on {}".format(self.line()))
+            self.qemu_instances.sort(key=timestamp_key)
              for q in self.qemu_instances: 
              for q in self.qemu_instances: 
-                header (q.line(),banner=False)
+                header (q.line(), banner=False)
  
      def free_slots (self):
          return self.max_qemus - len(self.qemu_instances)
  
      def driver(self):
  
      def free_slots (self):
          return self.max_qemus - len(self.qemu_instances)
  
      def driver(self):
-        if hasattr(self,'_driver') and self._driver: return self._driver
+        if hasattr(self,'_driver') and self._driver:
+            return self._driver
          return '*undef* driver'
  
          return '*undef* driver'
  
-    def qemu_instance_by_pid (self,pid):
+    def qemu_instance_by_pid(self, pid):
          for q in self.qemu_instances:
          for q in self.qemu_instances:
-            if q.pid==pid: return q
+            if q.pid == pid:
+                return q
          return None
  
          return None
  
-    def qemu_instance_by_nodename_buildname (self,nodename,buildname):
+    def qemu_instance_by_nodename_buildname (self, nodename, buildname):
          for q in self.qemu_instances:
          for q in self.qemu_instances:
-            if q.nodename==nodename and q.buildname==buildname:
+            if q.nodename == nodename and q.buildname == buildname:
                  return q
          return None
  
      def reboot (self, options):
          if not options.soft:
                  return q
          return None
  
      def reboot (self, options):
          if not options.soft:
-            Box.reboot(self,options)
+            Box.reboot(self, options)
          else:
          else:
-            self.run_ssh(['pkill','qemu'],"Killing qemu instances",
+            self.run_ssh(['pkill','qemu'], "Killing qemu instances",
                           dry_run=options.dry_run)
  
      matcher=re.compile("\s*(?P<pid>[0-9]+).*-cdrom\s+(?P<nodename>[^\s]+)\.iso")
                           dry_run=options.dry_run)
  
      matcher=re.compile("\s*(?P<pid>[0-9]+).*-cdrom\s+(?P<nodename>[^\s]+)\.iso")
+
      def sense(self, options):
      def sense(self, options):
-        print 'qn',
-        modules=self.backquote_ssh(['lsmod']).split('\n')
-        self._driver='*NO kqemu/kvm_intel MODULE LOADED*'
+        print('qn', end=' ')
+        sys.stdout.flush()
+        modules = self.backquote_ssh(['lsmod']).split('\n')
+        self._driver = '*NO kqemu/kvm_intel MODULE LOADED*'
          for module in modules:
          for module in modules:
-            if module.find('kqemu')==0:
-                self._driver='kqemu module loaded'
+            if module.find('kqemu') == 0:
+                self._driver = 'kqemu module loaded'
              # kvm might be loaded without kvm_intel (we dont have AMD)
              # kvm might be loaded without kvm_intel (we dont have AMD)
-            elif module.find('kvm_intel')==0:
-                self._driver='kvm_intel OK'
+            elif module.find('kvm_intel') == 0:
+                self._driver = 'kvm_intel OK'
          ########## find out running pids
          ########## find out running pids
-        pids=self.backquote_ssh(['pgrep','qemu'])
-        if not pids: return
-        command=['ps','-o','pid,command'] + [ pid for pid in pids.split("\n") if pid]
-        ps_lines = self.backquote_ssh (command).split("\n")
+        pids = self.backquote_ssh(['pgrep','qemu'])
+        if not pids:
+            return
+        command = ['ps','-o','pid,command'] + [ pid for pid in pids.split("\n") if pid]
+        ps_lines = self.backquote_ssh(command).split("\n")
          for line in ps_lines:
          for line in ps_lines:
-            if not line.strip() or line.find('PID') >=0 : continue
-            m=QemuBox.matcher.match(line)
+            if not line.strip() or line.find('PID') >=0 :
+                continue
+            m = QemuBox.matcher.match(line)
              if m: 
              if m: 
-                self.add_node (m.group('nodename'),m.group('pid'))
+                self.add_node(m.group('nodename'), m.group('pid'))
                  continue
                  continue
-            header('QemuBox.sense: command %r returned line that failed to match'%command)
-            header(">>%s<<"%line)
+            header('QemuBox.sense: command {} returned line that failed to match'.format(command))
+            header(">>{}<<".format(line))
          ########## retrieve alive instances and map to build
          live_builds=[]
          ########## retrieve alive instances and map to build
          live_builds=[]
-        command=['grep','.','/vservers/*/*/qemu.pid','/dev/null']
-        pid_lines=self.backquote_ssh(command,trash_err=True).split('\n')
+        command = ['grep', '.', '/vservers/*/*/qemu.pid', '/dev/null']
+        pid_lines = self.backquote_ssh(command, trash_err=True).split('\n')
          for pid_line in pid_lines:
          for pid_line in pid_lines:
-            if not pid_line.strip(): continue
+            if not pid_line.strip():
+                continue
              # expect <build>/<nodename>/qemu.pid:<pid>pid
              try:
              # expect <build>/<nodename>/qemu.pid:<pid>pid
              try:
-                (_,__,buildname,nodename,tail)=pid_line.split('/')
-                (_,pid)=tail.split(':')
-                q=self.qemu_instance_by_pid (pid)
-                if not q: continue
+                (_, __, buildname, nodename, tail) = pid_line.split('/')
+                (_,pid) = tail.split(':')
+                q = self.qemu_instance_by_pid(pid)
+                if not q:
+                    continue
                  q.set_buildname(buildname)
                  live_builds.append(buildname)
                  q.set_buildname(buildname)
                  live_builds.append(buildname)
-            except: print 'WARNING, could not parse pid line',pid_line
+            except:
+                print('WARNING, could not parse pid line', pid_line)
          # retrieve timestamps
          # retrieve timestamps
-        if not live_builds: return
-        command=   ['grep','.']
-        command += ['/vservers/%s/*/timestamp'%b for b in live_builds]
+        if not live_builds:
+            return
+        command =  ['grep','.']
+        command += ['/vservers/{}/*/timestamp'.format(b) for b in live_builds]
          command += ['/dev/null']
          command += ['/dev/null']
-        ts_lines=self.backquote_ssh(command,trash_err=True).split('\n')
+        ts_lines = self.backquote_ssh(command, trash_err=True).split('\n')
          for ts_line in ts_lines:
          for ts_line in ts_lines:
-            if not ts_line.strip(): continue
+            if not ts_line.strip():
+                continue
              # expect <build>/<nodename>/timestamp:<timestamp>
              try:
              # expect <build>/<nodename>/timestamp:<timestamp>
              try:
-                (_,__,buildname,nodename,tail)=ts_line.split('/')
-                nodename=nodename.replace('qemu-','')
-                (_,timestamp)=tail.split(':')
-                timestamp=int(timestamp)
-                q=self.qemu_instance_by_nodename_buildname(nodename,buildname)
+                (_, __, buildname, nodename, tail) = ts_line.split('/')
+                nodename = nodename.replace('qemu-', '')
+                (_, timestamp) = tail.split(':')
+                timestamp = int(timestamp)
+                q = self.qemu_instance_by_nodename_buildname(nodename, buildname)
                  if not q: 
                  if not q: 
-                    print 'WARNING zombie qemu',self.hostname,ts_line
-                    print '... was expecting (',short_hostname(nodename),buildname,') in',\
-                        [ (short_hostname(i.nodename),i.buildname) for i in self.qemu_instances ]
+                    # this warning corresponds to qemu instances that were not killed properly 
+                    # and that have a dangling qemu.pid - and not even all of them as they need
+                    # to be attached to a build that has a node running...
+                    # it is more confusing than helpful, so let's just trash it
+                    #print 'WARNING zombie qemu',self.hostname,ts_line
+                    #print '... was expecting (',short_hostname(nodename),buildname,') in',\
+                    #    [ (short_hostname(i.nodename),i.buildname) for i in self.qemu_instances ]
                      continue
                  q.set_timestamp(timestamp)
                      continue
                  q.set_timestamp(timestamp)
-            except:  print 'WARNING, could not parse ts line',ts_line
+            except:
+                print('WARNING, could not parse ts line',ts_line)
  
  ####################
  class TestInstance:
  
  ####################
  class TestInstance:
-    def __init__ (self, buildname, pid=0):
-        self.pids=[]
-        if pid!=0: self.pid.append(pid)
-        self.buildname=buildname
+    def __init__(self, buildname, pid=0):
+        self.pids = []
+        if pid != 0:
+            self.pid.append(pid)
+        self.buildname = buildname
          # latest trace line
          # latest trace line
-        self.trace=''
+        self.trace = ''
          # has a KO test
          # has a KO test
-        self.broken_steps=[]
+        self.broken_steps = []
          self.timestamp = 0
          self.timestamp = 0
-
-    def set_timestamp (self,timestamp): self.timestamp=timestamp
-    def set_now (self): self.timestamp=int(time.time())
-    def pretty_timestamp (self): return time.strftime("%Y-%m-%d:%H-%M",time.localtime(self.timestamp))
-
-    def is_running (self): return len(self.pids) != 0
-
-    def add_pid (self,pid):
+    def __repr__(self):
+        return "<TestInstance {}>".format(self.buildname)
+
+    def set_timestamp(self, timestamp):
+        self.timestamp = timestamp
+    def set_now(self):
+        self.timestamp = int(time.time())
+    def pretty_timestamp(self):
+        return time.strftime("%Y-%m-%d:%H-%M", time.localtime(self.timestamp))
+    def is_running (self):
+        return len(self.pids) != 0
+    def add_pid(self, pid):
          self.pids.append(pid)
          self.pids.append(pid)
-    def set_broken (self, plcindex, step): 
-        self.broken_steps.append ( (plcindex, step,) )
+    def set_broken(self, plcindex, step): 
+        self.broken_steps.append( (plcindex, step,) )
  
  
-    def second_letter (self):
-        if not self.broken_steps: return '='
+    def second_letter(self):
+        if not self.broken_steps:
+            return '='
          else:
              really_broken = [ step for (i,step) in self.broken_steps if '_ignore' not in step ]
              # W is for warning like what's in the build mail
          else:
              really_broken = [ step for (i,step) in self.broken_steps if '_ignore' not in step ]
              # W is for warning like what's in the build mail
-            if len(really_broken)==0: return 'W'
-            else: return 'B'
+            if len(really_broken) == 0:
+                return 'W'
+            else:
+                return 'B'
  
      def line (self):
          # make up a 2-letter sign
  
      def line (self):
          # make up a 2-letter sign
@@ -758,120 +858,134 @@ class TestInstance:
          # second letter : '=' if fine, 'W' for warnings (only ignored steps) 'B' for broken
          letter2 = self.second_letter()
          double += letter2
          # second letter : '=' if fine, 'W' for warnings (only ignored steps) 'B' for broken
          letter2 = self.second_letter()
          double += letter2
-        msg = " %s %s =="%(double,self.buildname)
-        if not self.pids:       pass
-        elif len(self.pids)==1: msg += " (pid=%s)"%self.pids[0]
-        else:                   msg += " !!!pids=%s!!!"%self.pids
-        msg += " @%s"%self.pretty_timestamp()
+        msg = " {} {} ==".format(double, self.buildname)
+        if not self.pids:
+            pass
+        elif len(self.pids)==1:
+            msg += " (pid={})".format(self.pids[0])
+        else:
+            msg += " !!!pids={}!!!".format(self.pids)
+        msg += " @{}".format(self.pretty_timestamp())
          if letter2 != '=':
              msg2 = ( ' BROKEN' if letter2 == 'B' else ' WARNING' )
              # sometimes we have an empty plcindex
          if letter2 != '=':
              msg2 = ( ' BROKEN' if letter2 == 'B' else ' WARNING' )
              # sometimes we have an empty plcindex
-            msg += " [%s="%msg2 + " ".join( [ "%s@%s"%(s,i) if i else s for (i,s) in self.broken_steps ] ) + "]"
+            msg += " [{}=".format(msg2) \
+                   + " ".join(["{}@{}".format(s, i) if i else s for (i, s) in self.broken_steps]) \
+                   + "]"
          return msg
  
          return msg
  
-class TestBox (Box):
-    def __init__ (self,hostname):
-        Box.__init__(self,hostname)
-        self.starting_ips=[]
-        self.test_instances=[]
+class TestBox(Box):
+    def __init__(self, hostname):
+        Box.__init__(self, hostname)
+        self.starting_ips = []
+        self.test_instances = []
+    def __repr__(self):
+        return "<TestBox {}>".format(self.hostname)
  
  
-    def reboot (self, options):
+    def reboot(self, options):
          # can't reboot a vserver VM
          # can't reboot a vserver VM
-        self.run_ssh (['pkill','run_log'],"Terminating current runs",
+        self.run_ssh(['pkill', 'run_log'], "Terminating current runs",
                        dry_run=options.dry_run)
                        dry_run=options.dry_run)
-        self.run_ssh (['rm','-f',Starting.location],"Cleaning %s"%Starting.location,
+        self.run_ssh(['rm', '-f', Starting.location], "Cleaning {}".format(Starting.location),
                        dry_run=options.dry_run)
  
                        dry_run=options.dry_run)
  
-    def get_test (self, buildname):
+    def get_test(self, buildname):
          for i in self.test_instances:
          for i in self.test_instances:
-            if i.buildname==buildname: return i
+            if i.buildname == buildname:
+                return i
  
      # we scan ALL remaining test results, even the ones not running
  
      # we scan ALL remaining test results, even the ones not running
-    def add_timestamp (self, buildname, timestamp):
-        i=self.get_test(buildname)
+    def add_timestamp(self, buildname, timestamp):
+        i = self.get_test(buildname)
          if i:   
              i.set_timestamp(timestamp)
          else:   
          if i:   
              i.set_timestamp(timestamp)
          else:   
-            i=TestInstance(buildname,0)
+            i = TestInstance(buildname, 0)
              i.set_timestamp(timestamp)
              self.test_instances.append(i)
  
              i.set_timestamp(timestamp)
              self.test_instances.append(i)
  
-    def add_running_test (self, pid, buildname):
-        i=self.get_test(buildname)
+    def add_running_test(self, pid, buildname):
+        i = self.get_test(buildname)
          if not i:
          if not i:
-            self.test_instances.append (TestInstance (buildname,pid))
+            self.test_instances.append(TestInstance(buildname, pid))
              return
          if i.pids:
              return
          if i.pids:
-            print "WARNING: 2 concurrent tests run on same build %s"%buildname
-        i.add_pid (pid)
+            print("WARNING: 2 concurrent tests run on same build {}".format(buildname))
+        i.add_pid(pid)
  
  
-    def add_broken (self, buildname, plcindex, step):
-        i=self.get_test(buildname)
+    def add_broken(self, buildname, plcindex, step):
+        i = self.get_test(buildname)
          if not i:
          if not i:
-            i=TestInstance(buildname)
+            i = TestInstance(buildname)
              self.test_instances.append(i)
          i.set_broken(plcindex, step)
  
      matcher_proc=re.compile (".*/proc/(?P<pid>[0-9]+)/cwd.*/root/(?P<buildname>[^/]+)$")
      matcher_grep=re.compile ("/root/(?P<buildname>[^/]+)/logs/trace.*:TRACE:\s*(?P<plcindex>[0-9]+).*step=(?P<step>\S+).*")
      matcher_grep_missing=re.compile ("grep: /root/(?P<buildname>[^/]+)/logs/trace: No such file or directory")
              self.test_instances.append(i)
          i.set_broken(plcindex, step)
  
      matcher_proc=re.compile (".*/proc/(?P<pid>[0-9]+)/cwd.*/root/(?P<buildname>[^/]+)$")
      matcher_grep=re.compile ("/root/(?P<buildname>[^/]+)/logs/trace.*:TRACE:\s*(?P<plcindex>[0-9]+).*step=(?P<step>\S+).*")
      matcher_grep_missing=re.compile ("grep: /root/(?P<buildname>[^/]+)/logs/trace: No such file or directory")
-    def sense (self, options):
-        print 'tm',
-        self.starting_ips=[x for x in self.backquote_ssh(['cat',Starting.location], trash_err=True).strip().split('\n') if x]
+
+    def sense(self, options):
+        print('tm', end=' ')
+        self.starting_ips = [ x for x in self.backquote_ssh( ['cat', Starting.location], trash_err=True).strip().split('\n') if x ]
  
          # scan timestamps on all tests
          # this is likely to not invoke ssh so we need to be a bit smarter to get * expanded
          # xxx would make sense above too
  
          # scan timestamps on all tests
          # this is likely to not invoke ssh so we need to be a bit smarter to get * expanded
          # xxx would make sense above too
-        command=['bash','-c',"grep . /root/*/timestamp /dev/null"]
-        ts_lines=self.backquote_ssh(command,trash_err=True).split('\n')
+        command = ['bash', '-c', "grep . /root/*/timestamp /dev/null"]
+        ts_lines = self.backquote_ssh(command, trash_err=True).split('\n')
          for ts_line in ts_lines:
          for ts_line in ts_lines:
-            if not ts_line.strip(): continue
+            if not ts_line.strip():
+                continue
              # expect /root/<buildname>/timestamp:<timestamp>
              try:
              # expect /root/<buildname>/timestamp:<timestamp>
              try:
-                (ts_file,timestamp)=ts_line.split(':')
-                ts_file=os.path.dirname(ts_file)
-                buildname=os.path.basename(ts_file)
-                timestamp=int(timestamp)
-                t=self.add_timestamp(buildname,timestamp)
-            except:  print 'WARNING, could not parse ts line',ts_line
+                (ts_file, timestamp) = ts_line.split(':')
+                ts_file = os.path.dirname(ts_file)
+                buildname = os.path.basename(ts_file)
+                timestamp = int(timestamp)
+                t = self.add_timestamp(buildname, timestamp)
+            except:
+                print('WARNING, could not parse ts line', ts_line)
  
          # let's try to be robust here -- tests that fail very early like e.g.
          # "Cannot make space for a PLC instance: vplc IP pool exhausted", that occurs as part of provision
          # will result in a 'trace' symlink to an inexisting 'trace-<>.txt' because no step has gone through
          # simple 'trace' should exist though as it is created by run_log
  
          # let's try to be robust here -- tests that fail very early like e.g.
          # "Cannot make space for a PLC instance: vplc IP pool exhausted", that occurs as part of provision
          # will result in a 'trace' symlink to an inexisting 'trace-<>.txt' because no step has gone through
          # simple 'trace' should exist though as it is created by run_log
-        command=['bash','-c',"grep KO /root/*/logs/trace /dev/null 2>&1" ]
-        trace_lines=self.backquote_ssh (command).split('\n')
+        command = ['bash', '-c', "grep KO /root/*/logs/trace /dev/null 2>&1" ]
+        trace_lines = self.backquote_ssh(command).split('\n')
          for line in trace_lines:
          for line in trace_lines:
-            if not line.strip(): continue
-            m=TestBox.matcher_grep_missing.match(line)
+            if not line.strip():
+                continue
+            m = TestBox.matcher_grep_missing.match(line)
              if m:
              if m:
-                buildname=m.group('buildname')
-                self.add_broken(buildname,'','NO STEP DONE')
+                buildname = m.group('buildname')
+                self.add_broken(buildname, '', 'NO STEP DONE')
                  continue
                  continue
-            m=TestBox.matcher_grep.match(line)
+            m = TestBox.matcher_grep.match(line)
              if m: 
              if m: 
-                buildname=m.group('buildname')
-                plcindex=m.group('plcindex')
-                step=m.group('step')
-                self.add_broken(buildname,plcindex, step)
+                buildname = m.group('buildname')
+                plcindex = m.group('plcindex')
+                step = m.group('step')
+                self.add_broken(buildname, plcindex, step)
                  continue
                  continue
-            header("TestBox.sense: command %r returned line that failed to match\n%s"%(command,line))
-            header(">>%s<<"%line)
+            header("TestBox.sense: command {} returned line that failed to match\n{}".format(command, line))
+            header(">>{}<<".format(line))
  
  
-        pids = self.backquote_ssh (['pgrep','run_log'],trash_err=True)
-        if not pids: return
-        command=['ls','-ld'] + ["/proc/%s/cwd"%pid for pid in pids.split("\n") if pid]
-        ps_lines=self.backquote_ssh (command).split('\n')
+        pids = self.backquote_ssh (['pgrep', 'run_log'], trash_err=True)
+        if not pids:
+            return
+        command = ['ls','-ld'] + ["/proc/{}/cwd".format(pid) for pid in pids.split("\n") if pid]
+        ps_lines = self.backquote_ssh(command).split('\n')
          for line in ps_lines:
          for line in ps_lines:
-            if not line.strip(): continue
-            m=TestBox.matcher_proc.match(line)
+            if not line.strip():
+                continue
+            m = TestBox.matcher_proc.match(line)
              if m: 
              if m: 
-                pid=m.group('pid')
-                buildname=m.group('buildname')
+                pid = m.group('pid')
+                buildname = m.group('buildname')
                  self.add_running_test(pid, buildname)
                  continue
                  self.add_running_test(pid, buildname)
                  continue
-            header("TestBox.sense: command %r returned line that failed to match\n%s"%(command,line))
-            header(">>%s<<"%line)
+            header("TestBox.sense: command {} returned line that failed to match\n{}".format(command, line))
+            header(">>{}<<".format(line))
          
          
      def line (self):
          
          
      def line (self):
@@ -887,110 +1001,119 @@ class TestBox (Box):
              msg="running tests"
  
          if not instances:
              msg="running tests"
  
          if not instances:
-            header ("No %s on %s"%(msg,self.line()))
+            header ("No {} on {}".format(msg, self.line()))
          else:
          else:
-            header ("%s on %s"%(msg,self.line()))
-            instances.sort(timestamp_sort)
-            for i in instances: print i.line()
+            header ("{} on {}".format(msg, self.line()))
+            instances.sort(key=timestamp_key)
+            for i in instances:
+                print(i.line())
          # show 'starting' regardless of verbose
          if self.starting_ips:
          # show 'starting' regardless of verbose
          if self.starting_ips:
-            header ("Starting IP addresses on %s"%self.line())
+            header("Starting IP addresses on {}".format(self.line()))
              self.starting_ips.sort()
              self.starting_ips.sort()
-            for starting in self.starting_ips: print starting
+            for starting in self.starting_ips:
+                print(starting)
          else:
          else:
-            header ("Empty 'starting' on %s"%self.line())
+            header("Empty 'starting' on {}".format(self.line()))
  
  ############################################################
  class Options: pass
  
  class Substrate:
  
  
  ############################################################
  class Options: pass
  
  class Substrate:
  
-    def __init__ (self):
-        self.options=Options()
-        self.options.dry_run=False
-        self.options.verbose=False
-        self.options.reboot=False
-        self.options.soft=False
+    def __init__(self):
+        self.options = Options()
+        self.options.dry_run = False
+        self.options.verbose = False
+        self.options.reboot = False
+        self.options.soft = False
          self.test_box = TestBox (self.test_box_spec())
          self.build_lxc_boxes = [ BuildLxcBox(h) for h in self.build_lxc_boxes_spec() ]
          self.test_box = TestBox (self.test_box_spec())
          self.build_lxc_boxes = [ BuildLxcBox(h) for h in self.build_lxc_boxes_spec() ]
-        self.plc_lxc_boxes = [ PlcLxcBox (h,m) for (h,m) in self.plc_lxc_boxes_spec ()]
-        self.qemu_boxes = [ QemuBox (h,m) for (h,m) in self.qemu_boxes_spec ()]
-        self._sensed=False
+        self.plc_lxc_boxes = [ PlcLxcBox (h, m) for (h, m) in self.plc_lxc_boxes_spec ()]
+        self.qemu_boxes = [ QemuBox (h, m) for (h, m) in self.qemu_boxes_spec ()]
+        self._sensed = False
  
  
-        self.vplc_pool = Pool (self.vplc_ips(),"for vplcs",self)
-        self.vnode_pool = Pool (self.vnode_ips(),"for vnodes",self)
+        self.vplc_pool = Pool(self.vplc_ips(), "for vplcs", self)
+        self.vnode_pool = Pool(self.vnode_ips(), "for vnodes", self)
          
          self.build_boxes = self.build_lxc_boxes
          self.plc_boxes = self.plc_lxc_boxes
          self.default_boxes = self.plc_boxes + self.qemu_boxes
          self.all_boxes = self.build_boxes + [ self.test_box ] + self.plc_boxes + self.qemu_boxes
          
          self.build_boxes = self.build_lxc_boxes
          self.plc_boxes = self.plc_lxc_boxes
          self.default_boxes = self.plc_boxes + self.qemu_boxes
          self.all_boxes = self.build_boxes + [ self.test_box ] + self.plc_boxes + self.qemu_boxes
+    def __repr__(self):
+        return "<Substrate {}>".format(self.summary_line())
  
      def summary_line (self):
          msg  = "["
  
      def summary_line (self):
          msg  = "["
-        msg += " %d xp"%len(self.plc_lxc_boxes)
-        msg += " %d tried plc boxes"%len(self.plc_boxes)
+        msg += " {} xp".format(len(self.plc_lxc_boxes))
+        msg += " {} xq".format(len(self.qemu_boxes))
          msg += "]"
          return msg
  
      def fqdn (self, hostname):
          msg += "]"
          return msg
  
      def fqdn (self, hostname):
-        if hostname.find('.')<0: return "%s.%s"%(hostname,self.domain())
+        if hostname.find('.') < 0:
+            return "{}.{}".format(hostname, self.domain())
          return hostname
  
      # return True if actual sensing takes place
          return hostname
  
      # return True if actual sensing takes place
-    def sense (self,force=False):
-        if self._sensed and not force: return False
-        print 'Sensing local substrate...',
-        for b in self.default_boxes: b.sense(self.options)
-        print 'Done'
-        self._sensed=True
+    def sense(self, force=False):
+        if self._sensed and not force:
+            return False
+        print('Sensing local substrate...', end=' ')
+        sys.stdout.flush()
+        for b in self.default_boxes:
+            b.sense(self.options)
+        print('Done')
+        self._sensed = True
          return True
  
          return True
  
-    def list (self, verbose=False):
+    def list(self, verbose=False):
          for b in self.default_boxes:
              b.list()
  
          for b in self.default_boxes:
              b.list()
  
-    def add_dummy_plc (self, plc_boxname, plcname):
+    def add_dummy_plc(self, plc_boxname, plcname):
          for pb in self.plc_boxes:
          for pb in self.plc_boxes:
-            if pb.hostname==plc_boxname:
+            if pb.hostname == plc_boxname:
                  pb.add_dummy(plcname)
                  return True
                  pb.add_dummy(plcname)
                  return True
-    def add_dummy_qemu (self, qemu_boxname, qemuname):
+    def add_dummy_qemu(self, qemu_boxname, qemuname):
          for qb in self.qemu_boxes:
          for qb in self.qemu_boxes:
-            if qb.hostname==qemu_boxname:
+            if qb.hostname == qemu_boxname:
                  qb.add_dummy(qemuname)
                  return True
  
                  qb.add_dummy(qemuname)
                  return True
  
-    def add_starting_dummy (self, bname, vname):
-        return self.add_dummy_plc (bname, vname) or self.add_dummy_qemu (bname, vname)
+    def add_starting_dummy(self, bname, vname):
+        return self.add_dummy_plc(bname, vname) or self.add_dummy_qemu(bname, vname)
  
      ########## 
  
      ########## 
-    def provision (self,plcs,options):
+    def provision(self, plcs, options):
          try:
              # attach each plc to a plc box and an IP address
          try:
              # attach each plc to a plc box and an IP address
-            plcs = [ self.provision_plc (plc,options) for plc in plcs ]
+            plcs = [ self.provision_plc(plc, options) for plc in plcs ]
              # attach each node/qemu to a qemu box with an IP address
              # attach each node/qemu to a qemu box with an IP address
-            plcs = [ self.provision_qemus (plc,options) for plc in plcs ]
+            plcs = [ self.provision_qemus(plc,options) for plc in plcs ]
              # update the SFA spec accordingly
              # update the SFA spec accordingly
-            plcs = [ self.localize_sfa_rspec(plc,options) for plc in plcs ]
+            plcs = [ self.localize_sfa_rspec(plc, options) for plc in plcs ]
              self.list()
              return plcs
              self.list()
              return plcs
-        except Exception, e:
-            print '* Could not provision this test on current substrate','--',e,'--','exiting'
+        except Exception as e:
+            print('* Could not provision this test on current substrate','--',e,'--','exiting')
              traceback.print_exc()
              sys.exit(1)
  
      # it is expected that a couple of options like ips_bplc and ips_vplc 
      # are set or unset together
      @staticmethod
              traceback.print_exc()
              sys.exit(1)
  
      # it is expected that a couple of options like ips_bplc and ips_vplc 
      # are set or unset together
      @staticmethod
-    def check_options (x,y):
-        if not x and not y: return True
-        return len(x)==len(y)
+    def check_options(x, y):
+        if not x and not y:
+            return True
+        return len(x) == len(y)
  
      # find an available plc box (or make space)
      # and a free IP address (using options if present)
  
      # find an available plc box (or make space)
      # and a free IP address (using options if present)
-    def provision_plc (self, plc, options):
+    def provision_plc(self, plc, options):
          
          
-        assert Substrate.check_options (options.ips_bplc, options.ips_vplc)
+        assert Substrate.check_options(options.ips_bplc, options.ips_vplc)
  
          #### let's find an IP address for that plc
          # look in options 
  
          #### let's find an IP address for that plc
          # look in options 
@@ -999,236 +1122,276 @@ class Substrate:
              # we don't check anything here, 
              # it is the caller's responsability to cleanup and make sure this makes sense
              plc_boxname = options.ips_bplc.pop()
              # we don't check anything here, 
              # it is the caller's responsability to cleanup and make sure this makes sense
              plc_boxname = options.ips_bplc.pop()
-            vplc_hostname=options.ips_vplc.pop()
+            vplc_hostname = options.ips_vplc.pop()
          else:
          else:
-            if self.sense(): self.list()
-            plc_boxname=None
-            vplc_hostname=None
+            if self.sense():
+                self.list()
+            plc_boxname = None
+            vplc_hostname = None
              # try to find an available IP 
              self.vplc_pool.sense()
              # try to find an available IP 
              self.vplc_pool.sense()
-            couple=self.vplc_pool.next_free()
+            couple = self.vplc_pool.next_free()
              if couple:
              if couple:
-                (vplc_hostname,unused)=couple
+                (vplc_hostname, unused) = couple
              #### we need to find one plc box that still has a slot
              #### we need to find one plc box that still has a slot
-            max_free=0
+            max_free = 0
              # use the box that has max free spots for load balancing
              for pb in self.plc_boxes:
              # use the box that has max free spots for load balancing
              for pb in self.plc_boxes:
-                free=pb.free_slots()
-                if free>max_free:
-                    plc_boxname=pb.hostname
-                    max_free=free
+                free = pb.free_slots()
+                if free > max_free:
+                    plc_boxname = pb.hostname
+                    max_free = free
              # if there's no available slot in the plc_boxes, or we need a free IP address
              # make space by killing the oldest running instance
              if not plc_boxname or not vplc_hostname:
                  # find the oldest of all our instances
              # if there's no available slot in the plc_boxes, or we need a free IP address
              # make space by killing the oldest running instance
              if not plc_boxname or not vplc_hostname:
                  # find the oldest of all our instances
-                all_plc_instances=reduce(lambda x, y: x+y, 
-                                         [ pb.plc_instances for pb in self.plc_boxes ],
-                                         [])
-                all_plc_instances.sort(timestamp_sort)
+                all_plc_instances = reduce(lambda x, y: x+y, 
+                                           [ pb.plc_instances for pb in self.plc_boxes ],
+                                           [])
+                all_plc_instances.sort(key=timestamp_key)
                  try:
                  try:
-                    plc_instance_to_kill=all_plc_instances[0]
+                    plc_instance_to_kill = all_plc_instances[0]
                  except:
                  except:
-                    msg=""
-                    if not plc_boxname: msg += " PLC boxes are full"
-                    if not vplc_hostname: msg += " vplc IP pool exhausted"
-                    msg += " %s"%self.summary_line()
-                    raise Exception,"Cannot make space for a PLC instance:"+msg
-                freed_plc_boxname=plc_instance_to_kill.plc_box.hostname
-                freed_vplc_hostname=plc_instance_to_kill.vplcname()
-                message='killing oldest plc instance = %s on %s'%(plc_instance_to_kill.line(),
-                                                                  freed_plc_boxname)
+                    msg = ""
+                    if not plc_boxname:
+                        msg += " PLC boxes are full"
+                    if not vplc_hostname:
+                        msg += " vplc IP pool exhausted"
+                    msg += " {}".format(self.summary_line())
+                    raise Exception("Cannot make space for a PLC instance:" + msg)
+                freed_plc_boxname = plc_instance_to_kill.plc_box.hostname
+                freed_vplc_hostname = plc_instance_to_kill.vplcname()
+                message = 'killing oldest plc instance = {} on {}'\
+                    .format(plc_instance_to_kill.line(), freed_plc_boxname)
                  plc_instance_to_kill.kill()
                  # use this new plcbox if that was the problem
                  if not plc_boxname:
                  plc_instance_to_kill.kill()
                  # use this new plcbox if that was the problem
                  if not plc_boxname:
-                    plc_boxname=freed_plc_boxname
+                    plc_boxname = freed_plc_boxname
                  # ditto for the IP address
                  if not vplc_hostname:
                  # ditto for the IP address
                  if not vplc_hostname:
-                    vplc_hostname=freed_vplc_hostname
+                    vplc_hostname = freed_vplc_hostname
                      # record in pool as mine
                      self.vplc_pool.set_mine(vplc_hostname)
  
          # 
                      # record in pool as mine
                      self.vplc_pool.set_mine(vplc_hostname)
  
          # 
-        self.add_dummy_plc(plc_boxname,plc['name'])
+        self.add_dummy_plc(plc_boxname, plc['name'])
          vplc_ip = self.vplc_pool.get_ip(vplc_hostname)
          self.vplc_pool.add_starting(vplc_hostname, plc_boxname)
  
          #### compute a helpful vserver name
          # remove domain in hostname
          vplc_short = short_hostname(vplc_hostname)
          vplc_ip = self.vplc_pool.get_ip(vplc_hostname)
          self.vplc_pool.add_starting(vplc_hostname, plc_boxname)
  
          #### compute a helpful vserver name
          # remove domain in hostname
          vplc_short = short_hostname(vplc_hostname)
-        vservername = "%s-%d-%s" % (options.buildname,plc['index'],vplc_short)
-        plc_name = "%s_%s"%(plc['name'],vplc_short)
+        vservername = "{}-{}-{}".format(options.buildname, plc['index'], vplc_short)
+        plc_name = "{}_{}".format(plc['name'], vplc_short)
  
  
-        utils.header( 'PROVISION plc %s in box %s at IP %s as %s'%\
-                          (plc['name'],plc_boxname,vplc_hostname,vservername))
+        utils.header('PROVISION plc {} in box {} at IP {} as {}'\
+                     .format(plc['name'], plc_boxname, vplc_hostname, vservername))
  
          #### apply in the plc_spec
          # # informative
  
          #### apply in the plc_spec
          # # informative
-        # label=options.personality.replace("linux","")
-        mapper = {'plc': [ ('*' , {'host_box':plc_boxname,
-                                   # 'name':'%s-'+label,
-                                   'name': plc_name,
-                                   'vservername':vservername,
-                                   'vserverip':vplc_ip,
-                                   'PLC_DB_HOST':vplc_hostname,
-                                   'PLC_API_HOST':vplc_hostname,
-                                   'PLC_BOOT_HOST':vplc_hostname,
-                                   'PLC_WWW_HOST':vplc_hostname,
-                                   'PLC_NET_DNS1' : self.network_settings() [ 'interface_fields:dns1' ],
-                                   'PLC_NET_DNS2' : self.network_settings() [ 'interface_fields:dns2' ],
-                                   } ) ]
-                  }
+        # label = options.personality.replace("linux","")
+        mapper = {'plc' : [ ('*' , {'host_box' : plc_boxname,
+                                    'name' : plc_name,
+                                    'vservername' : vservername,
+                                    'vserverip' : vplc_ip,
+                                    'settings:PLC_DB_HOST' : vplc_hostname,
+                                    'settings:PLC_API_HOST' : vplc_hostname,
+                                    'settings:PLC_BOOT_HOST' : vplc_hostname,
+                                    'settings:PLC_WWW_HOST' : vplc_hostname,
+                                    'settings:PLC_NET_DNS1' : self.network_settings() [ 'interface_fields:dns1' ],
+                                    'settings:PLC_NET_DNS2' : self.network_settings() [ 'interface_fields:dns2' ],
+                                } ) ]
+              }
  
  
          # mappers only work on a list of plcs
  
  
          # mappers only work on a list of plcs
-        return TestMapper([plc],options).map(mapper)[0]
+        return TestMapper([plc], options).map(mapper)[0]
  
      ##########
  
      ##########
-    def provision_qemus (self, plc, options):
+    def provision_qemus(self, plc, options):
  
  
-        assert Substrate.check_options (options.ips_bnode, options.ips_vnode)
+        assert Substrate.check_options(options.ips_bnode, options.ips_vnode)
  
  
-        test_mapper = TestMapper ([plc], options)
+        test_mapper = TestMapper([plc], options)
          nodenames = test_mapper.node_names()
          nodenames = test_mapper.node_names()
-        maps=[]
+        maps = []
          for nodename in nodenames:
  
              if options.ips_vnode:
                  # as above, it's a rerun, take it for granted
          for nodename in nodenames:
  
              if options.ips_vnode:
                  # as above, it's a rerun, take it for granted
-                qemu_boxname=options.ips_bnode.pop()
-                vnode_hostname=options.ips_vnode.pop()
+                qemu_boxname = options.ips_bnode.pop()
+                vnode_hostname = options.ips_vnode.pop()
              else:
              else:
-                if self.sense(): self.list()
-                qemu_boxname=None
-                vnode_hostname=None
+                if self.sense():
+                    self.list()
+                qemu_boxname = None
+                vnode_hostname = None
                  # try to find an available IP 
                  self.vnode_pool.sense()
                  # try to find an available IP 
                  self.vnode_pool.sense()
-                couple=self.vnode_pool.next_free()
+                couple = self.vnode_pool.next_free()
                  if couple:
                  if couple:
-                    (vnode_hostname,unused)=couple
+                    (vnode_hostname, unused) = couple
                  # find a physical box
                  # find a physical box
-                max_free=0
+                max_free = 0
                  # use the box that has max free spots for load balancing
                  for qb in self.qemu_boxes:
                  # use the box that has max free spots for load balancing
                  for qb in self.qemu_boxes:
-                    free=qb.free_slots()
+                    free = qb.free_slots()
                      if free>max_free:
                      if free>max_free:
-                        qemu_boxname=qb.hostname
-                        max_free=free
+                        qemu_boxname = qb.hostname
+                        max_free = free
                  # if we miss the box or the IP, kill the oldest instance
                  if not qemu_boxname or not vnode_hostname:
                  # find the oldest of all our instances
                  # if we miss the box or the IP, kill the oldest instance
                  if not qemu_boxname or not vnode_hostname:
                  # find the oldest of all our instances
-                    all_qemu_instances=reduce(lambda x, y: x+y, 
-                                              [ qb.qemu_instances for qb in self.qemu_boxes ],
-                                              [])
-                    all_qemu_instances.sort(timestamp_sort)
+                    all_qemu_instances = reduce(lambda x, y: x+y, 
+                                                [ qb.qemu_instances for qb in self.qemu_boxes ],
+                                                [])
+                    all_qemu_instances.sort(key=timestamp_key)
                      try:
                      try:
-                        qemu_instance_to_kill=all_qemu_instances[0]
+                        qemu_instance_to_kill = all_qemu_instances[0]
                      except:
                      except:
-                        msg=""
-                        if not qemu_boxname: msg += " QEMU boxes are full"
-                        if not vnode_hostname: msg += " vnode IP pool exhausted" 
-                        msg += " %s"%self.summary_line()
-                        raise Exception,"Cannot make space for a QEMU instance:"+msg
-                    freed_qemu_boxname=qemu_instance_to_kill.qemu_box.hostname
-                    freed_vnode_hostname=short_hostname(qemu_instance_to_kill.nodename)
+                        msg = ""
+                        if not qemu_boxname:
+                            msg += " QEMU boxes are full"
+                        if not vnode_hostname:
+                            msg += " vnode IP pool exhausted" 
+                        msg += " {}".format(self.summary_line())
+                        raise Exception("Cannot make space for a QEMU instance:"+msg)
+                    freed_qemu_boxname = qemu_instance_to_kill.qemu_box.hostname
+                    freed_vnode_hostname = short_hostname(qemu_instance_to_kill.nodename)
                      # kill it
                      # kill it
-                    message='killing oldest qemu node = %s on %s'%(qemu_instance_to_kill.line(),
-                                                                   freed_qemu_boxname)
+                    message = 'killing oldest qemu node = {} on {}'.format(qemu_instance_to_kill.line(),
+                                                                           freed_qemu_boxname)
                      qemu_instance_to_kill.kill()
                      # use these freed resources where needed
                      if not qemu_boxname:
                      qemu_instance_to_kill.kill()
                      # use these freed resources where needed
                      if not qemu_boxname:
-                        qemu_boxname=freed_qemu_boxname
+                        qemu_boxname = freed_qemu_boxname
                      if not vnode_hostname:
                      if not vnode_hostname:
-                        vnode_hostname=freed_vnode_hostname
+                        vnode_hostname = freed_vnode_hostname
                          self.vnode_pool.set_mine(vnode_hostname)
  
                          self.vnode_pool.set_mine(vnode_hostname)
  
-            self.add_dummy_qemu (qemu_boxname,vnode_hostname)
-            mac=self.vnode_pool.retrieve_userdata(vnode_hostname)
-            ip=self.vnode_pool.get_ip (vnode_hostname)
-            self.vnode_pool.add_starting(vnode_hostname,qemu_boxname)
+            self.add_dummy_qemu(qemu_boxname, vnode_hostname)
+            mac = self.vnode_pool.retrieve_userdata(vnode_hostname)
+            ip = self.vnode_pool.get_ip(vnode_hostname)
+            self.vnode_pool.add_starting(vnode_hostname, qemu_boxname)
  
              vnode_fqdn = self.fqdn(vnode_hostname)
  
              vnode_fqdn = self.fqdn(vnode_hostname)
-            nodemap={'host_box':qemu_boxname,
-                     'node_fields:hostname':vnode_fqdn,
-                     'interface_fields:ip':ip, 
-                     'ipaddress_fields:ip_addr':ip, 
-                     'interface_fields:mac':mac,
-                     }
+            nodemap = {'host_box' : qemu_boxname,
+                       'node_fields:hostname' : vnode_fqdn,
+                       'interface_fields:ip' : ip, 
+                       'ipaddress_fields:ip_addr' : ip, 
+                       'interface_fields:mac' : mac,
+                   }
              nodemap.update(self.network_settings())
              nodemap.update(self.network_settings())
-            maps.append ( (nodename, nodemap) )
+            maps.append( (nodename, nodemap) )
  
  
-            utils.header("PROVISION node %s in box %s at IP %s with MAC %s"%\
-                             (nodename,qemu_boxname,vnode_hostname,mac))
+            utils.header("PROVISION node {} in box {} at IP {} with MAC {}"\
+                         .format(nodename, qemu_boxname, vnode_hostname, mac))
  
          return test_mapper.map({'node':maps})[0]
  
  
          return test_mapper.map({'node':maps})[0]
  
-    def localize_sfa_rspec (self,plc,options):
+    def localize_sfa_rspec(self, plc, options):
         
         
-        plc['sfa']['SFA_REGISTRY_HOST'] = plc['PLC_DB_HOST']
-        plc['sfa']['SFA_AGGREGATE_HOST'] = plc['PLC_DB_HOST']
-        plc['sfa']['SFA_SM_HOST'] = plc['PLC_DB_HOST']
-        plc['sfa']['SFA_DB_HOST'] = plc['PLC_DB_HOST']
-        plc['sfa']['SFA_PLC_URL'] = 'https://' + plc['PLC_API_HOST'] + ':443/PLCAPI/' 
-       return plc
+        plc['sfa']['settings']['SFA_REGISTRY_HOST'] = plc['settings']['PLC_DB_HOST']
+        plc['sfa']['settings']['SFA_AGGREGATE_HOST'] = plc['settings']['PLC_DB_HOST']
+        plc['sfa']['settings']['SFA_SM_HOST'] = plc['settings']['PLC_DB_HOST']
+        plc['sfa']['settings']['SFA_DB_HOST'] = plc['settings']['PLC_DB_HOST']
+        plc['sfa']['settings']['SFA_PLC_URL'] = 'https://{}:443/PLCAPI/'.format(plc['settings']['PLC_API_HOST'])
+        return plc
  
      #################### release:
  
      #################### release:
-    def release (self,options):
+    def release(self, options):
          self.vplc_pool.release_my_starting()
          self.vnode_pool.release_my_starting()
          pass
  
      #################### show results for interactive mode
          self.vplc_pool.release_my_starting()
          self.vnode_pool.release_my_starting()
          pass
  
      #################### show results for interactive mode
-    def get_box (self,boxname):
+    def get_box(self, boxname):
          for b in self.build_boxes + self.plc_boxes + self.qemu_boxes + [self.test_box] :
          for b in self.build_boxes + self.plc_boxes + self.qemu_boxes + [self.test_box] :
-            if b.shortname()==boxname:                          return b
+            if b.shortname() == boxname:
+                return b
              try:
              try:
-                if b.shortname()==boxname.split('.')[0]:        return b
-            except: pass
-        print "Could not find box %s"%boxname
+                if b.shortname() == boxname.split('.')[0]:
+                    return b
+            except:
+                pass
+        print("Could not find box {}".format(boxname))
          return None
  
          return None
  
-    def list_boxes(self,box_or_names):
-        print 'Sensing',
+    # deal with the mix of boxes and names and stores the current focus 
+    # as a list of Box instances in self.focus_all
+    def normalize(self, box_or_names):
+        self.focus_all = []
          for box in box_or_names:
          for box in box_or_names:
-            if not isinstance(box,Box): box=self.get_box(box)
-            if not box: continue
+            if not isinstance(box, Box):
+                box = self.get_box(box)
+            if not box: 
+                print('Warning - could not handle box',box)
+            self.focus_all.append(box)
+        # elaborate by type
+        self.focus_build = [ x for x in self.focus_all if isinstance(x, BuildBox) ]
+        self.focus_plc = [ x for x in self.focus_all if isinstance(x, PlcBox) ]
+        self.focus_qemu = [ x for x in self.focus_all if isinstance(x, QemuBox) ]
+                             
+    def list_boxes(self):
+        print('Sensing', end=' ')
+        sys.stdout.flush()
+        for box in self.focus_all:
              box.sense(self.options)
              box.sense(self.options)
-        print 'Done'
-        for box in box_or_names:
-            if not isinstance(box,Box): box=self.get_box(box)
-            if not box: continue
+        print('Done')
+        for box in self.focus_all:
              box.list(self.options.verbose)
  
              box.list(self.options.verbose)
  
-    def reboot_boxes(self,box_or_names):
-        for box in box_or_names:
-            if not isinstance(box,Box): box=self.get_box(box)
-            if not box: continue
+    def reboot_boxes(self):
+        for box in self.focus_all:
              box.reboot(self.options)
  
              box.reboot(self.options)
  
+    def sanity_check(self):
+        print('Sanity check')
+        self.sanity_check_plc()
+        self.sanity_check_qemu()
+
+    def sanity_check_plc(self):
+        pass
+
+    def sanity_check_qemu(self):
+        all_nodes = []
+        for box in self.focus_qemu:
+            all_nodes += box.node_names()
+        hash = {}
+        for node in all_nodes:
+            if node not in hash:
+                hash[node] = 0
+            hash[node]+=1
+        for (node,count) in list(hash.items()):
+            if count!=1:
+                print('WARNING - duplicate node', node)
+        
+
      ####################
      # can be run as a utility to probe/display/manage the local infrastructure
      ####################
      # can be run as a utility to probe/display/manage the local infrastructure
-    def main (self):
-        parser=OptionParser()
-        parser.add_option ('-r',"--reboot",action='store_true',dest='reboot',default=False,
-                           help='reboot mode (use shutdown -r)')
-        parser.add_option ('-s',"--soft",action='store_true',dest='soft',default=False,
-                           help='soft mode for reboot (terminates processes)')
-        parser.add_option ('-t',"--testbox",action='store_true',dest='testbox',default=False,
-                           help='add test box') 
-        parser.add_option ('-b',"--build",action='store_true',dest='builds',default=False,
-                           help='add build boxes')
-        parser.add_option ('-p',"--plc",action='store_true',dest='plcs',default=False,
-                           help='add plc boxes')
-        parser.add_option ('-q',"--qemu",action='store_true',dest='qemus',default=False,
-                           help='add qemu boxes') 
-        parser.add_option ('-a',"--all",action='store_true',dest='all',default=False,
-                           help='address all known  boxes, like -b -t -p -q')
-        parser.add_option ('-v',"--verbose",action='store_true',dest='verbose',default=False,
-                           help='verbose mode')
-        parser.add_option ('-n',"--dry_run",action='store_true',dest='dry_run',default=False,
-                           help='dry run mode')
-        (self.options,args)=parser.parse_args()
-
-        boxes=args
+    def main(self):
+        parser = OptionParser()
+        parser.add_option('-r', "--reboot", action='store_true', dest='reboot', default=False,
+                          help='reboot mode (use shutdown -r)')
+        parser.add_option('-s', "--soft", action='store_true', dest='soft', default=False,
+                          help='soft mode for reboot (terminates processes)')
+        parser.add_option('-t', "--testbox", action='store_true', dest='testbox', default=False,
+                          help='add test box') 
+        parser.add_option('-b', "--build", action='store_true', dest='builds', default=False,
+                          help='add build boxes')
+        parser.add_option('-p', "--plc", action='store_true', dest='plcs', default=False,
+                          help='add plc boxes')
+        parser.add_option('-q', "--qemu", action='store_true', dest='qemus', default=False,
+                          help='add qemu boxes') 
+        parser.add_option('-a', "--all", action='store_true', dest='all', default=False,
+                          help='address all known  boxes, like -b -t -p -q')
+        parser.add_option('-v', "--verbose", action='store_true', dest='verbose', default=False,
+                          help='verbose mode')
+        parser.add_option('-n', "--dry_run", action='store_true', dest='dry_run', default=False,
+                          help='dry run mode')
+        (self.options, args) = parser.parse_args()
+
+        boxes = args
          if self.options.testbox: boxes += [self.test_box]
          if self.options.builds: boxes += self.build_boxes
          if self.options.plcs: boxes += self.plc_boxes
          if self.options.testbox: boxes += [self.test_box]
          if self.options.builds: boxes += self.build_boxes
          if self.options.plcs: boxes += self.plc_boxes
@@ -1236,10 +1399,15 @@ class Substrate:
          if self.options.all: boxes += self.all_boxes
          
          global verbose
          if self.options.all: boxes += self.all_boxes
          
          global verbose
-        verbose=self.options.verbose
+        verbose = self.options.verbose
          # default scope is -b -p -q -t
          if not boxes:
              boxes = self.build_boxes + self.plc_boxes + self.qemu_boxes + [self.test_box]
  
          # default scope is -b -p -q -t
          if not boxes:
              boxes = self.build_boxes + self.plc_boxes + self.qemu_boxes + [self.test_box]
  
-        if self.options.reboot: self.reboot_boxes (boxes)
-        else:                   self.list_boxes (boxes)
+        self.normalize(boxes)
+
+        if self.options.reboot:
+            self.reboot_boxes()
+        else:
+            self.list_boxes()
+            self.sanity_check()