more consistently rename plc_spec['hostname'] into plc_spec['host_box']

[tests.git] / system / TestPlc.py

diff --git a/system/TestPlc.py b/system/TestPlc.py
index 487884f..85750c9 100644 (file)
--- a/system/TestPlc.py
+++ b/system/TestPlc.py
@@ -1,4 +1,6 @@
-# $Id$
+# Thierry Parmentelat <thierry.parmentelat@inria.fr>
+# Copyright (C) 2010 INRIA 
+#

 import os, os.path
 import datetime
 import time
 import os, os.path
 import datetime
 import time


@@ -14,7 +16,7 @@ from TestUser import TestUser
 from TestKey import TestKey
 from TestSlice import TestSlice
 from TestSliver import TestSliver
 from TestKey import TestKey
 from TestSlice import TestSlice
 from TestSliver import TestSliver

-from TestBox import TestBox
+from TestBoxQemu import TestBoxQemu

 from TestSsh import TestSsh
 from TestApiserver import TestApiserver
 from TestSliceSfa import TestSliceSfa
 from TestSsh import TestSsh
 from TestApiserver import TestApiserver
 from TestSliceSfa import TestSliceSfa


@@ -50,7 +52,7 @@ def node_mapper (method):
     actual.__doc__=method.__doc__
     return actual
 
     actual.__doc__=method.__doc__
     return actual
 

-def slice_mapper_options (method):
+def slice_mapper (method):

     def actual(self):
         overall=True
         slice_method = TestSlice.__dict__[method.__name__]
     def actual(self):
         overall=True
         slice_method = TestSlice.__dict__[method.__name__]


@@ -64,15 +66,14 @@ def slice_mapper_options (method):
     actual.__doc__=method.__doc__
     return actual
 
     actual.__doc__=method.__doc__
     return actual
 

-def slice_mapper_options_sfa (method):
+def slice_sfa_mapper (method):

     def actual(self):
     def actual(self):

-       test_plc=self

         overall=True
         slice_method = TestSliceSfa.__dict__[method.__name__]
         overall=True
         slice_method = TestSliceSfa.__dict__[method.__name__]

-        for slice_spec in self.plc_spec['sfa']['slices_sfa']:
+        for slice_spec in self.plc_spec['sfa']['sfa_slice_specs']:

             site_spec = self.locate_site (slice_spec['sitename'])
             test_site = TestSite(self,site_spec)
             site_spec = self.locate_site (slice_spec['sitename'])
             test_site = TestSite(self,site_spec)

-            test_slice=TestSliceSfa(test_plc,test_site,slice_spec)
+            test_slice=TestSliceSfa(self,test_site,slice_spec)

             if not slice_method(test_slice,self.options): overall=False
         return overall
     # restore the doc text
             if not slice_method(test_slice,self.options): overall=False
         return overall
     # restore the doc text


@@ -80,47 +81,71 @@ def slice_mapper_options_sfa (method):
     return actual
 
 SEP='<sep>'
     return actual
 
 SEP='<sep>'

+SEPSFA='<sep_sfa>'

 
 class TestPlc:
 
     default_steps = [
 
 class TestPlc:
 
     default_steps = [

-        'display', 'local_pre', SEP,
-        'delete','create','install', 'configure', 'start', SEP,
-        'fetch_keys', 'store_keys', 'clear_known_hosts', SEP,
-        'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', SEP,
-        'reinstall_node', 'init_node','bootcd', 'configure_qemu', 'export_qemu',
-        'kill_all_qemus', 'start_node', SEP,
-        # better use of time: do this now that the nodes are taking off
-        'plcsh_stress_test', SEP,
-        'nodes_ssh_debug', 'nodes_ssh_boot', 'check_slice', 'check_initscripts', SEP,
-       'install_sfa', 'configure_sfa', 'import_sfa', 'start_sfa', SEP,
-        'setup_sfa', 'add_sfa', 'update_sfa', SEP,
-        'view_sfa', 'check_slice_sfa', 'delete_sfa', 'stop_sfa', SEP,
-        'check_tcp',  'check_hooks',  SEP,
-        'force_gather_logs', 'force_local_post',
+        'show', SEP,
+        'vs_delete','vs_create','timestamp_vs', SEP,
+        'plc_install', 'plc_configure', 'plc_start', SEP,
+        'keys_fetch', 'keys_store', 'keys_clear_known_hosts', SEP,
+        'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', 'leases', SEP,
+        'nodestate_reinstall', 'qemu_local_init','bootcd', 'qemu_local_config', SEP,
+        'qemu_export', 'qemu_kill_all', 'qemu_start', 'timestamp_qemu', SEP,
+       'sfa_install', 'sfa_configure', 'cross_sfa_configure', 'sfa_import', 'sfa_start', SEPSFA,
+        'sfi_configure@1', 'sfa_add_user@1', 'sfa_add_slice@1', 'sfa_discover@1', SEPSFA,
+        'sfa_create_slice@1', 'sfa_check_slice_plc@1', SEPSFA, 
+        'sfa_update_user@1', 'sfa_update_slice@1', 'sfa_view@1', 'sfa_utest@1',SEPSFA,
+        # we used to run plcsh_stress_test, and then ssh_node_debug and ssh_node_boot
+        # but as the stress test might take a while, we sometimes missed the debug mode..
+        'ssh_node_debug', 'plcsh_stress_test@1', SEP,
+        'ssh_node_boot', 'ssh_slice', 'check_initscripts', SEP,
+        'ssh_slice_sfa@1', 'sfa_delete_slice@1', 'sfa_delete_user@1', SEPSFA,
+        'check_tcp',  SEP,
+        'force_gather_logs', SEP,

         ]
     other_steps = [ 
         ]
     other_steps = [ 

-        'fresh_install', 'stop', 'vs_start', SEP,
-        'clean_initscripts', 'clean_nodegroups','clean_all_sites', SEP,
-        'clean_sites', 'clean_nodes', 'clean_slices', 'clean_keys', SEP,
+        'check_hooks',  
+        'free_all',
+        'show_boxes', 'local_list','local_rel','local_rel_plc','local_rel_qemu',SEP,
+        'plc_stop', 'vs_start', 'vs_stop', SEP,
+        'delete_initscripts', 'delete_nodegroups','delete_all_sites', SEP,
+        'delete_sites', 'delete_nodes', 'delete_slices', 'keys_clean', SEP,
+        'delete_leases', 'list_leases', SEP,

         'populate' , SEP,
         'populate' , SEP,

-        'show_boxes', 'list_all_qemus', 'list_qemus', 'kill_qemus', SEP,
-        'db_dump' , 'db_restore', SEP,
-        'local_list','local_cleanup',SEP,
-        'standby_1 through 20',
+        'nodestate_show','nodestate_safeboot','nodestate_boot', SEP,
+        'qemu_list_all', 'qemu_list_mine', 'qemu_kill_mine', SEP,
+        'sfa_plcclean', 'sfa_dbclean', 'sfa_stop','sfa_uninstall', 'sfi_clean', SEP,
+        'plc_db_dump' , 'plc_db_restore', SEP,
+        'standby_1 through 20',SEP,

         ]
 
     @staticmethod
     def printable_steps (list):
         ]
 
     @staticmethod
     def printable_steps (list):

-        return " ".join(list).replace(" "+SEP+" "," \\\n")
+        single_line=" ".join(list)+" "
+        return single_line.replace(" "+SEP+" "," \\\n").replace(" "+SEPSFA+" "," \\\n")

     @staticmethod
     def valid_step (step):
     @staticmethod
     def valid_step (step):

-        return step != SEP
+        return step != SEP and step != SEPSFA
+
+    # turn off the sfa-related steps when build has skipped SFA
+    # this is originally for centos5 as recent SFAs won't build on this platformb
+    @staticmethod
+    def check_whether_build_has_sfa (rpms_url):
+        # warning, we're now building 'sface' so let's be a bit more picky
+        retcod=os.system ("curl --silent %s/ | grep -q sfa-"%rpms_url)
+        # full builds are expected to return with 0 here
+        if retcod!=0:
+            # move all steps containing 'sfa' from default_steps to other_steps
+            sfa_steps= [ step for step in TestPlc.default_steps if step.find('sfa')>=0 ]
+            TestPlc.other_steps += sfa_steps
+            for step in sfa_steps: TestPlc.default_steps.remove(step)

 
     def __init__ (self,plc_spec,options):
        self.plc_spec=plc_spec
         self.options=options
 
     def __init__ (self,plc_spec,options):
        self.plc_spec=plc_spec
         self.options=options

-       self.test_ssh=TestSsh(self.plc_spec['hostname'],self.options.buildname)
+       self.test_ssh=TestSsh(self.plc_spec['host_box'],self.options.buildname)

         try:
             self.vserverip=plc_spec['vserverip']
             self.vservername=plc_spec['vservername']
         try:
             self.vserverip=plc_spec['vserverip']
             self.vservername=plc_spec['vservername']


@@ -135,7 +160,7 @@ class TestPlc:
         return "%s.%s"%(name,self.vservername)
 
     def hostname(self):
         return "%s.%s"%(name,self.vservername)
 
     def hostname(self):

-        return self.plc_spec['hostname']
+        return self.plc_spec['host_box']

 
     def is_local (self):
         return self.test_ssh.is_local()
 
     def is_local (self):
         return self.test_ssh.is_local()


@@ -151,6 +176,9 @@ class TestPlc:
     def start_guest (self):
       return utils.system(self.test_ssh.actual_command(self.start_guest_in_host()))
     
     def start_guest (self):
       return utils.system(self.test_ssh.actual_command(self.start_guest_in_host()))
     

+    def stop_guest (self):
+      return utils.system(self.test_ssh.actual_command(self.stop_guest_in_host()))
+    

     def run_in_guest (self,command):
         return utils.system(self.actual_command_in_guest(command))
     
     def run_in_guest (self,command):
         return utils.system(self.actual_command_in_guest(command))
     


@@ -161,10 +189,13 @@ class TestPlc:
     def host_to_guest(self,command):
         return "vserver %s exec %s"%(self.vservername,command)
     
     def host_to_guest(self,command):
         return "vserver %s exec %s"%(self.vservername,command)
     

-    #command gets run in the vserver
+    #start/stop the vserver

     def start_guest_in_host(self):
         return "vserver %s start"%(self.vservername)
     
     def start_guest_in_host(self):
         return "vserver %s start"%(self.vservername)
     

+    def stop_guest_in_host(self):
+        return "vserver %s stop"%(self.vservername)
+    

     # xxx quick n dirty
     def run_in_guest_piped (self,local,remote):
         return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
     # xxx quick n dirty
     def run_in_guest_piped (self,local,remote):
         return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))


@@ -261,29 +292,32 @@ class TestPlc:
                     
     # a step for checking this stuff
     def show_boxes (self):
                     
     # a step for checking this stuff
     def show_boxes (self):

+        'print summary of nodes location'

         for (box,nodes) in self.gather_hostBoxes().iteritems():
             print box,":"," + ".join( [ node.name() for node in nodes ] )
         return True
 
     # make this a valid step
         for (box,nodes) in self.gather_hostBoxes().iteritems():
             print box,":"," + ".join( [ node.name() for node in nodes ] )
         return True
 
     # make this a valid step

-    def kill_all_qemus(self):
-        "all qemu boxes: kill all running qemus (even of former runs)"
+    def qemu_kill_all(self):
+        'kill all qemu instances on the qemu boxes involved by this setup'

         # this is the brute force version, kill all qemus on that host box
         for (box,nodes) in self.gather_hostBoxes().iteritems():
             # pass the first nodename, as we don't push template-qemu on testboxes
             nodedir=nodes[0].nodedir()
         # this is the brute force version, kill all qemus on that host box
         for (box,nodes) in self.gather_hostBoxes().iteritems():
             # pass the first nodename, as we don't push template-qemu on testboxes
             nodedir=nodes[0].nodedir()

-            TestBox(box,self.options.buildname).kill_all_qemus(nodedir)
+            TestBoxQemu(box,self.options.buildname).qemu_kill_all(nodedir)

         return True
 
     # make this a valid step
         return True
 
     # make this a valid step

-    def list_all_qemus(self):
+    def qemu_list_all(self):
+        'list all qemu instances on the qemu boxes involved by this setup'

         for (box,nodes) in self.gather_hostBoxes().iteritems():
             # this is the brute force version, kill all qemus on that host box
         for (box,nodes) in self.gather_hostBoxes().iteritems():
             # this is the brute force version, kill all qemus on that host box

-            TestBox(box,self.options.buildname).list_all_qemus()
+            TestBoxQemu(box,self.options.buildname).qemu_list_all()

         return True
 
     # kill only the right qemus
         return True
 
     # kill only the right qemus

-    def list_qemus(self):
+    def qemu_list_mine(self):
+        'list qemu instances for our nodes'

         for (box,nodes) in self.gather_hostBoxes().iteritems():
             # the fine-grain version
             for node in nodes:
         for (box,nodes) in self.gather_hostBoxes().iteritems():
             # the fine-grain version
             for node in nodes:


@@ -291,7 +325,8 @@ class TestPlc:
         return True
 
     # kill only the right qemus
         return True
 
     # kill only the right qemus

-    def kill_qemus(self):
+    def qemu_kill_mine(self):
+        'kill the qemu instances for our nodes'

         for (box,nodes) in self.gather_hostBoxes().iteritems():
             # the fine-grain version
             for node in nodes:
         for (box,nodes) in self.gather_hostBoxes().iteritems():
             # the fine-grain version
             for node in nodes:


@@ -299,15 +334,17 @@ class TestPlc:
         return True
 
     #################### display config
         return True
 
     #################### display config

-    def display (self):
+    def show (self):

         "show test configuration after localization"
         self.display_pass (1)
         self.display_pass (2)
         return True
 
     # entry point
         "show test configuration after localization"
         self.display_pass (1)
         self.display_pass (2)
         return True
 
     # entry point

+    always_display_keys=['PLC_WWW_HOST','nodes','sites',]

     def display_pass (self,passno):
         for (key,val) in self.plc_spec.iteritems():
     def display_pass (self,passno):
         for (key,val) in self.plc_spec.iteritems():

+            if not self.options.verbose and key not in TestPlc.always_display_keys: continue

             if passno == 2:
                 if key == 'sites':
                     for site in val:
             if passno == 2:
                 if key == 'sites':
                     for site in val:


@@ -324,12 +361,13 @@ class TestPlc:
                     for key in val:
                         self.display_key_spec (key)
             elif passno == 1:
                     for key in val:
                         self.display_key_spec (key)
             elif passno == 1:

-                if key not in ['sites','initscripts','slices','keys']:
+                if key not in ['sites','initscripts','slices','keys', 'sfa']:

                     print '+   ',key,':',val
 
     def display_site_spec (self,site):
         print '+ ======== site',site['site_fields']['name']
         for (k,v) in site.iteritems():
                     print '+   ',key,':',val
 
     def display_site_spec (self,site):
         print '+ ======== site',site['site_fields']['name']
         for (k,v) in site.iteritems():

+            if not self.options.verbose and k not in TestPlc.always_display_keys: continue

             if k=='nodes':
                 if v: 
                     print '+       ','nodes : ',
             if k=='nodes':
                 if v: 
                     print '+       ','nodes : ',


@@ -347,8 +385,8 @@ class TestPlc:
             elif k == 'address_fields':
                 pass
             else:
             elif k == 'address_fields':
                 pass
             else:

-                print '+       ',k,
-                PrettyPrinter(indent=8,depth=2).pprint(v)
+                print '+       ',
+                utils.pprint(k,v)

         
     def display_initscript_spec (self,initscript):
         print '+ ======== initscript',initscript['initscript_fields']['name']
         
     def display_initscript_spec (self,initscript):
         print '+ ======== initscript',initscript['initscript_fields']['name']


@@ -379,10 +417,11 @@ class TestPlc:
                 print '+       ',k,v
 
     def display_node_spec (self,node):
                 print '+       ',k,v
 
     def display_node_spec (self,node):

-        print "+           node",node['name'],"host_box=",node['host_box'],
+        print "+           node=%s host_box=%s"%(node['name'],node['host_box']),

         print "hostname=",node['node_fields']['hostname'],
         print "ip=",node['interface_fields']['ip']
         print "hostname=",node['node_fields']['hostname'],
         print "ip=",node['interface_fields']['ip']

-    
+        if self.options.verbose:
+            utils.pprint("node details",node,depth=3)

 
     # another entry point for just showing the boxes involved
     def display_mapping (self):
 
     # another entry point for just showing the boxes involved
     def display_mapping (self):


@@ -392,7 +431,7 @@ class TestPlc:
     @staticmethod
     def display_mapping_plc (plc_spec):
         print '+ MyPLC',plc_spec['name']
     @staticmethod
     def display_mapping_plc (plc_spec):
         print '+ MyPLC',plc_spec['name']

-        print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['hostname'],plc_spec['vservername'])
+        print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['host_box'],plc_spec['vservername'])

         print '+\tIP = %s/%s'%(plc_spec['PLC_API_HOST'],plc_spec['vserverip'])
         for site_spec in plc_spec['sites']:
             for node_spec in site_spec['nodes']:
         print '+\tIP = %s/%s'%(plc_spec['PLC_API_HOST'],plc_spec['vserverip'])
         for site_spec in plc_spec['sites']:
             for node_spec in site_spec['nodes']:


@@ -404,6 +443,12 @@ class TestPlc:
         print '+\tqemu box %s'%node_spec['host_box']
         print '+\thostname=%s'%node_spec['node_fields']['hostname']
 
         print '+\tqemu box %s'%node_spec['host_box']
         print '+\thostname=%s'%node_spec['node_fields']['hostname']
 

+    # write a timestamp in /vservers/<>/
+    def timestamp_vs (self):
+        now=int(time.time())
+        utils.system(self.test_ssh.actual_command("mkdir -p /vservers/%s"%self.vservername))
+        return utils.system(self.test_ssh.actual_command("echo %d > /vservers/%s/timestamp"%(now,self.vservername)))==0
+        

     def local_pre (self):
         "run site-dependant pre-test script as defined in LocalTestResources"
         from LocalTestResources import local_resources
     def local_pre (self):
         "run site-dependant pre-test script as defined in LocalTestResources"
         from LocalTestResources import local_resources


@@ -419,19 +464,33 @@ class TestPlc:
         from LocalTestResources import local_resources
         return local_resources.step_list(self)
  
         from LocalTestResources import local_resources
         return local_resources.step_list(self)
  

-    def local_cleanup (self):
-        "run site-dependant cleanup script as defined in LocalTestResources"
+    def local_rel (self):
+        "run site-dependant release script as defined in LocalTestResources"
+        from LocalTestResources import local_resources
+        return local_resources.step_release(self)
+ 
+    def local_rel_plc (self):
+        "run site-dependant release script as defined in LocalTestResources"
+        from LocalTestResources import local_resources
+        return local_resources.step_release_plc(self)
+ 
+    def local_rel_qemu (self):
+        "run site-dependant release script as defined in LocalTestResources"

         from LocalTestResources import local_resources
         from LocalTestResources import local_resources

-        return local_resources.step_cleanup(self)
+        return local_resources.step_release_qemu(self)

  
  

-    def delete(self):
+    def vs_delete(self):

         "vserver delete the test myplc"
         self.run_in_host("vserver --silent %s delete"%self.vservername)
         return True
 
     ### install
         "vserver delete the test myplc"
         self.run_in_host("vserver --silent %s delete"%self.vservername)
         return True
 
     ### install

-    def create (self):
+    # historically the build was being fetched by the tests
+    # now the build pushes itself as a subdir of the tests workdir
+    # so that the tests do not have to worry about extracting the build (svn, git, or whatever)
+    def vs_create (self):

         "vserver creation (no install done)"
         "vserver creation (no install done)"

+        # push the local build/ dir to the testplc box 

         if self.is_local():
             # a full path for the local calls
             build_dir=os.path.dirname(sys.argv[0])
         if self.is_local():
             # a full path for the local calls
             build_dir=os.path.dirname(sys.argv[0])


@@ -441,10 +500,9 @@ class TestPlc:
         else:
             # use a standard name - will be relative to remote buildname
             build_dir="build"
         else:
             # use a standard name - will be relative to remote buildname
             build_dir="build"

-       # run checkout in any case - would do an update if already exists
-        build_checkout = "svn checkout %s %s"%(self.options.build_url,build_dir)
-        if self.run_in_host(build_checkout) != 0:
-            return False
+            # remove for safety; do *not* mkdir first, otherwise we end up with build/build/
+            self.test_ssh.rmdir(build_dir)
+            self.test_ssh.copy(build_dir,recursive=True)

         # the repo url is taken from arch-rpms-url 
         # with the last step (i386) removed
         repo_url = self.options.arch_rpms_url
         # the repo url is taken from arch-rpms-url 
         # with the last step (i386) removed
         repo_url = self.options.arch_rpms_url


@@ -462,40 +520,47 @@ class TestPlc:
             vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
             vserver_options += " --hostname %s"%vserver_hostname
         except:
             vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
             vserver_options += " --hostname %s"%vserver_hostname
         except:

-            pass
+            print "Cannot reverse lookup %s"%self.vserverip
+            print "This is considered fatal, as this might pollute the test results"
+            return False

         create_vserver="%(build_dir)s/%(script)s %(test_env_options)s %(vserver_name)s %(repo_url)s -- %(vserver_options)s"%locals()
         return self.run_in_host(create_vserver) == 0
 
     ### install_rpm 
         create_vserver="%(build_dir)s/%(script)s %(test_env_options)s %(vserver_name)s %(repo_url)s -- %(vserver_options)s"%locals()
         return self.run_in_host(create_vserver) == 0
 
     ### install_rpm 

-    def install(self):
+    def plc_install(self):

         "yum install myplc, noderepo, and the plain bootstrapfs"
 
         "yum install myplc, noderepo, and the plain bootstrapfs"
 

-        # workaround for getting pgsql5.2 on centos5
+        # workaround for getting pgsql8.2 on centos5

         if self.options.fcdistro == "centos5":
         if self.options.fcdistro == "centos5":

-            self.run_in_guest("rpm -Uvh http://yum.pgsqlrpms.org/8.2/pgdg-centos-8.2-4.noarch.rpm")

             self.run_in_guest("rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-3.noarch.rpm")
 
             self.run_in_guest("rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-3.noarch.rpm")
 

+        # compute nodefamily

         if self.options.personality == "linux32":
             arch = "i386"
         elif self.options.personality == "linux64":
             arch = "x86_64"
         else:
             raise Exception, "Unsupported personality %r"%self.options.personality
         if self.options.personality == "linux32":
             arch = "i386"
         elif self.options.personality == "linux64":
             arch = "x86_64"
         else:
             raise Exception, "Unsupported personality %r"%self.options.personality

-        

         nodefamily="%s-%s-%s"%(self.options.pldistro,self.options.fcdistro,arch)
         nodefamily="%s-%s-%s"%(self.options.pldistro,self.options.fcdistro,arch)

-        return \
-            self.run_in_guest("yum -y install myplc")==0 and \
-            self.run_in_guest("yum -y install noderepo-%s"%nodefamily)==0 and \
-            self.run_in_guest("yum -y install bootstrapfs-%s-plain"%nodefamily)==0 
+
+        pkgs_list=[]
+        pkgs_list.append ("slicerepo-%s"%nodefamily)
+        pkgs_list.append ("myplc")
+        pkgs_list.append ("noderepo-%s"%nodefamily)
+        pkgs_list.append ("bootstrapfs-%s-plain"%nodefamily)
+        pkgs_string=" ".join(pkgs_list)
+        self.run_in_guest("yum -y install %s"%pkgs_string)
+        return self.run_in_guest("rpm -q %s"%pkgs_string)==0

 
     ### 
 
     ### 

-    def configure(self):
+    def plc_configure(self):

         "run plc-config-tty"
         tmpname='%s.plc-config-tty'%(self.name())
         fileconf=open(tmpname,'w')
         for var in [ 'PLC_NAME',
         "run plc-config-tty"
         tmpname='%s.plc-config-tty'%(self.name())
         fileconf=open(tmpname,'w')
         for var in [ 'PLC_NAME',

-                     'PLC_ROOT_PASSWORD',

                      'PLC_ROOT_USER',
                      'PLC_ROOT_USER',

+                     'PLC_ROOT_PASSWORD',
+                     'PLC_SLICE_PREFIX',

                      'PLC_MAIL_ENABLED',
                      'PLC_MAIL_SUPPORT_ADDRESS',
                      'PLC_DB_HOST',
                      'PLC_MAIL_ENABLED',
                      'PLC_MAIL_SUPPORT_ADDRESS',
                      'PLC_DB_HOST',


@@ -505,7 +570,11 @@ class TestPlc:
                      'PLC_WWW_HOST',
                      'PLC_BOOT_HOST',
                      'PLC_NET_DNS1',
                      'PLC_WWW_HOST',
                      'PLC_BOOT_HOST',
                      'PLC_NET_DNS1',

-                     'PLC_NET_DNS2']:
+                     'PLC_NET_DNS2',
+                     'PLC_RESERVATION_GRANULARITY',
+                     'PLC_OMF_ENABLED',
+                     'PLC_OMF_XMPP_SERVER',
+                     ]:

             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
         fileconf.write('w\n')
         fileconf.write('q\n')
             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
         fileconf.write('w\n')
         fileconf.write('q\n')


@@ -515,33 +584,41 @@ class TestPlc:
         utils.system('rm %s'%tmpname)
         return True
 
         utils.system('rm %s'%tmpname)
         return True
 

-    def start(self):
+    def plc_start(self):

         "service plc start"
         self.run_in_guest('service plc start')
         return True
 
         "service plc start"
         self.run_in_guest('service plc start')
         return True
 

-    def stop(self):
+    def plc_stop(self):

         "service plc stop"
         self.run_in_guest('service plc stop')
         return True
         
     def vs_start (self):
         "service plc stop"
         self.run_in_guest('service plc stop')
         return True
         
     def vs_start (self):

+        "start the PLC vserver"

         self.start_guest()
         return True
 
         self.start_guest()
         return True
 

+    def vs_stop (self):
+        "stop the PLC vserver"
+        self.stop_guest()
+        return True
+

     # stores the keys from the config for further use
     # stores the keys from the config for further use

-    def store_keys(self):
+    def keys_store(self):

         "stores test users ssh keys in keys/"
         for key_spec in self.plc_spec['keys']:
                TestKey(self,key_spec).store_key()
         return True
 
         "stores test users ssh keys in keys/"
         for key_spec in self.plc_spec['keys']:
                TestKey(self,key_spec).store_key()
         return True
 

-    def clean_keys(self):
-        utils.system("rm -rf %s/keys/"%os.path(sys.argv[0]))
+    def keys_clean(self):
+        "removes keys cached in keys/"
+        utils.system("rm -rf ./keys")
+        return True

 
     # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
     # for later direct access to the nodes
 
     # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
     # for later direct access to the nodes

-    def fetch_keys(self):
+    def keys_fetch(self):

         "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
         dir="./keys"
         if not os.path.isdir(dir):
         "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
         dir="./keys"
         if not os.path.isdir(dir):


@@ -559,7 +636,7 @@ class TestPlc:
         "create sites with PLCAPI"
         return self.do_sites()
     
         "create sites with PLCAPI"
         return self.do_sites()
     

-    def clean_sites (self):
+    def delete_sites (self):

         "delete sites with PLCAPI"
         return self.do_sites(action="delete")
     
         "delete sites with PLCAPI"
         return self.do_sites(action="delete")
     


@@ -578,17 +655,19 @@ class TestPlc:
                 test_site.create_users()
         return True
 
                 test_site.create_users()
         return True
 

-    def clean_all_sites (self):
+    def delete_all_sites (self):
+        "Delete all sites in PLC, and related objects"

         print 'auth_root',self.auth_root()
         site_ids = [s['site_id'] for s in self.apiserver.GetSites(self.auth_root(), {}, ['site_id'])]
         for site_id in site_ids:
             print 'Deleting site_id',site_id
             self.apiserver.DeleteSite(self.auth_root(),site_id)
         print 'auth_root',self.auth_root()
         site_ids = [s['site_id'] for s in self.apiserver.GetSites(self.auth_root(), {}, ['site_id'])]
         for site_id in site_ids:
             print 'Deleting site_id',site_id
             self.apiserver.DeleteSite(self.auth_root(),site_id)

+        return True

 
     def nodes (self):
         "create nodes with PLCAPI"
         return self.do_nodes()
 
     def nodes (self):
         "create nodes with PLCAPI"
         return self.do_nodes()

-    def clean_nodes (self):
+    def delete_nodes (self):

         "delete nodes with PLCAPI"
         return self.do_nodes(action="delete")
 
         "delete nodes with PLCAPI"
         return self.do_nodes(action="delete")
 


@@ -612,10 +691,72 @@ class TestPlc:
     def nodegroups (self):
         "create nodegroups with PLCAPI"
         return self.do_nodegroups("add")
     def nodegroups (self):
         "create nodegroups with PLCAPI"
         return self.do_nodegroups("add")

-    def clean_nodegroups (self):
+    def delete_nodegroups (self):

         "delete nodegroups with PLCAPI"
         return self.do_nodegroups("delete")
 
         "delete nodegroups with PLCAPI"
         return self.do_nodegroups("delete")
 

+    YEAR = 365*24*3600
+    @staticmethod
+    def translate_timestamp (start,grain,timestamp):
+        if timestamp < TestPlc.YEAR:    return start+timestamp*grain
+        else:                           return timestamp
+
+    @staticmethod
+    def timestamp_printable (timestamp):
+        return time.strftime('%m-%d %H:%M:%S UTC',time.gmtime(timestamp))
+
+    def leases(self):
+        "create leases (on reservable nodes only, use e.g. run -c default -c resa)"
+        now=int(time.time())
+        grain=self.apiserver.GetLeaseGranularity(self.auth_root())
+        print 'API answered grain=',grain
+        start=(now/grain)*grain
+        start += grain
+        # find out all nodes that are reservable
+        nodes=self.all_reservable_nodenames()
+        if not nodes: 
+            utils.header ("No reservable node found - proceeding without leases")
+            return True
+        ok=True
+        # attach them to the leases as specified in plc_specs
+        # this is where the 'leases' field gets interpreted as relative of absolute
+        for lease_spec in self.plc_spec['leases']:
+            # skip the ones that come with a null slice id
+            if not lease_spec['slice']: continue
+            lease_spec['t_from']=TestPlc.translate_timestamp(start,grain,lease_spec['t_from'])
+            lease_spec['t_until']=TestPlc.translate_timestamp(start,grain,lease_spec['t_until'])
+            lease_addition=self.apiserver.AddLeases(self.auth_root(),nodes,
+                                                    lease_spec['slice'],lease_spec['t_from'],lease_spec['t_until'])
+            if lease_addition['errors']:
+                utils.header("Cannot create leases, %s"%lease_addition['errors'])
+                ok=False
+            else:
+                utils.header('Leases on nodes %r for %s from %d (%s) until %d (%s)'%\
+                              (nodes,lease_spec['slice'],
+                               lease_spec['t_from'],TestPlc.timestamp_printable(lease_spec['t_from']),
+                               lease_spec['t_until'],TestPlc.timestamp_printable(lease_spec['t_until'])))
+                
+        return ok
+
+    def delete_leases (self):
+        "remove all leases in the myplc side"
+        lease_ids= [ l['lease_id'] for l in self.apiserver.GetLeases(self.auth_root())]
+        utils.header("Cleaning leases %r"%lease_ids)
+        self.apiserver.DeleteLeases(self.auth_root(),lease_ids)
+        return True
+
+    def list_leases (self):
+        "list all leases known to the myplc"
+        leases = self.apiserver.GetLeases(self.auth_root())
+        now=int(time.time())
+        for l in leases:
+            current=l['t_until']>=now
+            if self.options.verbose or current:
+                utils.header("%s %s from %s until %s"%(l['hostname'],l['name'],
+                                                       TestPlc.timestamp_printable(l['t_from']), 
+                                                       TestPlc.timestamp_printable(l['t_until'])))
+        return True
+

     # create nodegroups if needed, and populate
     def do_nodegroups (self, action="add"):
         # 1st pass to scan contents
     # create nodegroups if needed, and populate
     def do_nodegroups (self, action="add"):
         # 1st pass to scan contents


@@ -645,8 +786,7 @@ class TestPlc:
                     tag_type_id = self.apiserver.AddTagType(auth,
                                                             {'tagname':nodegroupname,
                                                              'description': 'for nodegroup %s'%nodegroupname,
                     tag_type_id = self.apiserver.AddTagType(auth,
                                                             {'tagname':nodegroupname,
                                                              'description': 'for nodegroup %s'%nodegroupname,

-                                                             'category':'test',
-                                                             'min_role_id':10})
+                                                             'category':'test'})

                 print 'located tag (type)',nodegroupname,'as',tag_type_id
                 # create nodegroup
                 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
                 print 'located tag (type)',nodegroupname,'as',tag_type_id
                 # create nodegroup
                 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})


@@ -687,10 +827,18 @@ class TestPlc:
         node_infos = []
         for site_spec in self.plc_spec['sites']:
             node_infos += [ (node_spec['node_fields']['hostname'],node_spec['host_box']) \
         node_infos = []
         for site_spec in self.plc_spec['sites']:
             node_infos += [ (node_spec['node_fields']['hostname'],node_spec['host_box']) \

-                           for node_spec in site_spec['nodes'] ]
+                                for node_spec in site_spec['nodes'] ]

         return node_infos
     
     def all_nodenames (self): return [ x[0] for x in self.all_node_infos() ]
         return node_infos
     
     def all_nodenames (self): return [ x[0] for x in self.all_node_infos() ]

+    def all_reservable_nodenames (self): 
+        res=[]
+        for site_spec in self.plc_spec['sites']:
+            for node_spec in site_spec['nodes']:
+                node_fields=node_spec['node_fields']
+                if 'node_type' in node_fields and node_fields['node_type']=='reservable':
+                    res.append(node_fields['hostname'])
+        return res

 
     # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
     def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period=15):
 
     # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
     def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period=15):


@@ -786,16 +934,16 @@ class TestPlc:
         # only useful in empty plcs
         return True
         
         # only useful in empty plcs
         return True
         

-    def nodes_ssh_debug(self):
+    def ssh_node_debug(self):

         "Tries to ssh into nodes in debug mode with the debug ssh key"
         "Tries to ssh into nodes in debug mode with the debug ssh key"

-        return self.check_nodes_ssh(debug=True,timeout_minutes=30,silent_minutes=5)
+        return self.check_nodes_ssh(debug=True,timeout_minutes=10,silent_minutes=5)

     
     

-    def nodes_ssh_boot(self):
+    def ssh_node_boot(self):

         "Tries to ssh into nodes in production mode with the root ssh key"
         "Tries to ssh into nodes in production mode with the root ssh key"

-        return self.check_nodes_ssh(debug=False,timeout_minutes=30,silent_minutes=15)
+        return self.check_nodes_ssh(debug=False,timeout_minutes=40,silent_minutes=15)

     
     @node_mapper
     
     @node_mapper

-    def init_node (self): 
+    def qemu_local_init (self): 

         "all nodes : init a clean local directory for holding node-dep stuff like iso image..."
         pass
     @node_mapper
         "all nodes : init a clean local directory for holding node-dep stuff like iso image..."
         pass
     @node_mapper


@@ -803,15 +951,27 @@ class TestPlc:
         "all nodes: invoke GetBootMedium and store result locally"
         pass
     @node_mapper
         "all nodes: invoke GetBootMedium and store result locally"
         pass
     @node_mapper

-    def configure_qemu (self): 
+    def qemu_local_config (self): 

         "all nodes: compute qemu config qemu.conf and store it locally"
         pass
     @node_mapper
         "all nodes: compute qemu config qemu.conf and store it locally"
         pass
     @node_mapper

-    def reinstall_node (self): 
+    def nodestate_reinstall (self): 

         "all nodes: mark PLCAPI boot_state as reinstall"
         pass
     @node_mapper
         "all nodes: mark PLCAPI boot_state as reinstall"
         pass
     @node_mapper

-    def export_qemu (self): 
+    def nodestate_safeboot (self): 
+        "all nodes: mark PLCAPI boot_state as safeboot"
+        pass
+    @node_mapper
+    def nodestate_boot (self): 
+        "all nodes: mark PLCAPI boot_state as boot"
+        pass
+    @node_mapper
+    def nodestate_show (self): 
+        "all nodes: show PLCAPI boot_state"
+        pass
+    @node_mapper
+    def qemu_export (self): 

         "all nodes: push local node-dep directory on the qemu box"
         pass
         
         "all nodes: push local node-dep directory on the qemu box"
         pass
         


@@ -829,9 +989,9 @@ class TestPlc:
     def do_check_initscripts(self):
         overall = True
         for slice_spec in self.plc_spec['slices']:
     def do_check_initscripts(self):
         overall = True
         for slice_spec in self.plc_spec['slices']:

-            if not slice_spec.has_key('initscriptname'):
+            if not slice_spec.has_key('initscriptstamp'):

                 continue
                 continue

-            initscript=slice_spec['initscriptname']
+            stamp=slice_spec['initscriptstamp']

             for nodename in slice_spec['nodenames']:
                 (site,node) = self.locate_node (nodename)
                 # xxx - passing the wrong site - probably harmless
             for nodename in slice_spec['nodenames']:
                 (site,node) = self.locate_node (nodename)
                 # xxx - passing the wrong site - probably harmless


@@ -839,7 +999,7 @@ class TestPlc:
                 test_slice = TestSlice (self,test_site,slice_spec)
                 test_node = TestNode (self,test_site,node)
                 test_sliver = TestSliver (self, test_node, test_slice)
                 test_slice = TestSlice (self,test_site,slice_spec)
                 test_node = TestNode (self,test_site,node)
                 test_sliver = TestSliver (self, test_node, test_slice)

-                if not test_sliver.check_initscript(initscript):
+                if not test_sliver.check_initscript_stamp(stamp):

                     overall = False
         return overall
            
                     overall = False
         return overall
            


@@ -854,7 +1014,7 @@ class TestPlc:
             self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
         return True
 
             self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
         return True
 

-    def clean_initscripts (self):
+    def delete_initscripts (self):

         "delete initscripts with PLCAPI"
         for initscript in self.plc_spec['initscripts']:
             initscript_name = initscript['initscript_fields']['name']
         "delete initscripts with PLCAPI"
         for initscript in self.plc_spec['initscripts']:
             initscript_name = initscript['initscript_fields']['name']


@@ -871,7 +1031,7 @@ class TestPlc:
         "create slices with PLCAPI"
         return self.do_slices()
 
         "create slices with PLCAPI"
         return self.do_slices()
 

-    def clean_slices (self):
+    def delete_slices (self):

         "delete slices with PLCAPI"
         return self.do_slices("delete")
 
         "delete slices with PLCAPI"
         return self.do_slices("delete")
 


@@ -889,18 +1049,23 @@ class TestPlc:
                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
         return True
         
                 utils.header('Created Slice %s'%slice['slice_fields']['name'])
         return True
         

-    @slice_mapper_options
-    def check_slice(self): 
+    @slice_mapper
+    def ssh_slice(self): 

         "tries to ssh-enter the slice with the user key, to ensure slice creation"
         pass
 
     @node_mapper
         "tries to ssh-enter the slice with the user key, to ensure slice creation"
         pass
 
     @node_mapper

-    def clear_known_hosts (self): 
+    def keys_clear_known_hosts (self): 

         "remove test nodes entries from the local known_hosts file"
         pass
     
     @node_mapper
         "remove test nodes entries from the local known_hosts file"
         pass
     
     @node_mapper

-    def start_node (self) : 
+    def qemu_start (self) : 
+        "all nodes: start the qemu instance (also runs qemu-bridge-init start)"
+        pass
+
+    @node_mapper
+    def timestamp_qemu (self) : 

         "all nodes: start the qemu instance (also runs qemu-bridge-init start)"
         pass
 
         "all nodes: start the qemu instance (also runs qemu-bridge-init start)"
         pass
 


@@ -938,24 +1103,97 @@ class TestPlc:
     # in particular runs with --preserve (dont cleanup) and without --check
     # also it gets run twice, once with the --foreign option for creating fake foreign entries
 
     # in particular runs with --preserve (dont cleanup) and without --check
     # also it gets run twice, once with the --foreign option for creating fake foreign entries
 

-    ### install_sfa_rpm
-    def install_sfa(self):
+    ### sfa_install_rpm
+    def sfa_install(self):

         "yum install sfa, sfa-plc and sfa-client"
         "yum install sfa, sfa-plc and sfa-client"

-        if self.options.personality == "linux32":
-            arch = "i386"
-        elif self.options.personality == "linux64":
-            arch = "x86_64"
-        else:
-            raise Exception, "Unsupported personality %r"%self.options.personality
-        return self.run_in_guest("yum -y install sfa sfa-client sfa-plc sfa-sfatables")==0
+        # ignore yum retcod
+        self.run_in_guest("yum -y install sfa sfa-client sfa-plc sfa-sfatables")
+        return  self.run_in_guest("rpm -q sfa sfa-client sfa-plc sfa-sfatables")==0
+        
+
+    def sfa_dbclean(self):
+        "thoroughly wipes off the SFA database"
+        self.run_in_guest("sfa-nuke-plc.py")==0
+        return True
+
+    def sfa_plcclean(self):
+        "cleans the PLC entries that were created as a side effect of running the script"
+        # ignore result 
+        sfa_spec=self.plc_spec['sfa']
+
+        slicename='%s_%s'%(sfa_spec['login_base'],sfa_spec['slicename'])
+        try: self.apiserver.DeleteSlice(self.auth_root(),slicename)
+        except: print "Slice %s already absent from PLC db"%slicename
+
+        username="%s@%s"%(sfa_spec['regularuser'],sfa_spec['domain'])
+        try: self.apiserver.DeletePerson(self.auth_root(),username)
+        except: print "User %s already absent from PLC db"%username
+
+        print "REMEMBER TO RUN sfa_import AGAIN"
+        return True
+
+    def sfa_uninstall(self):
+        "uses rpm to uninstall sfa - ignore result"
+        self.run_in_guest("rpm -e sfa sfa-sfatables sfa-client sfa-plc")
+        self.run_in_guest("rm -rf /var/lib/sfa")
+        self.run_in_guest("rm -rf /etc/sfa")
+        self.run_in_guest("rm -rf /var/log/sfa_access.log /var/log/sfa_import_plc.log /var/log/sfa.daemon")
+        # xxx tmp 
+        self.run_in_guest("rpm -e --noscripts sfa-plc")
+        return True
+
+    ### run unit tests for SFA
+    # NOTE: for some reason on f14/i386, yum install sfa-tests fails for no reason
+    # Running Transaction
+    # Transaction couldn't start:
+    # installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem
+    # [('installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem', (9, '/', 208896L))]
+    # no matter how many Gbs are available on the testplc
+    # could not figure out what's wrong, so...
+    # if the yum install phase fails, consider the test is successful
+    # other combinations will eventually run it hopefully
+    def sfa_utest(self):
+        "yum install sfa-tests and run SFA unittests"
+        self.run_in_guest("yum -y install sfa-tests")
+        # failed to install - forget it
+        if self.run_in_guest("rpm -q sfa-tests")!=0: 
+            utils.header("WARNING: SFA unit tests failed to install, ignoring")
+            return True
+        return self.run_in_guest("/usr/share/sfa/tests/testAll.py")==0

 
     ###
 
     ###

-    def configure_sfa(self):
+    def confdir(self):
+        dirname="conf.%s"%self.plc_spec['name']
+        if not os.path.isdir(dirname):
+            utils.system("mkdir -p %s"%dirname)
+        if not os.path.isdir(dirname):
+            raise "Cannot create config dir for plc %s"%self.name()
+        return dirname
+
+    def conffile(self,filename):
+        return "%s/%s"%(self.confdir(),filename)
+    def confsubdir(self,dirname,clean,dry_run=False):
+        subdirname="%s/%s"%(self.confdir(),dirname)
+        if clean:
+            utils.system("rm -rf %s"%subdirname)
+        if not os.path.isdir(subdirname): 
+            utils.system("mkdir -p %s"%subdirname)
+        if not dry_run and not os.path.isdir(subdirname):
+            raise "Cannot create config subdir %s for plc %s"%(dirname,self.name())
+        return subdirname
+        
+    def conffile_clean (self,filename):
+        filename=self.conffile(filename)
+        return utils.system("rm -rf %s"%filename)==0
+    
+    ###
+    def sfa_configure(self):

         "run sfa-config-tty"
         "run sfa-config-tty"

-        tmpname='%s.sfa-config-tty'%(self.name())
+        tmpname=self.conffile("sfa-config-tty")

         fileconf=open(tmpname,'w')
         for var in [ 'SFA_REGISTRY_ROOT_AUTH',
         fileconf=open(tmpname,'w')
         for var in [ 'SFA_REGISTRY_ROOT_AUTH',

-                     'SFA_REGISTRY_LEVEL1_AUTH',
+                     'SFA_INTERFACE_HRN',
+#                     'SFA_REGISTRY_LEVEL1_AUTH',

                     'SFA_REGISTRY_HOST',
                     'SFA_AGGREGATE_HOST',
                      'SFA_SM_HOST',
                     'SFA_REGISTRY_HOST',
                     'SFA_AGGREGATE_HOST',
                      'SFA_SM_HOST',


@@ -964,145 +1202,190 @@ class TestPlc:
                      'SFA_PLC_DB_HOST',
                      'SFA_PLC_DB_USER',
                      'SFA_PLC_DB_PASSWORD',
                      'SFA_PLC_DB_HOST',
                      'SFA_PLC_DB_USER',
                      'SFA_PLC_DB_PASSWORD',

-                    'SFA_PLC_URL']:
+                    'SFA_PLC_URL',
+                     ]:

             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec['sfa'][var]))
             fileconf.write ('e %s\n%s\n'%(var,self.plc_spec['sfa'][var]))

+        # the way plc_config handles booleans just sucks..
+        for var in ['SFA_API_DEBUG']:
+            val='false'
+            if self.plc_spec['sfa'][var]: val='true'
+            fileconf.write ('e %s\n%s\n'%(var,val))

         fileconf.write('w\n')
         fileconf.write('R\n')
         fileconf.write('q\n')
         fileconf.close()
         utils.system('cat %s'%tmpname)
         self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
         fileconf.write('w\n')
         fileconf.write('R\n')
         fileconf.write('q\n')
         fileconf.close()
         utils.system('cat %s'%tmpname)
         self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')

-        utils.system('rm %s'%tmpname)

         return True
 
         return True
 

-    def import_sfa(self):
+    def aggregate_xml_line(self):
+        port=self.plc_spec['sfa']['neighbours-port']
+        return '<aggregate addr="%s" hrn="%s" port="%r"/>' % \
+            (self.vserverip,self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH'],port)
+
+    def registry_xml_line(self):
+        return '<registry addr="%s" hrn="%s" port="12345"/>' % \
+            (self.vserverip,self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH'])
+
+
+    # a cross step that takes all other plcs in argument
+    def cross_sfa_configure(self, other_plcs):
+        "writes aggregates.xml and registries.xml that point to all other PLCs in the test"
+        # of course with a single plc, other_plcs is an empty list
+        if not other_plcs:
+            return True
+        agg_fname=self.conffile("agg.xml")
+        file(agg_fname,"w").write("<aggregates>%s</aggregates>\n" % \
+                                     " ".join([ plc.aggregate_xml_line() for plc in other_plcs ]))
+        utils.header ("(Over)wrote %s"%agg_fname)
+        reg_fname=self.conffile("reg.xml")
+        file(reg_fname,"w").write("<registries>%s</registries>\n" % \
+                                     " ".join([ plc.registry_xml_line() for plc in other_plcs ]))
+        utils.header ("(Over)wrote %s"%reg_fname)
+        return self.test_ssh.copy_abs(agg_fname,'/vservers/%s/etc/sfa/aggregates.xml'%self.vservername)==0 \
+            and  self.test_ssh.copy_abs(reg_fname,'/vservers/%s/etc/sfa/registries.xml'%self.vservername)==0
+
+    def sfa_import(self):

         "sfa-import-plc"
        auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
         return self.run_in_guest('sfa-import-plc.py')==0
 # not needed anymore
 #        self.run_in_guest('cp /etc/sfa/authorities/%s/%s.pkey /etc/sfa/authorities/server.key'%(auth,auth))
 
         "sfa-import-plc"
        auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
         return self.run_in_guest('sfa-import-plc.py')==0
 # not needed anymore
 #        self.run_in_guest('cp /etc/sfa/authorities/%s/%s.pkey /etc/sfa/authorities/server.key'%(auth,auth))
 

-    def start_sfa(self):
+    def sfa_start(self):

         "service sfa start"
         return self.run_in_guest('service sfa start')==0
 
         "service sfa start"
         return self.run_in_guest('service sfa start')==0
 

-    def setup_sfa(self):
-        "sfi client configuration"
-       dir_name=".sfi"
-       if os.path.exists(dir_name):
-           utils.system('rm -rf %s'%dir_name)
-       utils.system('mkdir %s'%dir_name)
-       file_name=dir_name + os.sep + 'fake-pi1.pkey'
+    def sfi_configure(self):
+        "Create /root/.sfi on the plc side for sfi client configuration"
+        sfa_spec=self.plc_spec['sfa']
+       dir_name=self.confsubdir("dot-sfi",clean=True,dry_run=self.options.dry_run)
+        if self.options.dry_run: return True
+       file_name=dir_name + os.sep + sfa_spec['piuser'] + '.pkey'

         fileconf=open(file_name,'w')
         fileconf.write (self.plc_spec['keys'][0]['private'])
         fileconf.close()
         fileconf=open(file_name,'w')
         fileconf.write (self.plc_spec['keys'][0]['private'])
         fileconf.close()

+        utils.header ("(Over)wrote %s"%file_name)

 
        file_name=dir_name + os.sep + 'sfi_config'
         fileconf=open(file_name,'w')
 
        file_name=dir_name + os.sep + 'sfi_config'
         fileconf=open(file_name,'w')

-       SFI_AUTH=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']+".main"
+       SFI_AUTH="%s.%s"%(sfa_spec['SFA_REGISTRY_ROOT_AUTH'],sfa_spec['login_base'])

         fileconf.write ("SFI_AUTH='%s'"%SFI_AUTH)
        fileconf.write('\n')
         fileconf.write ("SFI_AUTH='%s'"%SFI_AUTH)
        fileconf.write('\n')

-       SFI_USER=SFI_AUTH+'.fake-pi1'
+       SFI_USER=SFI_AUTH + '.' + sfa_spec['piuser']

         fileconf.write ("SFI_USER='%s'"%SFI_USER)
        fileconf.write('\n')
         fileconf.write ("SFI_USER='%s'"%SFI_USER)
        fileconf.write('\n')

-       SFI_REGISTRY='http://' + self.plc_spec['sfa']['SFA_PLC_DB_HOST'] + ':12345/'
+       SFI_REGISTRY='http://' + sfa_spec['SFA_PLC_DB_HOST'] + ':12345/'

         fileconf.write ("SFI_REGISTRY='%s'"%SFI_REGISTRY)
        fileconf.write('\n')
         fileconf.write ("SFI_REGISTRY='%s'"%SFI_REGISTRY)
        fileconf.write('\n')

-       SFI_SM='http://' + self.plc_spec['sfa']['SFA_PLC_DB_HOST'] + ':12347/'
+       SFI_SM='http://' + sfa_spec['SFA_PLC_DB_HOST'] + ':12347/'

         fileconf.write ("SFI_SM='%s'"%SFI_SM)
        fileconf.write('\n')
         fileconf.close()
         fileconf.write ("SFI_SM='%s'"%SFI_SM)
        fileconf.write('\n')
         fileconf.close()

+        utils.header ("(Over)wrote %s"%file_name)

 
        file_name=dir_name + os.sep + 'person.xml'
         fileconf=open(file_name,'w')
 
        file_name=dir_name + os.sep + 'person.xml'
         fileconf=open(file_name,'w')

-       for record in self.plc_spec['sfa']['sfa_person_xml']:
+       for record in sfa_spec['sfa_person_xml']:

           person_record=record
        fileconf.write(person_record)
        fileconf.write('\n')
         fileconf.close()
           person_record=record
        fileconf.write(person_record)
        fileconf.write('\n')
         fileconf.close()

+        utils.header ("(Over)wrote %s"%file_name)

 
        file_name=dir_name + os.sep + 'slice.xml'
         fileconf=open(file_name,'w')
 
        file_name=dir_name + os.sep + 'slice.xml'
         fileconf=open(file_name,'w')

-       for record in self.plc_spec['sfa']['sfa_slice_xml']:
+       for record in sfa_spec['sfa_slice_xml']:

            slice_record=record
            slice_record=record

-       #slice_record=self.plc_spec['sfa']['sfa_slice_xml']
+       #slice_record=sfa_spec['sfa_slice_xml']

        fileconf.write(slice_record)
        fileconf.write('\n')
        fileconf.write(slice_record)
        fileconf.write('\n')

+        utils.header ("(Over)wrote %s"%file_name)

         fileconf.close()
 
        file_name=dir_name + os.sep + 'slice.rspec'
         fileconf=open(file_name,'w')
        slice_rspec=''
         fileconf.close()
 
        file_name=dir_name + os.sep + 'slice.rspec'
         fileconf=open(file_name,'w')
        slice_rspec=''

-       for (key, value) in self.plc_spec['sfa']['sfa_slice_rspec'].items():
+       for (key, value) in sfa_spec['sfa_slice_rspec'].items():

            slice_rspec +=value 
        fileconf.write(slice_rspec)
        fileconf.write('\n')
         fileconf.close()
            slice_rspec +=value 
        fileconf.write(slice_rspec)
        fileconf.write('\n')
         fileconf.close()

-        location = "root/"
+        utils.header ("(Over)wrote %s"%file_name)
+        
+        # push to the remote root's .sfi
+        location = "root/.sfi"

         remote="/vservers/%s/%s"%(self.vservername,location)
        self.test_ssh.copy_abs(dir_name, remote, recursive=True)
 
         remote="/vservers/%s/%s"%(self.vservername,location)
        self.test_ssh.copy_abs(dir_name, remote, recursive=True)
 

-        #utils.system('cat %s'%tmpname)
-        utils.system('rm -rf %s'%dir_name)

         return True
 
         return True
 

-    def add_sfa(self):
-        "run sfi.py add (on Registry) and sfi.py create (on SM) to form new objects"
-       test_plc=self
-        test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
-        success=test_user_sfa.add_user()
+    def sfi_clean (self):
+        "clean up /root/.sfi on the plc side"
+        self.run_in_guest("rm -rf /root/.sfi")
+        return True

 
 

-       for slice_spec in self.plc_spec['sfa']['slices_sfa']:
-            site_spec = self.locate_site (slice_spec['sitename'])
-            test_site = TestSite(self,site_spec)
-           test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
-           success1=test_slice_sfa.add_slice()
-           success2=test_slice_sfa.create_slice()
-       return success and success1 and success2
-
-    def update_sfa(self):
-        "run sfi.py update (on Registry) and sfi.py create (on SM) on existing objects"
-       test_plc=self
-       test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
-       success1=test_user_sfa.update_user()
-       
-       for slice_spec in self.plc_spec['sfa']['slices_sfa']:
-           site_spec = self.locate_site (slice_spec['sitename'])
-           test_site = TestSite(self,site_spec)
-           test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
-           success2=test_slice_sfa.update_slice()
-       return success1 and success2
-
-    def view_sfa(self):
+    def sfa_add_user(self):
+        "run sfi.py add using person.xml"
+        return TestUserSfa(self).add_user()
+
+    def sfa_update_user(self):
+        "run sfi.py update using person.xml"
+        return TestUserSfa(self).update_user()
+
+    @slice_sfa_mapper
+    def sfa_add_slice(self):
+        "run sfi.py add (on Registry) from slice.xml"
+        pass
+
+    @slice_sfa_mapper
+    def sfa_discover(self):
+        "discover resources into resouces_in.rspec"
+        pass
+
+    @slice_sfa_mapper
+    def sfa_create_slice(self):
+        "run sfi.py create (on SM) - 1st time"
+        pass
+
+    @slice_sfa_mapper
+    def sfa_check_slice_plc(self):
+        "check sfa_create_slice at the plcs - all local nodes should be in slice"
+        pass
+
+    @slice_sfa_mapper
+    def sfa_update_slice(self):
+        "run sfi.py create (on SM) on existing object"
+        pass
+
+    def sfa_view(self):

         "run sfi.py list and sfi.py show (both on Registry) and sfi.py slices and sfi.py resources (both on SM)"
         "run sfi.py list and sfi.py show (both on Registry) and sfi.py slices and sfi.py resources (both on SM)"

-       auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
+        sfa_spec=self.plc_spec['sfa']
+       auth=sfa_spec['SFA_REGISTRY_ROOT_AUTH']

        return \
        return \

-       self.run_in_guest("sfi.py -d /root/.sfi/ list %s.main"%auth)==0 and \
-       self.run_in_guest("sfi.py -d /root/.sfi/ show %s.main"%auth)==0 and \
+       self.run_in_guest("sfi.py -d /root/.sfi/ list %s.%s"%(auth,sfa_spec['login_base']))==0 and \
+       self.run_in_guest("sfi.py -d /root/.sfi/ show %s.%s"%(auth,sfa_spec['login_base']))==0 and \

        self.run_in_guest("sfi.py -d /root/.sfi/ slices")==0 and \
        self.run_in_guest("sfi.py -d /root/.sfi/ resources -o resources")==0
 
        self.run_in_guest("sfi.py -d /root/.sfi/ slices")==0 and \
        self.run_in_guest("sfi.py -d /root/.sfi/ resources -o resources")==0
 

-    @slice_mapper_options_sfa
-    def check_slice_sfa(self): 
+    @slice_sfa_mapper
+    def ssh_slice_sfa(self): 

        "tries to ssh-enter the SFA slice"
         pass
 
        "tries to ssh-enter the SFA slice"
         pass
 

-    def delete_sfa(self):
-       "run sfi.py delete (on SM), sfi.py remove (on Registry)"
-       test_plc=self
-       test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
-       success1=test_user_sfa.delete_user()
-       for slice_spec in self.plc_spec['sfa']['slices_sfa']:
-            site_spec = self.locate_site (slice_spec['sitename'])
-            test_site = TestSite(self,site_spec)
-           test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
-           success2=test_slice_sfa.delete_slice()
+    def sfa_delete_user(self):
+       "run sfi.py delete (on SM) for user"
+       test_user_sfa=TestUserSfa(self)
+       return test_user_sfa.delete_user()

 
 

-       return success1 and success2
+    @slice_sfa_mapper
+    def sfa_delete_slice(self):
+       "run sfi.py delete (on SM), sfi.py remove (on Registry) to clean slices"
+        pass

 
 

-    def stop_sfa(self):
+    def sfa_stop(self):

         "service sfa stop"
         "service sfa stop"

-        self.run_in_guest('service sfa stop')
+        self.run_in_guest('service sfa stop')==0

         return True
 
     def populate (self):
         return True
 
     def populate (self):


@@ -1194,20 +1477,22 @@ class TestPlc:
             name=str(d)
         return "/root/%s-%s.sql"%(database,name)
 
             name=str(d)
         return "/root/%s-%s.sql"%(database,name)
 

-    def db_dump(self):
-        dump=self.dbfile("planetab4")
-        self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
-        utils.header('Dumped planetlab4 database in %s'%dump)
+    def plc_db_dump(self):
+        'dump the planetlab5 DB in /root in the PLC - filename has time'
+        dump=self.dbfile("planetab5")
+        self.run_in_guest('pg_dump -U pgsqluser planetlab5 -f '+ dump)
+        utils.header('Dumped planetlab5 database in %s'%dump)

         return True
 
         return True
 

-    def db_restore(self):
-        dump=self.dbfile("planetab4")
+    def plc_db_restore(self):
+        'restore the planetlab5 DB - looks broken, but run -n might help'
+        dump=self.dbfile("planetab5")

         ##stop httpd service
         self.run_in_guest('service httpd stop')
         # xxx - need another wrapper
         ##stop httpd service
         self.run_in_guest('service httpd stop')
         # xxx - need another wrapper

-        self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
-        self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
-        self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
+        self.run_in_guest_piped('echo drop database planetlab5','psql --user=pgsqluser template1')
+        self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab5')
+        self.run_in_guest('psql -U pgsqluser planetlab5 -f '+dump)

         ##starting httpd service
         self.run_in_guest('service httpd start')
 
         ##starting httpd service
         self.run_in_guest('service httpd start')