system/TestPlc.py

   1 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
   2 # Copyright (C) 2010 INRIA
   3 #
   4 import sys
   5 import time
   6 import os, os.path
   7 import traceback
   8 import socket
   9 from datetime import datetime, timedelta
  10
  11 import utils
  12 from Completer import Completer, CompleterTask
  13 from TestSite import TestSite
  14 from TestNode import TestNode, CompleterTaskNodeSsh
  15 from TestUser import TestUser
  16 from TestKey import TestKey
  17 from TestSlice import TestSlice
  18 from TestSliver import TestSliver
  19 from TestBoxQemu import TestBoxQemu
  20 from TestSsh import TestSsh
  21 from TestApiserver import TestApiserver
  22 from TestAuthSfa import TestAuthSfa
  23 from PlcapiUrlScanner import PlcapiUrlScanner
  24
  25 from TestBonding import TestBonding
  26
  27 has_sfa_cache_filename="sfa-cache"
  28
  29 # step methods must take (self) and return a boolean (options is a member of the class)
  30
  31 def standby(minutes, dry_run):
  32     utils.header('Entering StandBy for {:d} mn'.format(minutes))
  33     if dry_run:
  34         print('dry_run')
  35     else:
  36         time.sleep(60*minutes)
  37     return True
  38
  39 def standby_generic(func):
  40     def actual(self):
  41         minutes = int(func.__name__.split("_")[1])
  42         return standby(minutes, self.options.dry_run)
  43     return actual
  44
  45 def node_mapper(method):
  46     def map_on_nodes(self, *args, **kwds):
  47         overall = True
  48         node_method = TestNode.__dict__[method.__name__]
  49         for test_node in self.all_nodes():
  50             if not node_method(test_node, *args, **kwds):
  51                 overall=False
  52         return overall
  53     # maintain __name__ for ignore_result
  54     map_on_nodes.__name__ = method.__name__
  55     # restore the doc text
  56     map_on_nodes.__doc__ = TestNode.__dict__[method.__name__].__doc__
  57     return map_on_nodes
  58
  59 def slice_mapper(method):
  60     def map_on_slices(self):
  61         overall = True
  62         slice_method = TestSlice.__dict__[method.__name__]
  63         for slice_spec in self.plc_spec['slices']:
  64             site_spec = self.locate_site (slice_spec['sitename'])
  65             test_site = TestSite(self,site_spec)
  66             test_slice = TestSlice(self,test_site,slice_spec)
  67             if not slice_method(test_slice, self.options):
  68                 overall=False
  69         return overall
  70     # maintain __name__ for ignore_result
  71     map_on_slices.__name__ = method.__name__
  72     # restore the doc text
  73     map_on_slices.__doc__ = TestSlice.__dict__[method.__name__].__doc__
  74     return map_on_slices
  75
  76 def bonding_redirector(method):
  77     bonding_name = method.__name__.replace('bonding_', '')
  78     def redirect(self):
  79         bonding_method = TestBonding.__dict__[bonding_name]
  80         return bonding_method(self.test_bonding)
  81     # maintain __name__ for ignore_result
  82     redirect.__name__ = method.__name__
  83     # restore the doc text
  84     redirect.__doc__ = TestBonding.__dict__[bonding_name].__doc__
  85     return redirect
  86
  87 # run a step but return True so that we can go on
  88 def ignore_result(method):
  89     def ignoring(self):
  90         # ssh_slice_ignore->ssh_slice
  91         ref_name = method.__name__.replace('_ignore', '').replace('force_', '')
  92         ref_method = TestPlc.__dict__[ref_name]
  93         result = ref_method(self)
  94         print("Actual (but ignored) result for {ref_name} is {result}".format(**locals()))
  95         return Ignored(result)
  96     name = method.__name__.replace('_ignore', '').replace('force_', '')
  97     ignoring.__name__ = name
  98     ignoring.__doc__ = "ignored version of " + name
  99     return ignoring
 100
 101 # a variant that expects the TestSlice method to return a list of CompleterTasks that
 102 # are then merged into a single Completer run to avoid wating for all the slices
 103 # esp. useful when a test fails of course
 104 # because we need to pass arguments we use a class instead..
 105 class slice_mapper__tasks(object):
 106     # could not get this to work with named arguments
 107     def __init__(self, timeout_minutes, silent_minutes, period_seconds):
 108         self.timeout = timedelta(minutes = timeout_minutes)
 109         self.silent = timedelta(minutes = silent_minutes)
 110         self.period = timedelta(seconds = period_seconds)
 111     def __call__(self, method):
 112         decorator_self=self
 113         # compute augmented method name
 114         method_name = method.__name__ + "__tasks"
 115         # locate in TestSlice
 116         slice_method = TestSlice.__dict__[ method_name ]
 117         def wrappee(self):
 118             tasks=[]
 119             for slice_spec in self.plc_spec['slices']:
 120                 site_spec = self.locate_site (slice_spec['sitename'])
 121                 test_site = TestSite(self, site_spec)
 122                 test_slice = TestSlice(self, test_site, slice_spec)
 123                 tasks += slice_method (test_slice, self.options)
 124             return Completer (tasks, message=method.__name__).\
 125                 run(decorator_self.timeout, decorator_self.silent, decorator_self.period)
 126         # restore the doc text from the TestSlice method even if a bit odd
 127         wrappee.__name__ = method.__name__
 128         wrappee.__doc__ = slice_method.__doc__
 129         return wrappee
 130
 131 def auth_sfa_mapper(method):
 132     def actual(self):
 133         overall = True
 134         auth_method = TestAuthSfa.__dict__[method.__name__]
 135         for auth_spec in self.plc_spec['sfa']['auth_sfa_specs']:
 136             test_auth = TestAuthSfa(self, auth_spec)
 137             if not auth_method(test_auth, self.options):
 138                 overall=False
 139         return overall
 140     # restore the doc text
 141     actual.__doc__ = TestAuthSfa.__dict__[method.__name__].__doc__
 142     return actual
 143
 144 class Ignored:
 145     def __init__(self, result):
 146         self.result = result
 147
 148 SEP = '<sep>'
 149 SEPSFA = '<sep_sfa>'
 150
 151 class TestPlc:
 152
 153     default_steps = [
 154         'show', SEP,
 155         'plcvm_delete','plcvm_timestamp','plcvm_create', SEP,
 156         'plc_install', 'plc_configure', 'plc_start', SEP,
 157         'keys_fetch', 'keys_store', 'keys_clear_known_hosts', SEP,
 158         'plcapi_urls','speed_up_slices', SEP,
 159         'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', 'leases', SEP,
 160 # slices created under plcsh interactively seem to be fine but these ones don't have the tags
 161 # keep this our of the way for now
 162         'check_vsys_defaults_ignore', SEP,
 163 # run this first off so it's easier to re-run on another qemu box
 164         'qemu_kill_mine', 'nodestate_reinstall', 'qemu_local_init','bootcd', 'qemu_local_config', SEP,
 165         'qemu_clean_mine', 'qemu_export', 'qemu_cleanlog', SEP,
 166         'qemu_start', 'qemu_timestamp', 'qemu_nodefamily', SEP,
 167         'sfa_install_all', 'sfa_configure', 'cross_sfa_configure', 'sfa_start', 'sfa_import', SEPSFA,
 168         'sfi_configure@1', 'sfa_register_site@1','sfa_register_pi@1', SEPSFA,
 169         'sfa_register_user@1', 'sfa_update_user@1', 'sfa_register_slice@1', 'sfa_renew_slice@1', SEPSFA,
 170         'sfa_remove_user_from_slice@1','sfi_show_slice_researchers@1',
 171         'sfa_insert_user_in_slice@1','sfi_show_slice_researchers@1', SEPSFA,
 172         'sfa_discover@1', 'sfa_rspec@1', SEPSFA,
 173         'sfa_allocate@1', 'sfa_provision@1', 'sfa_describe@1', SEPSFA,
 174         'sfa_check_slice_plc@1', 'sfa_update_slice@1', SEPSFA,
 175         'sfi_list@1', 'sfi_show_site@1', 'sfa_utest@1', SEPSFA,
 176         # we used to run plcsh_stress_test, and then ssh_node_debug and ssh_node_boot
 177         # but as the stress test might take a while, we sometimes missed the debug mode..
 178         'probe_kvm_iptables',
 179         'ping_node', 'ssh_node_debug', 'plcsh_stress_test@1', SEP,
 180         'ssh_node_boot', 'node_bmlogs', 'ssh_slice', 'ssh_slice_basics', 'check_initscripts', SEP,
 181         'ssh_slice_sfa@1', SEPSFA,
 182         'sfa_rspec_empty@1', 'sfa_allocate_empty@1', 'sfa_provision_empty@1','sfa_check_slice_plc_empty@1', SEPSFA,
 183         'sfa_delete_slice@1', 'sfa_delete_user@1', SEPSFA,
 184         'cross_check_tcp@1', 'check_system_slice', SEP,
 185         # for inspecting the slice while it runs the first time
 186         #'fail',
 187         # check slices are turned off properly
 188         'empty_slices', 'ssh_slice_off', 'slice_fs_deleted_ignore', SEP,
 189         # check they are properly re-created with the same name
 190         'fill_slices', 'ssh_slice_again', SEP,
 191         'gather_logs_force', SEP,
 192         ]
 193     other_steps = [
 194         'export', 'show_boxes', 'super_speed_up_slices', SEP,
 195         'check_hooks', 'plc_stop', 'plcvm_start', 'plcvm_stop', SEP,
 196         'delete_initscripts', 'delete_nodegroups','delete_all_sites', SEP,
 197         'delete_sites', 'delete_nodes', 'delete_slices', 'keys_clean', SEP,
 198         'delete_leases', 'list_leases', SEP,
 199         'populate', SEP,
 200         'nodestate_show','nodestate_safeboot','nodestate_boot', 'nodestate_upgrade', SEP,
 201         'nodeflavour_show','nodedistro_f14','nodedistro_f18', SEP,
 202         'nodedistro_f20', 'nodedistro_f21','nodedistro_f22', SEP,
 203         'nodeplain_on','nodeplain_off','nodeplain_show', SEP,
 204         'qemu_list_all', 'qemu_list_mine', 'qemu_kill_all', SEP,
 205         'sfa_install_core', 'sfa_install_sfatables', 'sfa_install_plc', 'sfa_install_client', SEPSFA,
 206         'sfa_plcclean', 'sfa_dbclean', 'sfa_stop','sfa_uninstall', 'sfi_clean', SEPSFA,
 207         'sfa_get_expires', SEPSFA,
 208         'plc_db_dump' , 'plc_db_restore', SEP,
 209         'check_netflow','check_drl', SEP,
 210         'debug_nodemanager', 'slice_fs_present', SEP,
 211         'standby_1_through_20','yes','no',SEP,
 212         'install_syslinux6', 'bonding_builds', 'bonding_nodes', SEP,
 213         ]
 214     default_bonding_steps = [
 215         'bonding_init_partial',
 216         'bonding_add_yum',
 217         'bonding_install_rpms', SEP,
 218         ]
 219
 220     @staticmethod
 221     def printable_steps(list):
 222         single_line = " ".join(list) + " "
 223         return single_line.replace(" "+SEP+" ", " \\\n").replace(" "+SEPSFA+" ", " \\\n")
 224     @staticmethod
 225     def valid_step(step):
 226         return step != SEP and step != SEPSFA
 227
 228     # turn off the sfa-related steps when build has skipped SFA
 229     # this was originally for centos5 but is still valid
 230     # for up to f12 as recent SFAs with sqlalchemy won't build before f14
 231     @staticmethod
 232     def _has_sfa_cached(rpms_url):
 233         if os.path.isfile(has_sfa_cache_filename):
 234             with open(has_sfa_cache_filename) as cache:
 235                 cached = cache.read() == "yes"
 236             utils.header("build provides SFA (cached):{}".format(cached))
 237             return cached
 238         # warning, we're now building 'sface' so let's be a bit more picky
 239         # full builds are expected to return with 0 here
 240         utils.header("Checking if build provides SFA package...")
 241         retcod = utils.system("curl --silent {}/ | grep -q sfa-".format(rpms_url)) == 0
 242         encoded = 'yes' if retcod else 'no'
 243         with open(has_sfa_cache_filename,'w') as cache:
 244             cache.write(encoded)
 245         return retcod
 246
 247     @staticmethod
 248     def check_whether_build_has_sfa(rpms_url):
 249         has_sfa = TestPlc._has_sfa_cached(rpms_url)
 250         if has_sfa:
 251             utils.header("build does provide SFA")
 252         else:
 253             # move all steps containing 'sfa' from default_steps to other_steps
 254             utils.header("SFA package not found - removing steps with sfa or sfi")
 255             sfa_steps = [ step for step in TestPlc.default_steps
 256                           if step.find('sfa') >= 0 or step.find("sfi") >= 0 ]
 257             TestPlc.other_steps += sfa_steps
 258             for step in sfa_steps:
 259                 TestPlc.default_steps.remove(step)
 260
 261     def __init__(self, plc_spec, options):
 262         self.plc_spec = plc_spec
 263         self.options = options
 264         self.test_ssh = TestSsh(self.plc_spec['host_box'], self.options.buildname)
 265         self.vserverip = plc_spec['vserverip']
 266         self.vservername = plc_spec['vservername']
 267         self.vplchostname = self.vservername.split('-')[-1]
 268         self.url = "https://{}:443/PLCAPI/".format(plc_spec['vserverip'])
 269         self.apiserver = TestApiserver(self.url, options.dry_run)
 270         (self.ssh_node_boot_timeout, self.ssh_node_boot_silent) = plc_spec['ssh_node_boot_timers']
 271         (self.ssh_node_debug_timeout, self.ssh_node_debug_silent) = plc_spec['ssh_node_debug_timers']
 272
 273     def has_addresses_api(self):
 274         return self.apiserver.has_method('AddIpAddress')
 275
 276     def name(self):
 277         name = self.plc_spec['name']
 278         return "{}.{}".format(name,self.vservername)
 279
 280     def hostname(self):
 281         return self.plc_spec['host_box']
 282
 283     def is_local(self):
 284         return self.test_ssh.is_local()
 285
 286     # define the API methods on this object through xmlrpc
 287     # would help, but not strictly necessary
 288     def connect(self):
 289         pass
 290
 291     def actual_command_in_guest(self,command, backslash=False):
 292         raw1 = self.host_to_guest(command)
 293         raw2 = self.test_ssh.actual_command(raw1, dry_run=self.options.dry_run, backslash=backslash)
 294         return raw2
 295
 296     def start_guest(self):
 297       return utils.system(self.test_ssh.actual_command(self.start_guest_in_host(),
 298                                                        dry_run=self.options.dry_run))
 299
 300     def stop_guest(self):
 301       return utils.system(self.test_ssh.actual_command(self.stop_guest_in_host(),
 302                                                        dry_run=self.options.dry_run))
 303
 304     def run_in_guest(self, command, backslash=False):
 305         raw = self.actual_command_in_guest(command, backslash)
 306         return utils.system(raw)
 307
 308     def run_in_host(self,command):
 309         return self.test_ssh.run_in_buildname(command, dry_run=self.options.dry_run)
 310
 311     # backslashing turned out so awful at some point that I've turned off auto-backslashing
 312     # see e.g. plc_start esp. the version for f14
 313     #command gets run in the plc's vm
 314     def host_to_guest(self, command):
 315         ssh_leg = TestSsh(self.vplchostname)
 316         return ssh_leg.actual_command(command, keep_stdin=True)
 317
 318     # this /vservers thing is legacy...
 319     def vm_root_in_host(self):
 320         return "/vservers/{}/".format(self.vservername)
 321
 322     def vm_timestamp_path(self):
 323         return "/vservers/{}/{}.timestamp".format(self.vservername, self.vservername)
 324
 325     #start/stop the vserver
 326     def start_guest_in_host(self):
 327         return "virsh -c lxc:/// start {}".format(self.vservername)
 328
 329     def stop_guest_in_host(self):
 330         return "virsh -c lxc:/// destroy {}".format(self.vservername)
 331
 332     # xxx quick n dirty
 333     def run_in_guest_piped(self,local,remote):
 334         return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),
 335                                                                      keep_stdin = True))
 336
 337     def yum_check_installed(self, rpms):
 338         if isinstance(rpms, list):
 339             rpms=" ".join(rpms)
 340         return self.run_in_guest("rpm -q {}".format(rpms)) == 0
 341
 342     # does a yum install in the vs, ignore yum retcod, check with rpm
 343     def yum_install(self, rpms):
 344         if isinstance(rpms, list):
 345             rpms=" ".join(rpms)
 346         self.run_in_guest("yum -y install {}".format(rpms))
 347         # yum-complete-transaction comes with yum-utils, that is in vtest.pkgs
 348         self.run_in_guest("yum-complete-transaction -y")
 349         return self.yum_check_installed(rpms)
 350
 351     def auth_root(self):
 352         return {'Username'   : self.plc_spec['settings']['PLC_ROOT_USER'],
 353                 'AuthMethod' : 'password',
 354                 'AuthString' : self.plc_spec['settings']['PLC_ROOT_PASSWORD'],
 355                 'Role'       : self.plc_spec['role'],
 356                 }
 357
 358     def locate_site(self,sitename):
 359         for site in self.plc_spec['sites']:
 360             if site['site_fields']['name'] == sitename:
 361                 return site
 362             if site['site_fields']['login_base'] == sitename:
 363                 return site
 364         raise Exception("Cannot locate site {}".format(sitename))
 365
 366     def locate_node(self, nodename):
 367         for site in self.plc_spec['sites']:
 368             for node in site['nodes']:
 369                 if node['name'] == nodename:
 370                     return site, node
 371         raise Exception("Cannot locate node {}".format(nodename))
 372
 373     def locate_hostname(self, hostname):
 374         for site in self.plc_spec['sites']:
 375             for node in site['nodes']:
 376                 if node['node_fields']['hostname'] == hostname:
 377                     return(site, node)
 378         raise Exception("Cannot locate hostname {}".format(hostname))
 379
 380     def locate_key(self, key_name):
 381         for key in self.plc_spec['keys']:
 382             if key['key_name'] == key_name:
 383                 return key
 384         raise Exception("Cannot locate key {}".format(key_name))
 385
 386     def locate_private_key_from_key_names(self, key_names):
 387         # locate the first avail. key
 388         found = False
 389         for key_name in key_names:
 390             key_spec = self.locate_key(key_name)
 391             test_key = TestKey(self,key_spec)
 392             publickey = test_key.publicpath()
 393             privatekey = test_key.privatepath()
 394             if os.path.isfile(publickey) and os.path.isfile(privatekey):
 395                 found = True
 396         if found:
 397             return privatekey
 398         else:
 399             return None
 400
 401     def locate_slice(self, slicename):
 402         for slice in self.plc_spec['slices']:
 403             if slice['slice_fields']['name'] == slicename:
 404                 return slice
 405         raise Exception("Cannot locate slice {}".format(slicename))
 406
 407     def all_sliver_objs(self):
 408         result = []
 409         for slice_spec in self.plc_spec['slices']:
 410             slicename = slice_spec['slice_fields']['name']
 411             for nodename in slice_spec['nodenames']:
 412                 result.append(self.locate_sliver_obj(nodename, slicename))
 413         return result
 414
 415     def locate_sliver_obj(self, nodename, slicename):
 416         site,node = self.locate_node(nodename)
 417         slice = self.locate_slice(slicename)
 418         # build objects
 419         test_site = TestSite(self, site)
 420         test_node = TestNode(self, test_site, node)
 421         # xxx the slice site is assumed to be the node site - mhh - probably harmless
 422         test_slice = TestSlice(self, test_site, slice)
 423         return TestSliver(self, test_node, test_slice)
 424
 425     def locate_first_node(self):
 426         nodename = self.plc_spec['slices'][0]['nodenames'][0]
 427         site,node = self.locate_node(nodename)
 428         test_site = TestSite(self, site)
 429         test_node = TestNode(self, test_site, node)
 430         return test_node
 431
 432     def locate_first_sliver(self):
 433         slice_spec = self.plc_spec['slices'][0]
 434         slicename = slice_spec['slice_fields']['name']
 435         nodename = slice_spec['nodenames'][0]
 436         return self.locate_sliver_obj(nodename,slicename)
 437
 438     # all different hostboxes used in this plc
 439     def get_BoxNodes(self):
 440         # maps on sites and nodes, return [ (host_box,test_node) ]
 441         tuples = []
 442         for site_spec in self.plc_spec['sites']:
 443             test_site = TestSite(self,site_spec)
 444             for node_spec in site_spec['nodes']:
 445                 test_node = TestNode(self, test_site, node_spec)
 446                 if not test_node.is_real():
 447                     tuples.append( (test_node.host_box(),test_node) )
 448         # transform into a dict { 'host_box' -> [ test_node .. ] }
 449         result = {}
 450         for (box,node) in tuples:
 451             if box not in result:
 452                 result[box] = [node]
 453             else:
 454                 result[box].append(node)
 455         return result
 456
 457     # a step for checking this stuff
 458     def show_boxes(self):
 459         'print summary of nodes location'
 460         for box,nodes in self.get_BoxNodes().items():
 461             print(box,":"," + ".join( [ node.name() for node in nodes ] ))
 462         return True
 463
 464     # make this a valid step
 465     def qemu_kill_all(self):
 466         'kill all qemu instances on the qemu boxes involved by this setup'
 467         # this is the brute force version, kill all qemus on that host box
 468         for (box,nodes) in self.get_BoxNodes().items():
 469             # pass the first nodename, as we don't push template-qemu on testboxes
 470             nodedir = nodes[0].nodedir()
 471             TestBoxQemu(box, self.options.buildname).qemu_kill_all(nodedir)
 472         return True
 473
 474     # make this a valid step
 475     def qemu_list_all(self):
 476         'list all qemu instances on the qemu boxes involved by this setup'
 477         for box,nodes in self.get_BoxNodes().items():
 478             # this is the brute force version, kill all qemus on that host box
 479             TestBoxQemu(box, self.options.buildname).qemu_list_all()
 480         return True
 481
 482     # kill only the qemus related to this test
 483     def qemu_list_mine(self):
 484         'list qemu instances for our nodes'
 485         for (box,nodes) in self.get_BoxNodes().items():
 486             # the fine-grain version
 487             for node in nodes:
 488                 node.list_qemu()
 489         return True
 490
 491     # kill only the qemus related to this test
 492     def qemu_clean_mine(self):
 493         'cleanup (rm -rf) qemu instances for our nodes'
 494         for box,nodes in self.get_BoxNodes().items():
 495             # the fine-grain version
 496             for node in nodes:
 497                 node.qemu_clean()
 498         return True
 499
 500     # kill only the right qemus
 501     def qemu_kill_mine(self):
 502         'kill the qemu instances for our nodes'
 503         for box,nodes in self.get_BoxNodes().items():
 504             # the fine-grain version
 505             for node in nodes:
 506                 node.kill_qemu()
 507         return True
 508
 509     #################### display config
 510     def show(self):
 511         "show test configuration after localization"
 512         self.show_pass(1)
 513         self.show_pass(2)
 514         return True
 515
 516     # uggly hack to make sure 'run export' only reports about the 1st plc
 517     # to avoid confusion - also we use 'inri_slice1' in various aliases..
 518     exported_id = 1
 519     def export(self):
 520         "print cut'n paste-able stuff to export env variables to your shell"
 521         # guess local domain from hostname
 522         if TestPlc.exported_id > 1:
 523             print("export GUESTHOSTNAME{:d}={}".format(TestPlc.exported_id, self.plc_spec['vservername']))
 524             return True
 525         TestPlc.exported_id += 1
 526         domain = socket.gethostname().split('.',1)[1]
 527         fqdn   = "{}.{}".format(self.plc_spec['host_box'], domain)
 528         print("export BUILD={}".format(self.options.buildname))
 529         print("export PLCHOSTLXC={}".format(fqdn))
 530         print("export GUESTNAME={}".format(self.vservername))
 531         print("export GUESTHOSTNAME={}.{}".format(self.vplchostname, domain))
 532         # find hostname of first node
 533         hostname, qemubox = self.all_node_infos()[0]
 534         print("export KVMHOST={}.{}".format(qemubox, domain))
 535         print("export NODE={}".format(hostname))
 536         return True
 537
 538     # entry point
 539     always_display_keys=['PLC_WWW_HOST', 'nodes', 'sites']
 540     def show_pass(self, passno):
 541         for (key,val) in self.plc_spec.items():
 542             if not self.options.verbose and key not in TestPlc.always_display_keys:
 543                 continue
 544             if passno == 2:
 545                 if key == 'sites':
 546                     for site in val:
 547                         self.display_site_spec(site)
 548                         for node in site['nodes']:
 549                             self.display_node_spec(node)
 550                 elif key == 'initscripts':
 551                     for initscript in val:
 552                         self.display_initscript_spec(initscript)
 553                 elif key == 'slices':
 554                     for slice in val:
 555                         self.display_slice_spec(slice)
 556                 elif key == 'keys':
 557                     for key in val:
 558                         self.display_key_spec(key)
 559             elif passno == 1:
 560                 if key not in ['sites', 'initscripts', 'slices', 'keys']:
 561                     print('+   ', key, ':', val)
 562
 563     def display_site_spec(self, site):
 564         print('+ ======== site', site['site_fields']['name'])
 565         for k,v in site.items():
 566             if not self.options.verbose and k not in TestPlc.always_display_keys:
 567                 continue
 568             if k == 'nodes':
 569                 if v:
 570                     print('+       ','nodes : ', end=' ')
 571                     for node in v:
 572                         print(node['node_fields']['hostname'],'', end=' ')
 573                     print('')
 574             elif k == 'users':
 575                 if v:
 576                     print('+       users : ', end=' ')
 577                     for user in v:
 578                         print(user['name'],'', end=' ')
 579                     print('')
 580             elif k == 'site_fields':
 581                 print('+       login_base', ':', v['login_base'])
 582             elif k == 'address_fields':
 583                 pass
 584             else:
 585                 print('+       ', end=' ')
 586                 utils.pprint(k, v)
 587
 588     def display_initscript_spec(self, initscript):
 589         print('+ ======== initscript', initscript['initscript_fields']['name'])
 590
 591     def display_key_spec(self, key):
 592         print('+ ======== key', key['key_name'])
 593
 594     def display_slice_spec(self, slice):
 595         print('+ ======== slice', slice['slice_fields']['name'])
 596         for k,v in slice.items():
 597             if k == 'nodenames':
 598                 if v:
 599                     print('+       nodes : ', end=' ')
 600                     for nodename in v:
 601                         print(nodename,'', end=' ')
 602                     print('')
 603             elif k == 'usernames':
 604                 if v:
 605                     print('+       users : ', end=' ')
 606                     for username in v:
 607                         print(username,'', end=' ')
 608                     print('')
 609             elif k == 'slice_fields':
 610                 print('+       fields',':', end=' ')
 611                 print('max_nodes=',v['max_nodes'], end=' ')
 612                 print('')
 613             else:
 614                 print('+       ',k,v)
 615
 616     def display_node_spec(self, node):
 617         print("+           node={} host_box={}".format(node['name'], node['host_box']), end=' ')
 618         print("hostname=", node['node_fields']['hostname'], end=' ')
 619         print("ip=", node['interface_fields']['ip'])
 620         if self.options.verbose:
 621             utils.pprint("node details", node, depth=3)
 622
 623     # another entry point for just showing the boxes involved
 624     def display_mapping(self):
 625         TestPlc.display_mapping_plc(self.plc_spec)
 626         return True
 627
 628     @staticmethod
 629     def display_mapping_plc(plc_spec):
 630         print('+ MyPLC',plc_spec['name'])
 631         # WARNING this would not be right for lxc-based PLC's - should be harmless though
 632         print('+\tvserver address = root@{}:/vservers/{}'.format(plc_spec['host_box'], plc_spec['vservername']))
 633         print('+\tIP = {}/{}'.format(plc_spec['settings']['PLC_API_HOST'], plc_spec['vserverip']))
 634         for site_spec in plc_spec['sites']:
 635             for node_spec in site_spec['nodes']:
 636                 TestPlc.display_mapping_node(node_spec)
 637
 638     @staticmethod
 639     def display_mapping_node(node_spec):
 640         print('+   NODE {}'.format(node_spec['name']))
 641         print('+\tqemu box {}'.format(node_spec['host_box']))
 642         print('+\thostname={}'.format(node_spec['node_fields']['hostname']))
 643
 644     # write a timestamp in /vservers/<>.timestamp
 645     # cannot be inside the vserver, that causes vserver .. build to cough
 646     def plcvm_timestamp(self):
 647         "Create a timestamp to remember creation date for this plc"
 648         now = int(time.time())
 649         # TODO-lxc check this one
 650         # a first approx. is to store the timestamp close to the VM root like vs does
 651         stamp_path = self.vm_timestamp_path()
 652         stamp_dir = os.path.dirname(stamp_path)
 653         utils.system(self.test_ssh.actual_command("mkdir -p {}".format(stamp_dir)))
 654         return utils.system(self.test_ssh.actual_command("echo {:d} > {}".format(now, stamp_path))) == 0
 655
 656     # this is called inconditionnally at the beginning of the test sequence
 657     # just in case this is a rerun, so if the vm is not running it's fine
 658     def plcvm_delete(self):
 659         "vserver delete the test myplc"
 660         stamp_path = self.vm_timestamp_path()
 661         self.run_in_host("rm -f {}".format(stamp_path))
 662         self.run_in_host("virsh -c lxc:// destroy {}".format(self.vservername))
 663         self.run_in_host("virsh -c lxc:// undefine {}".format(self.vservername))
 664         self.run_in_host("rm -fr /vservers/{}".format(self.vservername))
 665         return True
 666
 667     ### install
 668     # historically the build was being fetched by the tests
 669     # now the build pushes itself as a subdir of the tests workdir
 670     # so that the tests do not have to worry about extracting the build (svn, git, or whatever)
 671     def plcvm_create(self):
 672         "vserver creation (no install done)"
 673         # push the local build/ dir to the testplc box
 674         if self.is_local():
 675             # a full path for the local calls
 676             build_dir = os.path.dirname(sys.argv[0])
 677             # sometimes this is empty - set to "." in such a case
 678             if not build_dir:
 679                 build_dir="."
 680             build_dir += "/build"
 681         else:
 682             # use a standard name - will be relative to remote buildname
 683             build_dir = "build"
 684             # remove for safety; do *not* mkdir first, otherwise we end up with build/build/
 685             self.test_ssh.rmdir(build_dir)
 686             self.test_ssh.copy(build_dir, recursive=True)
 687         # the repo url is taken from arch-rpms-url
 688         # with the last step (i386) removed
 689         repo_url = self.options.arch_rpms_url
 690         for level in [ 'arch' ]:
 691             repo_url = os.path.dirname(repo_url)
 692
 693         # invoke initvm (drop support for vs)
 694         script = "lbuild-initvm.sh"
 695         script_options = ""
 696         # pass the vbuild-nightly options to [lv]test-initvm
 697         script_options += " -p {}".format(self.options.personality)
 698         script_options += " -d {}".format(self.options.pldistro)
 699         script_options += " -f {}".format(self.options.fcdistro)
 700         script_options += " -r {}".format(repo_url)
 701         vserver_name = self.vservername
 702         try:
 703             vserver_hostname = socket.gethostbyaddr(self.vserverip)[0]
 704             script_options += " -n {}".format(vserver_hostname)
 705         except:
 706             print("Cannot reverse lookup {}".format(self.vserverip))
 707             print("This is considered fatal, as this might pollute the test results")
 708             return False
 709         create_vserver="{build_dir}/{script} {script_options} {vserver_name}".format(**locals())
 710         return self.run_in_host(create_vserver) == 0
 711
 712     ### install_rpm
 713     def plc_install(self):
 714         """
 715         yum install myplc, noderepo + plain bootstrapfs as well
 716         """
 717
 718         # compute nodefamily
 719         if self.options.personality == "linux32":
 720             arch = "i386"
 721         elif self.options.personality == "linux64":
 722             arch = "x86_64"
 723         else:
 724             raise Exception("Unsupported personality {}".format(self.options.personality))
 725         nodefamily = "{}-{}-{}".format(self.options.pldistro, self.options.fcdistro, arch)
 726
 727         pkgs_list=[]
 728         pkgs_list.append("slicerepo-{}".format(nodefamily))
 729         pkgs_list.append("myplc")
 730         pkgs_list.append("noderepo-{}".format(nodefamily))
 731         pkgs_list.append("nodeimage-{}-plain".format(nodefamily))
 732         pkgs_string=" ".join(pkgs_list)
 733         return self.yum_install(pkgs_list)
 734
 735     def install_syslinux6(self):
 736         """
 737         install syslinux6 from the fedora21 release
 738         """
 739         key = 'http://mirror.onelab.eu/keys/RPM-GPG-KEY-fedora-21-primary'
 740
 741         rpms = [
 742             'http://mirror.onelab.eu/fedora/releases/21/Everything/x86_64/os/Packages/s/syslinux-6.03-1.fc21.x86_64.rpm',
 743             'http://mirror.onelab.eu/fedora/releases/21/Everything/x86_64/os/Packages/s/syslinux-nonlinux-6.03-1.fc21.noarch.rpm',
 744             'http://mirror.onelab.eu/fedora/releases/21/Everything/x86_64/os/Packages/s/syslinux-perl-6.03-1.fc21.x86_64.rpm',
 745         ]
 746         # this can be done several times
 747         self.run_in_guest("rpm --import {key}".format(**locals()))
 748         return self.run_in_guest("yum -y localinstall {}".format(" ".join(rpms))) == 0
 749
 750     def bonding_builds(self):
 751         """
 752         list /etc/yum.repos.d on the myplc side
 753         """
 754         self.run_in_guest("ls /etc/yum.repos.d/*partial.repo")
 755         return True
 756
 757     def bonding_nodes(self):
 758         """
 759         List nodes known to the myplc together with their nodefamiliy
 760         """
 761         print("---------------------------------------- nodes")
 762         for node in self.apiserver.GetNodes(self.auth_root()):
 763             print("{} -> {}".format(node['hostname'],
 764                                     self.apiserver.GetNodeFlavour(self.auth_root(),node['hostname'])['nodefamily']))
 765         print("---------------------------------------- nodes")
 766
 767
 768     ###
 769     def mod_python(self):
 770         """yum install mod_python, useful on f18 and above so as to avoid broken wsgi"""
 771         return self.yum_install( ['mod_python'] )
 772
 773     ###
 774     def plc_configure(self):
 775         "run plc-config-tty"
 776         tmpname = '{}.plc-config-tty'.format(self.name())
 777         with open(tmpname,'w') as fileconf:
 778             for (var,value) in self.plc_spec['settings'].items():
 779                 fileconf.write('e {}\n{}\n'.format(var, value))
 780             fileconf.write('w\n')
 781             fileconf.write('q\n')
 782         utils.system('cat {}'.format(tmpname))
 783         self.run_in_guest_piped('cat {}'.format(tmpname), 'plc-config-tty')
 784         utils.system('rm {}'.format(tmpname))
 785         return True
 786
 787 # f14 is a bit odd in this respect, although this worked fine in guests up to f18
 788 # however using a vplc guest under f20 requires this trick
 789 # the symptom is this: service plc start
 790 # Starting plc (via systemctl):  Failed to get D-Bus connection: \
 791 #    Failed to connect to socket /org/freedesktop/systemd1/private: Connection refused
 792 # weird thing is the doc says f14 uses upstart by default and not systemd
 793 # so this sounds kind of harmless
 794     def start_service(self, service):
 795         return self.start_stop_service(service, 'start')
 796     def stop_service(self, service):
 797         return self.start_stop_service(service, 'stop')
 798
 799     def start_stop_service(self, service, start_or_stop):
 800         "utility to start/stop a service with the special trick for f14"
 801         if self.options.fcdistro != 'f14':
 802             return self.run_in_guest("service {} {}".format(service, start_or_stop)) == 0
 803         else:
 804             # patch /sbin/service so it does not reset environment
 805             self.run_in_guest('sed -i -e \\"s,env -i,env,\\" /sbin/service')
 806             # this is because our own scripts in turn call service
 807             return self.run_in_guest("SYSTEMCTL_SKIP_REDIRECT=true service {} {}"\
 808                                      .format(service, start_or_stop)) == 0
 809
 810     def plc_start(self):
 811         "service plc start"
 812         return self.start_service('plc')
 813
 814     def plc_stop(self):
 815         "service plc stop"
 816         return self.stop_service('plc')
 817
 818     def plcvm_start(self):
 819         "start the PLC vserver"
 820         self.start_guest()
 821         return True
 822
 823     def plcvm_stop(self):
 824         "stop the PLC vserver"
 825         self.stop_guest()
 826         return True
 827
 828     # stores the keys from the config for further use
 829     def keys_store(self):
 830         "stores test users ssh keys in keys/"
 831         for key_spec in self.plc_spec['keys']:
 832                 TestKey(self,key_spec).store_key()
 833         return True
 834
 835     def keys_clean(self):
 836         "removes keys cached in keys/"
 837         utils.system("rm -rf ./keys")
 838         return True
 839
 840     # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
 841     # for later direct access to the nodes
 842     def keys_fetch(self):
 843         "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
 844         dir="./keys"
 845         if not os.path.isdir(dir):
 846             os.mkdir(dir)
 847         vservername = self.vservername
 848         vm_root = self.vm_root_in_host()
 849         overall = True
 850         prefix = 'debug_ssh_key'
 851         for ext in ['pub', 'rsa'] :
 852             src = "{vm_root}/etc/planetlab/{prefix}.{ext}".format(**locals())
 853             dst = "keys/{vservername}-debug.{ext}".format(**locals())
 854             if self.test_ssh.fetch(src, dst) != 0:
 855                 overall=False
 856         return overall
 857
 858     def sites(self):
 859         "create sites with PLCAPI"
 860         return self.do_sites()
 861
 862     def delete_sites(self):
 863         "delete sites with PLCAPI"
 864         return self.do_sites(action="delete")
 865
 866     def do_sites(self, action="add"):
 867         for site_spec in self.plc_spec['sites']:
 868             test_site = TestSite(self,site_spec)
 869             if (action != "add"):
 870                 utils.header("Deleting site {} in {}".format(test_site.name(), self.name()))
 871                 test_site.delete_site()
 872                 # deleted with the site
 873                 #test_site.delete_users()
 874                 continue
 875             else:
 876                 utils.header("Creating site {} & users in {}".format(test_site.name(), self.name()))
 877                 test_site.create_site()
 878                 test_site.create_users()
 879         return True
 880
 881     def delete_all_sites(self):
 882         "Delete all sites in PLC, and related objects"
 883         print('auth_root', self.auth_root())
 884         sites = self.apiserver.GetSites(self.auth_root(), {}, ['site_id','login_base'])
 885         for site in sites:
 886             # keep automatic site - otherwise we shoot in our own foot, root_auth is not valid anymore
 887             if site['login_base'] == self.plc_spec['settings']['PLC_SLICE_PREFIX']:
 888                 continue
 889             site_id = site['site_id']
 890             print('Deleting site_id', site_id)
 891             self.apiserver.DeleteSite(self.auth_root(), site_id)
 892         return True
 893
 894     def nodes(self):
 895         "create nodes with PLCAPI"
 896         return self.do_nodes()
 897     def delete_nodes(self):
 898         "delete nodes with PLCAPI"
 899         return self.do_nodes(action="delete")
 900
 901     def do_nodes(self, action="add"):
 902         for site_spec in self.plc_spec['sites']:
 903             test_site = TestSite(self, site_spec)
 904             if action != "add":
 905                 utils.header("Deleting nodes in site {}".format(test_site.name()))
 906                 for node_spec in site_spec['nodes']:
 907                     test_node = TestNode(self, test_site, node_spec)
 908                     utils.header("Deleting {}".format(test_node.name()))
 909                     test_node.delete_node()
 910             else:
 911                 utils.header("Creating nodes for site {} in {}".format(test_site.name(), self.name()))
 912                 for node_spec in site_spec['nodes']:
 913                     utils.pprint('Creating node {}'.format(node_spec), node_spec)
 914                     test_node = TestNode(self, test_site, node_spec)
 915                     test_node.create_node()
 916         return True
 917
 918     def nodegroups(self):
 919         "create nodegroups with PLCAPI"
 920         return self.do_nodegroups("add")
 921     def delete_nodegroups(self):
 922         "delete nodegroups with PLCAPI"
 923         return self.do_nodegroups("delete")
 924
 925     YEAR = 365*24*3600
 926     @staticmethod
 927     def translate_timestamp(start, grain, timestamp):
 928         if timestamp < TestPlc.YEAR:
 929             return start + timestamp*grain
 930         else:
 931             return timestamp
 932
 933     @staticmethod
 934     def timestamp_printable(timestamp):
 935         return time.strftime('%m-%d %H:%M:%S UTC', time.gmtime(timestamp))
 936
 937     def leases(self):
 938         "create leases (on reservable nodes only, use e.g. run -c default -c resa)"
 939         now = int(time.time())
 940         grain = self.apiserver.GetLeaseGranularity(self.auth_root())
 941         print('API answered grain=', grain)
 942         start = (now//grain)*grain
 943         start += grain
 944         # find out all nodes that are reservable
 945         nodes = self.all_reservable_nodenames()
 946         if not nodes:
 947             utils.header("No reservable node found - proceeding without leases")
 948             return True
 949         ok = True
 950         # attach them to the leases as specified in plc_specs
 951         # this is where the 'leases' field gets interpreted as relative of absolute
 952         for lease_spec in self.plc_spec['leases']:
 953             # skip the ones that come with a null slice id
 954             if not lease_spec['slice']:
 955                 continue
 956             lease_spec['t_from']  = TestPlc.translate_timestamp(start, grain, lease_spec['t_from'])
 957             lease_spec['t_until'] = TestPlc.translate_timestamp(start, grain, lease_spec['t_until'])
 958             lease_addition = self.apiserver.AddLeases(self.auth_root(), nodes, lease_spec['slice'],
 959                                                       lease_spec['t_from'], lease_spec['t_until'])
 960             if lease_addition['errors']:
 961                 utils.header("Cannot create leases, {}".format(lease_addition['errors']))
 962                 ok = False
 963             else:
 964                 utils.header('Leases on nodes {} for {} from {:d} ({}) until {:d} ({})'\
 965                              .format(nodes, lease_spec['slice'],
 966                                      lease_spec['t_from'],  TestPlc.timestamp_printable(lease_spec['t_from']),
 967                                      lease_spec['t_until'], TestPlc.timestamp_printable(lease_spec['t_until'])))
 968
 969         return ok
 970
 971     def delete_leases(self):
 972         "remove all leases in the myplc side"
 973         lease_ids = [ l['lease_id'] for l in self.apiserver.GetLeases(self.auth_root())]
 974         utils.header("Cleaning leases {}".format(lease_ids))
 975         self.apiserver.DeleteLeases(self.auth_root(), lease_ids)
 976         return True
 977
 978     def list_leases(self):
 979         "list all leases known to the myplc"
 980         leases = self.apiserver.GetLeases(self.auth_root())
 981         now = int(time.time())
 982         for l in leases:
 983             current = l['t_until'] >= now
 984             if self.options.verbose or current:
 985                 utils.header("{} {} from {} until {}"\
 986                              .format(l['hostname'], l['name'],
 987                                      TestPlc.timestamp_printable(l['t_from']),
 988                                      TestPlc.timestamp_printable(l['t_until'])))
 989         return True
 990
 991     # create nodegroups if needed, and populate
 992     def do_nodegroups(self, action="add"):
 993         # 1st pass to scan contents
 994         groups_dict = {}
 995         for site_spec in self.plc_spec['sites']:
 996             test_site = TestSite(self,site_spec)
 997             for node_spec in site_spec['nodes']:
 998                 test_node = TestNode(self, test_site, node_spec)
 999                 if 'nodegroups' in node_spec:
1000                     nodegroupnames = node_spec['nodegroups']
1001                     if isinstance(nodegroupnames, str):
1002                         nodegroupnames = [ nodegroupnames ]
1003                     for nodegroupname in nodegroupnames:
1004                         if nodegroupname not in groups_dict:
1005                             groups_dict[nodegroupname] = []
1006                         groups_dict[nodegroupname].append(test_node.name())
1007         auth = self.auth_root()
1008         overall = True
1009         for (nodegroupname,group_nodes) in groups_dict.items():
1010             if action == "add":
1011                 print('nodegroups:', 'dealing with nodegroup',\
1012                     nodegroupname, 'on nodes', group_nodes)
1013                 # first, check if the nodetagtype is here
1014                 tag_types = self.apiserver.GetTagTypes(auth, {'tagname':nodegroupname})
1015                 if tag_types:
1016                     tag_type_id = tag_types[0]['tag_type_id']
1017                 else:
1018                     tag_type_id = self.apiserver.AddTagType(auth,
1019                                                             {'tagname' : nodegroupname,
1020                                                              'description' : 'for nodegroup {}'.format(nodegroupname),
1021                                                              'category' : 'test'})
1022                 print('located tag (type)', nodegroupname, 'as', tag_type_id)
1023                 # create nodegroup
1024                 nodegroups = self.apiserver.GetNodeGroups(auth, {'groupname' : nodegroupname})
1025                 if not nodegroups:
1026                     self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
1027                     print('created nodegroup', nodegroupname, \
1028                         'from tagname', nodegroupname, 'and value', 'yes')
1029                 # set node tag on all nodes, value='yes'
1030                 for nodename in group_nodes:
1031                     try:
1032                         self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
1033                     except:
1034                         traceback.print_exc()
1035                         print('node', nodename, 'seems to already have tag', nodegroupname)
1036                     # check anyway
1037                     try:
1038                         expect_yes = self.apiserver.GetNodeTags(auth,
1039                                                                 {'hostname' : nodename,
1040                                                                  'tagname'  : nodegroupname},
1041                                                                 ['value'])[0]['value']
1042                         if expect_yes != "yes":
1043                             print('Mismatch node tag on node',nodename,'got',expect_yes)
1044                             overall = False
1045                     except:
1046                         if not self.options.dry_run:
1047                             print('Cannot find tag', nodegroupname, 'on node', nodename)
1048                             overall = False
1049             else:
1050                 try:
1051                     print('cleaning nodegroup', nodegroupname)
1052                     self.apiserver.DeleteNodeGroup(auth, nodegroupname)
1053                 except:
1054                     traceback.print_exc()
1055                     overall = False
1056         return overall
1057
1058     # a list of TestNode objs
1059     def all_nodes(self):
1060         nodes=[]
1061         for site_spec in self.plc_spec['sites']:
1062             test_site = TestSite(self,site_spec)
1063             for node_spec in site_spec['nodes']:
1064                 nodes.append(TestNode(self, test_site, node_spec))
1065         return nodes
1066
1067     # return a list of tuples (nodename,qemuname)
1068     def all_node_infos(self) :
1069         node_infos = []
1070         for site_spec in self.plc_spec['sites']:
1071             node_infos += [ (node_spec['node_fields']['hostname'], node_spec['host_box']) \
1072                                 for node_spec in site_spec['nodes'] ]
1073         return node_infos
1074
1075     def all_nodenames(self):
1076         return [ x[0] for x in self.all_node_infos() ]
1077     def all_reservable_nodenames(self):
1078         res = []
1079         for site_spec in self.plc_spec['sites']:
1080             for node_spec in site_spec['nodes']:
1081                 node_fields = node_spec['node_fields']
1082                 if 'node_type' in node_fields and node_fields['node_type'] == 'reservable':
1083                     res.append(node_fields['hostname'])
1084         return res
1085
1086     # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
1087     def nodes_check_boot_state(self, target_boot_state, timeout_minutes,
1088                                silent_minutes, period_seconds = 15):
1089         if self.options.dry_run:
1090             print('dry_run')
1091             return True
1092
1093         class CompleterTaskBootState(CompleterTask):
1094             def __init__(self, test_plc, hostname):
1095                 self.test_plc = test_plc
1096                 self.hostname = hostname
1097                 self.last_boot_state = 'undef'
1098             def actual_run(self):
1099                 try:
1100                     node = self.test_plc.apiserver.GetNodes(self.test_plc.auth_root(),
1101                                                             [ self.hostname ],
1102                                                             ['boot_state'])[0]
1103                     self.last_boot_state = node['boot_state']
1104                     return self.last_boot_state == target_boot_state
1105                 except:
1106                     return False
1107             def message(self):
1108                 return "CompleterTaskBootState with node {}".format(self.hostname)
1109             def failure_epilogue(self):
1110                 print("node {} in state {} - expected {}"\
1111                     .format(self.hostname, self.last_boot_state, target_boot_state))
1112
1113         timeout = timedelta(minutes=timeout_minutes)
1114         graceout = timedelta(minutes=silent_minutes)
1115         period   = timedelta(seconds=period_seconds)
1116         # the nodes that haven't checked yet - start with a full list and shrink over time
1117         utils.header("checking nodes boot state (expected {})".format(target_boot_state))
1118         tasks = [ CompleterTaskBootState(self,hostname) \
1119                       for (hostname,_) in self.all_node_infos() ]
1120         message = 'check_boot_state={}'.format(target_boot_state)
1121         return Completer(tasks, message=message).run(timeout, graceout, period)
1122
1123     def nodes_booted(self):
1124         return self.nodes_check_boot_state('boot', timeout_minutes=30, silent_minutes=28)
1125
1126     def probe_kvm_iptables(self):
1127         (_,kvmbox) = self.all_node_infos()[0]
1128         TestSsh(kvmbox).run("iptables-save")
1129         return True
1130
1131     # probing nodes
1132     def check_nodes_ping(self, timeout_seconds=60, period_seconds=10):
1133         class CompleterTaskPingNode(CompleterTask):
1134             def __init__(self, hostname):
1135                 self.hostname = hostname
1136             def run(self, silent):
1137                 command="ping -c 1 -w 1 {} >& /dev/null".format(self.hostname)
1138                 return utils.system(command, silent=silent) == 0
1139             def failure_epilogue(self):
1140                 print("Cannot ping node with name {}".format(self.hostname))
1141         timeout = timedelta(seconds = timeout_seconds)
1142         graceout = timeout
1143         period = timedelta(seconds = period_seconds)
1144         node_infos = self.all_node_infos()
1145         tasks = [ CompleterTaskPingNode(h) for (h,_) in node_infos ]
1146         return Completer(tasks, message='ping_node').run(timeout, graceout, period)
1147
1148     # ping node before we try to reach ssh, helpful for troubleshooting failing bootCDs
1149     def ping_node(self):
1150         "Ping nodes"
1151         return self.check_nodes_ping()
1152
1153     def check_nodes_ssh(self, debug, timeout_minutes, silent_minutes, period_seconds=15):
1154         # various delays
1155         timeout  = timedelta(minutes=timeout_minutes)
1156         graceout = timedelta(minutes=silent_minutes)
1157         period   = timedelta(seconds=period_seconds)
1158         vservername = self.vservername
1159         if debug:
1160             message = "debug"
1161             completer_message = 'ssh_node_debug'
1162             local_key = "keys/{vservername}-debug.rsa".format(**locals())
1163         else:
1164             message = "boot"
1165             completer_message = 'ssh_node_boot'
1166             local_key = "keys/key_admin.rsa"
1167         utils.header("checking ssh access to nodes (expected in {} mode)".format(message))
1168         node_infos = self.all_node_infos()
1169         tasks = [ CompleterTaskNodeSsh(nodename, qemuname, local_key,
1170                                         boot_state=message, dry_run=self.options.dry_run) \
1171                       for (nodename, qemuname) in node_infos ]
1172         return Completer(tasks, message=completer_message).run(timeout, graceout, period)
1173
1174     def ssh_node_debug(self):
1175         "Tries to ssh into nodes in debug mode with the debug ssh key"
1176         return self.check_nodes_ssh(debug = True,
1177                                     timeout_minutes = self.ssh_node_debug_timeout,
1178                                     silent_minutes = self.ssh_node_debug_silent)
1179
1180     def ssh_node_boot(self):
1181         "Tries to ssh into nodes in production mode with the root ssh key"
1182         return self.check_nodes_ssh(debug = False,
1183                                     timeout_minutes = self.ssh_node_boot_timeout,
1184                                     silent_minutes = self.ssh_node_boot_silent)
1185
1186     def node_bmlogs(self):
1187         "Checks that there's a non-empty dir. /var/log/bm/raw"
1188         return utils.system(self.actual_command_in_guest("ls /var/log/bm/raw")) == 0
1189
1190     @node_mapper
1191     def qemu_local_init(self): pass
1192     @node_mapper
1193     def bootcd(self): pass
1194     @node_mapper
1195     def qemu_local_config(self): pass
1196     @node_mapper
1197     def qemu_export(self): pass
1198     @node_mapper
1199     def qemu_cleanlog(self): pass
1200     @node_mapper
1201     def nodestate_reinstall(self): pass
1202     @node_mapper
1203     def nodestate_upgrade(self): pass
1204     @node_mapper
1205     def nodestate_safeboot(self): pass
1206     @node_mapper
1207     def nodestate_boot(self): pass
1208     @node_mapper
1209     def nodestate_show(self): pass
1210     @node_mapper
1211     def nodedistro_f14(self): pass
1212     @node_mapper
1213     def nodedistro_f18(self): pass
1214     @node_mapper
1215     def nodedistro_f20(self): pass
1216     @node_mapper
1217     def nodedistro_f21(self): pass
1218     @node_mapper
1219     def nodedistro_f22(self): pass
1220     @node_mapper
1221     def nodeflavour_show(self): pass
1222     @node_mapper
1223     def nodeplain_on(self): pass
1224     @node_mapper
1225     def nodeplain_off(self): pass
1226     @node_mapper
1227     def nodeplain_show(self): pass
1228
1229     ### check hooks : invoke scripts from hooks/{node,slice}
1230     def check_hooks_node(self):
1231         return self.locate_first_node().check_hooks()
1232     def check_hooks_sliver(self) :
1233         return self.locate_first_sliver().check_hooks()
1234
1235     def check_hooks(self):
1236         "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
1237         return self.check_hooks_node() and self.check_hooks_sliver()
1238
1239     ### initscripts
1240     def do_check_initscripts(self):
1241         class CompleterTaskInitscript(CompleterTask):
1242             def __init__(self, test_sliver, stamp):
1243                 self.test_sliver = test_sliver
1244                 self.stamp = stamp
1245             def actual_run(self):
1246                 return self.test_sliver.check_initscript_stamp(self.stamp)
1247             def message(self):
1248                 return "initscript checker for {}".format(self.test_sliver.name())
1249             def failure_epilogue(self):
1250                 print("initscript stamp {} not found in sliver {}"\
1251                     .format(self.stamp, self.test_sliver.name()))
1252
1253         tasks = []
1254         for slice_spec in self.plc_spec['slices']:
1255             if 'initscriptstamp' not in slice_spec:
1256                 continue
1257             stamp = slice_spec['initscriptstamp']
1258             slicename = slice_spec['slice_fields']['name']
1259             for nodename in slice_spec['nodenames']:
1260                 print('nodename', nodename, 'slicename', slicename, 'stamp', stamp)
1261                 site,node = self.locate_node(nodename)
1262                 # xxx - passing the wrong site - probably harmless
1263                 test_site = TestSite(self, site)
1264                 test_slice = TestSlice(self, test_site, slice_spec)
1265                 test_node = TestNode(self, test_site, node)
1266                 test_sliver = TestSliver(self, test_node, test_slice)
1267                 tasks.append(CompleterTaskInitscript(test_sliver, stamp))
1268         return Completer(tasks, message='check_initscripts').\
1269             run (timedelta(minutes=5), timedelta(minutes=4), timedelta(seconds=10))
1270
1271     def check_initscripts(self):
1272         "check that the initscripts have triggered"
1273         return self.do_check_initscripts()
1274
1275     def initscripts(self):
1276         "create initscripts with PLCAPI"
1277         for initscript in self.plc_spec['initscripts']:
1278             utils.pprint('Adding Initscript in plc {}'.format(self.plc_spec['name']), initscript)
1279             self.apiserver.AddInitScript(self.auth_root(), initscript['initscript_fields'])
1280         return True
1281
1282     def delete_initscripts(self):
1283         "delete initscripts with PLCAPI"
1284         for initscript in self.plc_spec['initscripts']:
1285             initscript_name = initscript['initscript_fields']['name']
1286             print(('Attempting to delete {} in plc {}'.format(initscript_name, self.plc_spec['name'])))
1287             try:
1288                 self.apiserver.DeleteInitScript(self.auth_root(), initscript_name)
1289                 print(initscript_name, 'deleted')
1290             except:
1291                 print('deletion went wrong - probably did not exist')
1292         return True
1293
1294     ### manage slices
1295     def slices(self):
1296         "create slices with PLCAPI"
1297         return self.do_slices(action="add")
1298
1299     def delete_slices(self):
1300         "delete slices with PLCAPI"
1301         return self.do_slices(action="delete")
1302
1303     def fill_slices(self):
1304         "add nodes in slices with PLCAPI"
1305         return self.do_slices(action="fill")
1306
1307     def empty_slices(self):
1308         "remove nodes from slices with PLCAPI"
1309         return self.do_slices(action="empty")
1310
1311     def do_slices(self,  action="add"):
1312         for slice in self.plc_spec['slices']:
1313             site_spec = self.locate_site(slice['sitename'])
1314             test_site = TestSite(self,site_spec)
1315             test_slice=TestSlice(self,test_site,slice)
1316             if action == "delete":
1317                 test_slice.delete_slice()
1318             elif action == "fill":
1319                 test_slice.add_nodes()
1320             elif action == "empty":
1321                 test_slice.delete_nodes()
1322             else:
1323                 test_slice.create_slice()
1324         return True
1325
1326     @slice_mapper__tasks(20, 10, 15)
1327     def ssh_slice(self): pass
1328     @slice_mapper__tasks(20, 19, 15)
1329     def ssh_slice_off(self): pass
1330     @slice_mapper__tasks(1, 1, 15)
1331     def slice_fs_present(self): pass
1332     @slice_mapper__tasks(1, 1, 15)
1333     def slice_fs_deleted(self): pass
1334
1335     # use another name so we can exclude/ignore it from the tests on the nightly command line
1336     def ssh_slice_again(self): return self.ssh_slice()
1337     # note that simply doing ssh_slice_again=ssh_slice would kind of work too
1338     # but for some reason the ignore-wrapping thing would not
1339
1340     @slice_mapper
1341     def ssh_slice_basics(self): pass
1342     @slice_mapper
1343     def check_vsys_defaults(self): pass
1344
1345     @node_mapper
1346     def keys_clear_known_hosts(self): pass
1347
1348     def plcapi_urls(self):
1349         """
1350         attempts to reach the PLCAPI with various forms for the URL
1351         """
1352         return PlcapiUrlScanner(self.auth_root(), ip=self.vserverip).scan()
1353
1354     def speed_up_slices(self):
1355         "tweak nodemanager cycle (wait time) to 30+/-10 s"
1356         return self._speed_up_slices (30, 10)
1357     def super_speed_up_slices(self):
1358         "dev mode: tweak nodemanager cycle (wait time) to 5+/-1 s"
1359         return self._speed_up_slices(5, 1)
1360
1361     def _speed_up_slices(self, p, r):
1362         # create the template on the server-side
1363         template = "{}.nodemanager".format(self.name())
1364         with open(template,"w") as template_file:
1365             template_file.write('OPTIONS="-p {} -r {} -d"\n'.format(p, r))
1366         in_vm = "/var/www/html/PlanetLabConf/nodemanager"
1367         remote = "{}/{}".format(self.vm_root_in_host(), in_vm)
1368         self.test_ssh.copy_abs(template, remote)
1369         # Add a conf file
1370         if not self.apiserver.GetConfFiles(self.auth_root(),
1371                                            {'dest' : '/etc/sysconfig/nodemanager'}):
1372             self.apiserver.AddConfFile(self.auth_root(),
1373                                         {'dest' : '/etc/sysconfig/nodemanager',
1374                                          'source' : 'PlanetLabConf/nodemanager',
1375                                          'postinstall_cmd' : 'service nm restart',})
1376         return True
1377
1378     def debug_nodemanager(self):
1379         "sets verbose mode for nodemanager, and speeds up cycle even more (needs speed_up_slices first)"
1380         template = "{}.nodemanager".format(self.name())
1381         with open(template,"w") as template_file:
1382             template_file.write('OPTIONS="-p 10 -r 6 -v -d"\n')
1383         in_vm = "/var/www/html/PlanetLabConf/nodemanager"
1384         remote = "{}/{}".format(self.vm_root_in_host(), in_vm)
1385         self.test_ssh.copy_abs(template, remote)
1386         return True
1387
1388     @node_mapper
1389     def qemu_start(self) : pass
1390
1391     @node_mapper
1392     def qemu_timestamp(self) : pass
1393
1394     @node_mapper
1395     def qemu_nodefamily(self): pass
1396
1397     # when a spec refers to a node possibly on another plc
1398     def locate_sliver_obj_cross(self, nodename, slicename, other_plcs):
1399         for plc in [ self ] + other_plcs:
1400             try:
1401                 return plc.locate_sliver_obj(nodename, slicename)
1402             except:
1403                 pass
1404         raise Exception("Cannot locate sliver {}@{} among all PLCs".format(nodename, slicename))
1405
1406     # implement this one as a cross step so that we can take advantage of different nodes
1407     # in multi-plcs mode
1408     def cross_check_tcp(self, other_plcs):
1409         "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
1410         if 'tcp_specs' not in self.plc_spec or not self.plc_spec['tcp_specs']:
1411             utils.header("check_tcp: no/empty config found")
1412             return True
1413         specs = self.plc_spec['tcp_specs']
1414         overall = True
1415
1416         # first wait for the network to be up and ready from the slices
1417         class CompleterTaskNetworkReadyInSliver(CompleterTask):
1418             def __init__(self, test_sliver):
1419                 self.test_sliver = test_sliver
1420             def actual_run(self):
1421                 return self.test_sliver.check_tcp_ready(port = 9999)
1422             def message(self):
1423                 return "network ready checker for {}".format(self.test_sliver.name())
1424             def failure_epilogue(self):
1425                 print("could not bind port from sliver {}".format(self.test_sliver.name()))
1426
1427         sliver_specs = {}
1428         tasks = []
1429         managed_sliver_names = set()
1430         for spec in specs:
1431             # locate the TestSliver instances involved, and cache them in the spec instance
1432             spec['s_sliver'] = self.locate_sliver_obj_cross(spec['server_node'], spec['server_slice'], other_plcs)
1433             spec['c_sliver'] = self.locate_sliver_obj_cross(spec['client_node'], spec['client_slice'], other_plcs)
1434             message = "Will check TCP between s={} and c={}"\
1435                       .format(spec['s_sliver'].name(), spec['c_sliver'].name())
1436             if 'client_connect' in spec:
1437                 message += " (using {})".format(spec['client_connect'])
1438             utils.header(message)
1439             # we need to check network presence in both slivers, but also
1440             # avoid to insert a sliver several times
1441             for sliver in [ spec['s_sliver'], spec['c_sliver'] ]:
1442                 if sliver.name() not in managed_sliver_names:
1443                     tasks.append(CompleterTaskNetworkReadyInSliver(sliver))
1444                     # add this sliver's name in the set
1445                     managed_sliver_names .update( {sliver.name()} )
1446
1447         # wait for the netork to be OK in all server sides
1448         if not Completer(tasks, message='check for network readiness in slivers').\
1449            run(timedelta(seconds=30), timedelta(seconds=24), period=timedelta(seconds=5)):
1450             return False
1451
1452         # run server and client
1453         for spec in specs:
1454             port = spec['port']
1455             # server side
1456             # the issue here is that we have the server run in background
1457             # and so we have no clue if it took off properly or not
1458             # looks like in some cases it does not
1459             if not spec['s_sliver'].run_tcp_server(port, timeout=20):
1460                 overall = False
1461                 break
1462
1463             # idem for the client side
1464             # use nodename from located sliver, unless 'client_connect' is set
1465             if 'client_connect' in spec:
1466                 destination = spec['client_connect']
1467             else:
1468                 destination = spec['s_sliver'].test_node.name()
1469             if not spec['c_sliver'].run_tcp_client(destination, port):
1470                 overall = False
1471         return overall
1472
1473     # painfully enough, we need to allow for some time as netflow might show up last
1474     def check_system_slice(self):
1475         "all nodes: check that a system slice is alive"
1476         # netflow currently not working in the lxc distro
1477         # drl not built at all in the wtx distro
1478         # if we find either of them we're happy
1479         return self.check_netflow() or self.check_drl()
1480
1481     # expose these
1482     def check_netflow(self): return self._check_system_slice('netflow')
1483     def check_drl(self): return self._check_system_slice('drl')
1484
1485     # we have the slices up already here, so it should not take too long
1486     def _check_system_slice(self, slicename, timeout_minutes=5, period_seconds=15):
1487         class CompleterTaskSystemSlice(CompleterTask):
1488             def __init__(self, test_node, dry_run):
1489                 self.test_node = test_node
1490                 self.dry_run = dry_run
1491             def actual_run(self):
1492                 return self.test_node._check_system_slice(slicename, dry_run=self.dry_run)
1493             def message(self):
1494                 return "System slice {} @ {}".format(slicename, self.test_node.name())
1495             def failure_epilogue(self):
1496                 print("COULD not find system slice {} @ {}".format(slicename, self.test_node.name()))
1497         timeout = timedelta(minutes=timeout_minutes)
1498         silent  = timedelta(0)
1499         period  = timedelta(seconds=period_seconds)
1500         tasks = [ CompleterTaskSystemSlice(test_node, self.options.dry_run) \
1501                       for test_node in self.all_nodes() ]
1502         return Completer(tasks, message='_check_system_slice').run(timeout, silent, period)
1503
1504     def plcsh_stress_test(self):
1505         "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
1506         # install the stress-test in the plc image
1507         location = "/usr/share/plc_api/plcsh_stress_test.py"
1508         remote = "{}/{}".format(self.vm_root_in_host(), location)
1509         self.test_ssh.copy_abs("plcsh_stress_test.py", remote)
1510         command = location
1511         command += " -- --check"
1512         if self.options.size == 1:
1513             command +=  " --tiny"
1514         return self.run_in_guest(command) == 0
1515
1516     # populate runs the same utility without slightly different options
1517     # in particular runs with --preserve (dont cleanup) and without --check
1518     # also it gets run twice, once with the --foreign option for creating fake foreign entries
1519
1520     def sfa_install_all(self):
1521         "yum install sfa sfa-plc sfa-sfatables sfa-client"
1522         return self.yum_install("sfa sfa-plc sfa-sfatables sfa-client")
1523
1524     def sfa_install_core(self):
1525         "yum install sfa"
1526         return self.yum_install("sfa")
1527
1528     def sfa_install_plc(self):
1529         "yum install sfa-plc"
1530         return self.yum_install("sfa-plc")
1531
1532     def sfa_install_sfatables(self):
1533         "yum install sfa-sfatables"
1534         return self.yum_install("sfa-sfatables")
1535
1536     # for some very odd reason, this sometimes fails with the following symptom
1537     # # yum install sfa-client
1538     # Setting up Install Process
1539     # ...
1540     # Downloading Packages:
1541     # Running rpm_check_debug
1542     # Running Transaction Test
1543     # Transaction Test Succeeded
1544     # Running Transaction
1545     # Transaction couldn't start:
1546     # installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem
1547     # [('installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem', (9, '/', 69632L))]
1548     # even though in the same context I have
1549     # [2012.05.23--f14-32-sfastd1-1-vplc07] / # df -h
1550     # Filesystem            Size  Used Avail Use% Mounted on
1551     # /dev/hdv1             806G  264G  501G  35% /
1552     # none                   16M   36K   16M   1% /tmp
1553     #
1554     # so as a workaround, we first try yum install, and then invoke rpm on the cached rpm...
1555     def sfa_install_client(self):
1556         "yum install sfa-client"
1557         first_try = self.yum_install("sfa-client")
1558         if first_try:
1559             return True
1560         utils.header("********** Regular yum failed - special workaround in place, 2nd chance")
1561         code, cached_rpm_path = \
1562                 utils.output_of(self.actual_command_in_guest('find /var/cache/yum -name sfa-client\*.rpm'))
1563         utils.header("rpm_path=<<{}>>".format(rpm_path))
1564         # just for checking
1565         self.run_in_guest("rpm -i {}".format(cached_rpm_path))
1566         return self.yum_check_installed("sfa-client")
1567
1568     def sfa_dbclean(self):
1569         "thoroughly wipes off the SFA database"
1570         return self.run_in_guest("sfaadmin reg nuke") == 0 or \
1571             self.run_in_guest("sfa-nuke.py") == 0 or \
1572             self.run_in_guest("sfa-nuke-plc.py") == 0 or \
1573             self.run_in_guest("sfaadmin registry nuke") == 0
1574
1575     def sfa_fsclean(self):
1576         "cleanup /etc/sfa/trusted_roots and /var/lib/sfa"
1577         self.run_in_guest("rm -rf /etc/sfa/trusted_roots /var/lib/sfa/authorities")
1578         return True
1579
1580     def sfa_plcclean(self):
1581         "cleans the PLC entries that were created as a side effect of running the script"
1582         # ignore result
1583         sfa_spec = self.plc_spec['sfa']
1584
1585         for auth_sfa_spec in sfa_spec['auth_sfa_specs']:
1586             login_base = auth_sfa_spec['login_base']
1587             try:
1588                 self.apiserver.DeleteSite(self.auth_root(),login_base)
1589             except:
1590                 print("Site {} already absent from PLC db".format(login_base))
1591
1592             for spec_name in ['pi_spec','user_spec']:
1593                 user_spec = auth_sfa_spec[spec_name]
1594                 username = user_spec['email']
1595                 try:
1596                     self.apiserver.DeletePerson(self.auth_root(),username)
1597                 except:
1598                     # this in fact is expected as sites delete their members
1599                     #print "User {} already absent from PLC db".format(username)
1600                     pass
1601
1602         print("REMEMBER TO RUN sfa_import AGAIN")
1603         return True
1604
1605     def sfa_uninstall(self):
1606         "uses rpm to uninstall sfa - ignore result"
1607         self.run_in_guest("rpm -e sfa sfa-sfatables sfa-client sfa-plc")
1608         self.run_in_guest("rm -rf /var/lib/sfa")
1609         self.run_in_guest("rm -rf /etc/sfa")
1610         self.run_in_guest("rm -rf /var/log/sfa_access.log /var/log/sfa_import_plc.log /var/log/sfa.daemon")
1611         # xxx tmp
1612         self.run_in_guest("rpm -e --noscripts sfa-plc")
1613         return True
1614
1615     ### run unit tests for SFA
1616     # NOTE: for some reason on f14/i386, yum install sfa-tests fails for no reason
1617     # Running Transaction
1618     # Transaction couldn't start:
1619     # installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem
1620     # [('installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem', (9, '/', 208896L))]
1621     # no matter how many Gbs are available on the testplc
1622     # could not figure out what's wrong, so...
1623     # if the yum install phase fails, consider the test is successful
1624     # other combinations will eventually run it hopefully
1625     def sfa_utest(self):
1626         "yum install sfa-tests and run SFA unittests"
1627         self.run_in_guest("yum -y install sfa-tests")
1628         # failed to install - forget it
1629         if self.run_in_guest("rpm -q sfa-tests") != 0:
1630             utils.header("WARNING: SFA unit tests failed to install, ignoring")
1631             return True
1632         return self.run_in_guest("/usr/share/sfa/tests/testAll.py") == 0
1633
1634     ###
1635     def confdir(self):
1636         dirname = "conf.{}".format(self.plc_spec['name'])
1637         if not os.path.isdir(dirname):
1638             utils.system("mkdir -p {}".format(dirname))
1639         if not os.path.isdir(dirname):
1640             raise Exception("Cannot create config dir for plc {}".format(self.name()))
1641         return dirname
1642
1643     def conffile(self, filename):
1644         return "{}/{}".format(self.confdir(), filename)
1645     def confsubdir(self, dirname, clean, dry_run=False):
1646         subdirname = "{}/{}".format(self.confdir(), dirname)
1647         if clean:
1648             utils.system("rm -rf {}".format(subdirname))
1649         if not os.path.isdir(subdirname):
1650             utils.system("mkdir -p {}".format(subdirname))
1651         if not dry_run and not os.path.isdir(subdirname):
1652             raise "Cannot create config subdir {} for plc {}".format(dirname, self.name())
1653         return subdirname
1654
1655     def conffile_clean(self, filename):
1656         filename=self.conffile(filename)
1657         return utils.system("rm -rf {}".format(filename))==0
1658
1659     ###
1660     def sfa_configure(self):
1661         "run sfa-config-tty"
1662         tmpname = self.conffile("sfa-config-tty")
1663         with open(tmpname,'w') as fileconf:
1664             for (var,value) in self.plc_spec['sfa']['settings'].items():
1665                 fileconf.write('e {}\n{}\n'.format(var, value))
1666             fileconf.write('w\n')
1667             fileconf.write('R\n')
1668             fileconf.write('q\n')
1669         utils.system('cat {}'.format(tmpname))
1670         self.run_in_guest_piped('cat {}'.format(tmpname), 'sfa-config-tty')
1671         return True
1672
1673     def aggregate_xml_line(self):
1674         port = self.plc_spec['sfa']['neighbours-port']
1675         return '<aggregate addr="{}" hrn="{}" port="{}"/>'\
1676             .format(self.vserverip, self.plc_spec['sfa']['settings']['SFA_REGISTRY_ROOT_AUTH'], port)
1677
1678     def registry_xml_line(self):
1679         return '<registry addr="{}" hrn="{}" port="12345"/>'\
1680             .format(self.vserverip, self.plc_spec['sfa']['settings']['SFA_REGISTRY_ROOT_AUTH'])
1681
1682
1683     # a cross step that takes all other plcs in argument
1684     def cross_sfa_configure(self, other_plcs):
1685         "writes aggregates.xml and registries.xml that point to all other PLCs in the test"
1686         # of course with a single plc, other_plcs is an empty list
1687         if not other_plcs:
1688             return True
1689         agg_fname = self.conffile("agg.xml")
1690         with open(agg_fname,"w") as out:
1691             out.write("<aggregates>{}</aggregates>\n"\
1692                       .format(" ".join([ plc.aggregate_xml_line() for plc in other_plcs ])))
1693         utils.header("(Over)wrote {}".format(agg_fname))
1694         reg_fname=self.conffile("reg.xml")
1695         with open(reg_fname,"w") as out:
1696             out.write("<registries>{}</registries>\n"\
1697                       .format(" ".join([ plc.registry_xml_line() for plc in other_plcs ])))
1698         utils.header("(Over)wrote {}".format(reg_fname))
1699         return self.test_ssh.copy_abs(agg_fname,
1700                                       '/{}/etc/sfa/aggregates.xml'.format(self.vm_root_in_host())) == 0 \
1701            and self.test_ssh.copy_abs(reg_fname,
1702                                       '/{}/etc/sfa/registries.xml'.format(self.vm_root_in_host())) == 0
1703
1704     def sfa_import(self):
1705         "use sfaadmin to import from plc"
1706         auth = self.plc_spec['sfa']['settings']['SFA_REGISTRY_ROOT_AUTH']
1707         return self.run_in_guest('sfaadmin reg import_registry') == 0
1708
1709     def sfa_start(self):
1710         "service sfa start"
1711         return self.start_service('sfa')
1712
1713
1714     def sfi_configure(self):
1715         "Create /root/sfi on the plc side for sfi client configuration"
1716         if self.options.dry_run:
1717             utils.header("DRY RUN - skipping step")
1718             return True
1719         sfa_spec = self.plc_spec['sfa']
1720         # cannot use auth_sfa_mapper to pass dir_name
1721         for slice_spec in self.plc_spec['sfa']['auth_sfa_specs']:
1722             test_slice = TestAuthSfa(self, slice_spec)
1723             dir_basename = os.path.basename(test_slice.sfi_path())
1724             dir_name = self.confsubdir("dot-sfi/{}".format(dir_basename),
1725                                        clean=True, dry_run=self.options.dry_run)
1726             test_slice.sfi_configure(dir_name)
1727             # push into the remote /root/sfi area
1728             location = test_slice.sfi_path()
1729             remote = "{}/{}".format(self.vm_root_in_host(), location)
1730             self.test_ssh.mkdir(remote, abs=True)
1731             # need to strip last level or remote otherwise we get an extra dir level
1732             self.test_ssh.copy_abs(dir_name, os.path.dirname(remote), recursive=True)
1733
1734         return True
1735
1736     def sfi_clean(self):
1737         "clean up /root/sfi on the plc side"
1738         self.run_in_guest("rm -rf /root/sfi")
1739         return True
1740
1741     def sfa_rspec_empty(self):
1742         "expose a static empty rspec (ships with the tests module) in the sfi directory"
1743         filename = "empty-rspec.xml"
1744         overall = True
1745         for slice_spec in self.plc_spec['sfa']['auth_sfa_specs']:
1746             test_slice = TestAuthSfa(self, slice_spec)
1747             in_vm = test_slice.sfi_path()
1748             remote = "{}/{}".format(self.vm_root_in_host(), in_vm)
1749             if self.test_ssh.copy_abs(filename, remote) !=0:
1750                 overall = False
1751         return overall
1752
1753     @auth_sfa_mapper
1754     def sfa_register_site(self): pass
1755     @auth_sfa_mapper
1756     def sfa_register_pi(self): pass
1757     @auth_sfa_mapper
1758     def sfa_register_user(self): pass
1759     @auth_sfa_mapper
1760     def sfa_update_user(self): pass
1761     @auth_sfa_mapper
1762     def sfa_register_slice(self): pass
1763     @auth_sfa_mapper
1764     def sfa_renew_slice(self): pass
1765     @auth_sfa_mapper
1766     def sfa_get_expires(self): pass
1767     @auth_sfa_mapper
1768     def sfa_discover(self): pass
1769     @auth_sfa_mapper
1770     def sfa_rspec(self): pass
1771     @auth_sfa_mapper
1772     def sfa_allocate(self): pass
1773     @auth_sfa_mapper
1774     def sfa_allocate_empty(self): pass
1775     @auth_sfa_mapper
1776     def sfa_provision(self): pass
1777     @auth_sfa_mapper
1778     def sfa_provision_empty(self): pass
1779     @auth_sfa_mapper
1780     def sfa_describe(self): pass
1781     @auth_sfa_mapper
1782     def sfa_check_slice_plc(self): pass
1783     @auth_sfa_mapper
1784     def sfa_check_slice_plc_empty(self): pass
1785     @auth_sfa_mapper
1786     def sfa_update_slice(self): pass
1787     @auth_sfa_mapper
1788     def sfa_remove_user_from_slice(self): pass
1789     @auth_sfa_mapper
1790     def sfa_insert_user_in_slice(self): pass
1791     @auth_sfa_mapper
1792     def sfi_list(self): pass
1793     @auth_sfa_mapper
1794     def sfi_show_site(self): pass
1795     @auth_sfa_mapper
1796     def sfi_show_slice(self): pass
1797     @auth_sfa_mapper
1798     def sfi_show_slice_researchers(self): pass
1799     @auth_sfa_mapper
1800     def ssh_slice_sfa(self): pass
1801     @auth_sfa_mapper
1802     def sfa_delete_user(self): pass
1803     @auth_sfa_mapper
1804     def sfa_delete_slice(self): pass
1805
1806     def sfa_stop(self):
1807         "service sfa stop"
1808         return self.stop_service('sfa')
1809
1810     def populate(self):
1811         "creates random entries in the PLCAPI"
1812         # install the stress-test in the plc image
1813         location = "/usr/share/plc_api/plcsh_stress_test.py"
1814         remote = "{}/{}".format(self.vm_root_in_host(), location)
1815         self.test_ssh.copy_abs("plcsh_stress_test.py", remote)
1816         command = location
1817         command += " -- --preserve --short-names"
1818         local = (self.run_in_guest(command) == 0);
1819         # second run with --foreign
1820         command += ' --foreign'
1821         remote = (self.run_in_guest(command) == 0);
1822         return local and remote
1823
1824
1825     ####################
1826     @bonding_redirector
1827     def bonding_init_partial(self): pass
1828
1829     @bonding_redirector
1830     def bonding_add_yum(self): pass
1831
1832     @bonding_redirector
1833     def bonding_install_rpms(self): pass
1834
1835     ####################
1836
1837     def gather_logs(self):
1838         "gets all possible logs from plc's/qemu node's/slice's for future reference"
1839         # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1840         # (1.b) get the plc's  /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1841         # (1.c) get the plc's /root/sfi -> logs/sfi.<plcname>/
1842         # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1843         # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1844         # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1845         # (1.a)
1846         print("-------------------- TestPlc.gather_logs : PLC's /var/log")
1847         self.gather_var_logs()
1848         # (1.b)
1849         print("-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/")
1850         self.gather_pgsql_logs()
1851         # (1.c)
1852         print("-------------------- TestPlc.gather_logs : PLC's /root/sfi/")
1853         self.gather_root_sfi()
1854         # (2)
1855         print("-------------------- TestPlc.gather_logs : nodes's QEMU logs")
1856         for site_spec in self.plc_spec['sites']:
1857             test_site = TestSite(self,site_spec)
1858             for node_spec in site_spec['nodes']:
1859                 test_node = TestNode(self, test_site, node_spec)
1860                 test_node.gather_qemu_logs()
1861         # (3)
1862         print("-------------------- TestPlc.gather_logs : nodes's /var/log")
1863         self.gather_nodes_var_logs()
1864         # (4)
1865         print("-------------------- TestPlc.gather_logs : sample sliver's /var/log")
1866         self.gather_slivers_var_logs()
1867         return True
1868
1869     def gather_slivers_var_logs(self):
1870         for test_sliver in self.all_sliver_objs():
1871             remote = test_sliver.tar_var_logs()
1872             utils.system("mkdir -p logs/sliver.var-log.{}".format(test_sliver.name()))
1873             command = remote + " | tar -C logs/sliver.var-log.{} -xf -".format(test_sliver.name())
1874             utils.system(command)
1875         return True
1876
1877     def gather_var_logs(self):
1878         utils.system("mkdir -p logs/myplc.var-log.{}".format(self.name()))
1879         to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1880         command = to_plc + "| tar -C logs/myplc.var-log.{} -xf -".format(self.name())
1881         utils.system(command)
1882         command = "chmod a+r,a+x logs/myplc.var-log.{}/httpd".format(self.name())
1883         utils.system(command)
1884
1885     def gather_pgsql_logs(self):
1886         utils.system("mkdir -p logs/myplc.pgsql-log.{}".format(self.name()))
1887         to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1888         command = to_plc + "| tar -C logs/myplc.pgsql-log.{} -xf -".format(self.name())
1889         utils.system(command)
1890
1891     def gather_root_sfi(self):
1892         utils.system("mkdir -p logs/sfi.{}".format(self.name()))
1893         to_plc = self.actual_command_in_guest("tar -C /root/sfi/ -cf - .")
1894         command = to_plc + "| tar -C logs/sfi.{} -xf -".format(self.name())
1895         utils.system(command)
1896
1897     def gather_nodes_var_logs(self):
1898         for site_spec in self.plc_spec['sites']:
1899             test_site = TestSite(self, site_spec)
1900             for node_spec in site_spec['nodes']:
1901                 test_node = TestNode(self, test_site, node_spec)
1902                 test_ssh = TestSsh(test_node.name(), key="keys/key_admin.rsa")
1903                 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1904                 command = command + "| tar -C logs/node.var-log.{} -xf -".format(test_node.name())
1905                 utils.system("mkdir -p logs/node.var-log.{}".format(test_node.name()))
1906                 utils.system(command)
1907
1908
1909     # returns the filename to use for sql dump/restore, using options.dbname if set
1910     def dbfile(self, database):
1911         # uses options.dbname if it is found
1912         try:
1913             name = self.options.dbname
1914             if not isinstance(name, str):
1915                 raise Exception
1916         except:
1917             t = datetime.now()
1918             d = t.date()
1919             name = str(d)
1920         return "/root/{}-{}.sql".format(database, name)
1921
1922     def plc_db_dump(self):
1923         'dump the planetlab5 DB in /root in the PLC - filename has time'
1924         dump=self.dbfile("planetab5")
1925         self.run_in_guest('pg_dump -U pgsqluser planetlab5 -f '+ dump)
1926         utils.header('Dumped planetlab5 database in {}'.format(dump))
1927         return True
1928
1929     def plc_db_restore(self):
1930         'restore the planetlab5 DB - looks broken, but run -n might help'
1931         dump = self.dbfile("planetab5")
1932         ##stop httpd service
1933         self.run_in_guest('service httpd stop')
1934         # xxx - need another wrapper
1935         self.run_in_guest_piped('echo drop database planetlab5', 'psql --user=pgsqluser template1')
1936         self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab5')
1937         self.run_in_guest('psql -U pgsqluser planetlab5 -f ' + dump)
1938         ##starting httpd service
1939         self.run_in_guest('service httpd start')
1940
1941         utils.header('Database restored from ' + dump)
1942
1943     @staticmethod
1944     def create_ignore_steps():
1945         for step in TestPlc.default_steps + TestPlc.other_steps:
1946             # default step can have a plc qualifier
1947             if '@' in step:
1948                 step, qualifier = step.split('@')
1949             # or be defined as forced or ignored by default
1950             for keyword in ['_ignore','_force']:
1951                 if step.endswith(keyword):
1952                     step=step.replace(keyword,'')
1953             if step == SEP or step == SEPSFA :
1954                 continue
1955             method = getattr(TestPlc,step)
1956             name = step + '_ignore'
1957             wrapped = ignore_result(method)
1958 #            wrapped.__doc__ = method.__doc__ + " (run in ignore-result mode)"
1959             setattr(TestPlc, name, wrapped)
1960
1961 #    @ignore_result
1962 #    def ssh_slice_again_ignore (self): pass
1963 #    @ignore_result
1964 #    def check_initscripts_ignore (self): pass
1965
1966     def standby_1_through_20(self):
1967         """convenience function to wait for a specified number of minutes"""
1968         pass
1969     @standby_generic
1970     def standby_1(): pass
1971     @standby_generic
1972     def standby_2(): pass
1973     @standby_generic
1974     def standby_3(): pass
1975     @standby_generic
1976     def standby_4(): pass
1977     @standby_generic
1978     def standby_5(): pass
1979     @standby_generic
1980     def standby_6(): pass
1981     @standby_generic
1982     def standby_7(): pass
1983     @standby_generic
1984     def standby_8(): pass
1985     @standby_generic
1986     def standby_9(): pass
1987     @standby_generic
1988     def standby_10(): pass
1989     @standby_generic
1990     def standby_11(): pass
1991     @standby_generic
1992     def standby_12(): pass
1993     @standby_generic
1994     def standby_13(): pass
1995     @standby_generic
1996     def standby_14(): pass
1997     @standby_generic
1998     def standby_15(): pass
1999     @standby_generic
2000     def standby_16(): pass
2001     @standby_generic
2002     def standby_17(): pass
2003     @standby_generic
2004     def standby_18(): pass
2005     @standby_generic
2006     def standby_19(): pass
2007     @standby_generic
2008     def standby_20(): pass
2009
2010     # convenience for debugging the test logic
2011     def yes(self): return True
2012     def no(self): return False
2013     def fail(self): return False