system/TestMain.py

   1 #!/usr/bin/python -u
   2
   3 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
   4 # Copyright (C) 2010 INRIA
   5 #
   6 import sys, os, os.path
   7 from argparse import ArgumentParser
   8 import traceback
   9 import readline
  10 import glob
  11 from datetime import datetime
  12
  13 import utils
  14 from TestPlc import TestPlc, Ignored
  15 from TestBonding import TestBonding, onelab_bonding_spec
  16 from TestSite import TestSite
  17 from TestNode import TestNode
  18 from macros import sequences
  19
  20 # add $HOME in PYTHONPATH so we can import LocalSubstrate.py
  21 sys.path.append(os.environ['HOME'])
  22 import LocalSubstrate
  23
  24 class Step:
  25
  26     natives = TestPlc.__dict__
  27
  28     def display (self):
  29         return self.name.replace('_', '-')
  30     def internal (self):
  31         return self.name.replace('-', '_')
  32
  33     def __init__ (self, name):
  34         self.name = name
  35         # a native step is implemented as a method on TestPlc
  36         self.native = self.internal() in Step.natives
  37         if self.native:
  38             self.method = Step.natives[self.internal()]
  39         else:
  40             try:
  41                 self.substeps = sequences[self.internal()]
  42             except Exception,e:
  43                 print "macro step %s not found in macros.py (%s) - exiting" % (self.display(),e)
  44                 raise
  45
  46     def print_doc (self, level=0):
  47         tab = 32
  48         trail = 8
  49         if self.native:
  50             start = level*' ' + '* '
  51             # 2 is the len of '* '
  52             width = tab - level - 2
  53             format = "%%-%ds" % width
  54             line = start + format % self.display()
  55             print line,
  56             try:
  57                 print self.method.__doc__
  58             except:
  59                 print "*** no doc found"
  60         else:
  61             beg_start = level*' ' + '>>> '
  62             end_start = level*' ' + '<<< '
  63             trailer = trail * '-'
  64             # 4 is the len of '>>> '
  65             width = tab - level - 4 - trail
  66             format = "%%-%ds" % width
  67             beg_line = beg_start + format % self.display() + trail*'>'
  68             end_line = end_start + format % self.display() + trail*'<'
  69             print beg_line
  70             for step in self.substeps:
  71                 Step(step).print_doc(level+1)
  72             print end_line
  73
  74     # return a list of (name, method) for all native steps involved
  75     def tuples (self):
  76         if self.native:
  77             return [ (self.internal(), self.method,) ]
  78         else:
  79             result = []
  80             for substep in [ Step(name) for name in self.substeps ] :
  81                 result += substep.tuples()
  82             return result
  83
  84     # convenience for listing macros
  85     # just do a listdir, hoping we're in the right directory...
  86     @staticmethod
  87     def list_macros ():
  88         names= sequences.keys()
  89         names.sort()
  90         return names
  91
  92 class TestMain:
  93
  94     default_config = [ 'default' ]
  95 #    default_rspec_styles = [ 'pl', 'pg' ]
  96     default_rspec_styles = [ 'pg' ]
  97
  98     default_build_url = "git://git.onelab.eu/tests"
  99
 100     def __init__(self):
 101         self.path = os.path.dirname(sys.argv[0]) or "."
 102         os.chdir(self.path)
 103
 104     def show_env(self, options, message):
 105         if self.options.verbose:
 106             utils.header(message)
 107             utils.show_options("main options", options)
 108
 109     def init_steps(self):
 110         self.steps_message  = ""
 111         if not self.options.bonding:
 112             self.steps_message += 20*'x' + " Defaut steps are\n" + \
 113                                   TestPlc.printable_steps(TestPlc.default_steps)
 114             self.steps_message += 20*'x' + " Other useful steps are\n" + \
 115                                   TestPlc.printable_steps(TestPlc.other_steps)
 116             self.steps_message += 20*'x' + " Macro steps are\n" + \
 117                                   " ".join(Step.list_macros())
 118         else:
 119             self.steps_message += 20*'x' + " Default steps with bonding are\n" + \
 120                                   TestPlc.printable_steps(TestPlc.bonding_steps)
 121
 122     def list_steps(self):
 123         if not self.options.verbose:
 124             print self.steps_message
 125         else:
 126             # steps mentioned on the command line
 127             if self.options.args:
 128                 scopes = [("Argument steps",self.options.args)]
 129             else:
 130                 scopes = [("Default steps", TestPlc.default_steps)]
 131                 if self.options.all_steps:
 132                     scopes.append ( ("Other steps", TestPlc.other_steps) )
 133                     # try to list macro steps as well
 134                     scopes.append ( ("Macro steps", Step.list_macros()) )
 135             for (scope, steps) in scopes:
 136                 print '--------------------', scope
 137                 for step in [step for step in steps if TestPlc.valid_step(step)]:
 138                     try:
 139                         (step, qualifier) = step.split('@')
 140                     except:
 141                         pass
 142                     stepname = step
 143                     for special in ['force', 'ignore']:
 144                         stepname = stepname.replace('_'+special, "")
 145                     Step(stepname).print_doc()
 146
 147     def run (self):
 148         usage = """usage: %%prog [options] steps
 149 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
 150    no default
 151 config defaults to the last value used, as stored in arg-config,
 152    or %r
 153 ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode},
 154    default is to use IP scanning
 155 steps refer to a method in TestPlc or to a step_* module
 156
 157 run with -l to see a list of available steps
 158 ===
 159 """%(TestMain.default_config)
 160
 161         parser = ArgumentParser(usage = usage)
 162         parser.add_argument("-u", "--url", action="store",  dest="arch_rpms_url",
 163                             help="URL of the arch-dependent RPMS area - for locating what to test")
 164         parser.add_argument("-b", "--build", action="store", dest="build_url",
 165                             help="ignored, for legacy only")
 166         parser.add_argument("-c", "--config", action="append", dest="config", default=[],
 167                             help="Config module - can be set multiple times, or use quotes")
 168         parser.add_argument("-p", "--personality", action="store", dest="personality",
 169                             help="personality - as in vbuild-nightly")
 170         parser.add_argument("-d", "--pldistro", action="store", dest="pldistro",
 171                             help="pldistro - as in vbuild-nightly")
 172         parser.add_argument("-f", "--fcdistro", action="store", dest="fcdistro",
 173                             help="fcdistro - as in vbuild-nightly")
 174         parser.add_argument("-e", "--exclude", action="append", dest="exclude", default=[],
 175                             help="steps to exclude - can be set multiple times, or use quotes")
 176         parser.add_argument("-i", "--ignore", action="append", dest="ignore", default=[],
 177                             help="steps to run but ignore - can be set multiple times, or use quotes")
 178         parser.add_argument("-a", "--all", action="store_true", dest="all_steps", default=False,
 179                             help="Run all default steps")
 180         parser.add_argument("-l", "--list", action="store_true", dest="list_steps", default=False,
 181                             help="List known steps")
 182         parser.add_argument("-V", "--vserver", action="append", dest="ips_bplc", default=[],
 183                             help="Specify the set of hostnames for the boxes that host the plcs")
 184         parser.add_argument("-P", "--plcs", action="append", dest="ips_vplc", default=[],
 185                             help="Specify the set of hostname/IP's to use for vplcs")
 186         parser.add_argument("-Q", "--qemus", action="append", dest="ips_bnode", default=[],
 187                             help="Specify the set of hostnames for the boxes that host the nodes")
 188         parser.add_argument("-N", "--nodes", action="append", dest="ips_vnode", default=[],
 189                             help="Specify the set of hostname/IP's to use for vnodes")
 190         parser.add_argument("-s", "--size", action="store", dest="size", default=1,
 191                             type=int,
 192                             help="set test size in # of plcs - default is 1")
 193         parser.add_argument("-q", "--qualifier", action="store", dest="qualifier", default=None,
 194                             type=int,
 195                             help="run steps only on plc numbered <qualifier>, starting at 1")
 196         parser.add_argument("-y", "--rspec-style", action="append", dest="rspec_styles", default=[],
 197                             help="pl is for planetlab rspecs, pg is for protogeni")
 198         parser.add_argument("-k", "--keep-going", action="store", dest="keep_going", default=False,
 199                             help="proceeds even if some steps are failing")
 200         parser.add_argument("-D", "--dbname", action="store", dest="dbname", default=None,
 201                             help="Used by plc_db_dump and plc_db_restore")
 202         parser.add_argument("-v", "--verbose", action="store_true", dest="verbose", default=False,
 203                             help="Run in verbose mode")
 204         parser.add_argument("-I", "--interactive", action="store_true", dest="interactive", default=False,
 205                             help="prompts before each step")
 206         parser.add_argument("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
 207                             help="Show environment and exits")
 208         parser.add_argument("-t", "--trace", action="store", dest="trace_file", default=None,
 209                             help="Trace file location")
 210         parser.add_argument("-g", "--bonding", action='store', dest='bonding', default=None,
 211                             help="specify build to bond with")
 212         parser.add_argument("steps", nargs='*')
 213         self.options = parser.parse_args()
 214
 215         # allow things like "run -c 'c1 c2' -c c3"
 216         def flatten (x):
 217             result = []
 218             for el in x:
 219                 if hasattr(el, "__iter__") and not isinstance(el, basestring):
 220                     result.extend(flatten(el))
 221                 else:
 222                     result.append(el)
 223             return result
 224         # flatten relevant options
 225         for optname in ['config', 'exclude', 'ignore', 'ips_bplc', 'ips_vplc', 'ips_bnode', 'ips_vnode']:
 226             setattr(self.options, optname,
 227                     flatten([arg.split() for arg in getattr(self.options, optname)]))
 228
 229         if not self.options.rspec_styles:
 230             self.options.rspec_styles = TestMain.default_rspec_styles
 231
 232         # handle defaults and option persistence
 233         for recname, filename, default, need_reverse in (
 234             ('build_url', 'arg-build-url', TestMain.default_build_url, None),
 235             ('ips_bplc', 'arg-ips-bplc', [], True),
 236             ('ips_vplc', 'arg-ips-vplc', [], True),
 237             ('ips_bnode', 'arg-ips-bnode', [], True),
 238             ('ips_vnode', 'arg-ips-vnode', [], True),
 239             ('config', 'arg-config', TestMain.default_config, False),
 240             ('arch_rpms_url', 'arg-arch-rpms-url', "", None),
 241             ('personality', 'arg-personality', "linux64", None),
 242             ('pldistro', 'arg-pldistro', "onelab", None),
 243             ('fcdistro', 'arg-fcdistro', 'f14', None),
 244             ):
 245 #            print 'handling',recname
 246             path = filename
 247             is_list = isinstance(default, list)
 248             is_bool = isinstance(default, bool)
 249             if not getattr(self.options, recname):
 250                 try:
 251                     parsed = file(path).readlines()
 252                     if is_list:         # lists
 253                         parsed = [x.strip() for x in parsed]
 254                     else:               # strings and booleans
 255                         if len(parsed) != 1:
 256                             print "%s - error when parsing %s" % (sys.argv[1],path)
 257                             sys.exit(1)
 258                         parsed = parsed[0].strip()
 259                         if is_bool:
 260                             parsed = parsed.lower() == 'true'
 261                     setattr(self.options, recname, parsed)
 262                 except:
 263                     if default != "":
 264                         setattr(self.options, recname, default)
 265                     else:
 266                         print "Cannot determine", recname
 267                         print "Run %s --help for help" % sys.argv[0]
 268                         sys.exit(1)
 269
 270             # save for next run
 271             fsave = open(path, "w")
 272             if is_list:                 # lists
 273                 for value in getattr(self.options, recname):
 274                     fsave.write(value + "\n")
 275             else:                       # strings and booleans - just call str()
 276                 fsave.write(str(getattr(self.options, recname)) + "\n")
 277             fsave.close()
 278 #            utils.header('Saved %s into %s'%(recname,filename))
 279
 280             # lists need be reversed
 281             # I suspect this is useful for the various pools but for config, it's painful
 282             if isinstance(getattr(self.options, recname), list) and need_reverse:
 283                 getattr(self.options, recname).reverse()
 284
 285             if self.options.verbose:
 286                 utils.header('* Using %s = %s' % (recname, getattr(self.options, recname)))
 287
 288         # hack : if sfa is not among the published rpms, skip these tests
 289         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
 290
 291         # initialize steps
 292         if not self.options.steps:
 293             # defaults, depends on using bonding or not
 294             if self.options.bonding:
 295                 self.options.steps = TestPlc.bonding_steps
 296             else:
 297                 self.options.steps = TestPlc.default_steps
 298
 299         if self.options.list_steps:
 300             self.init_steps()
 301             self.list_steps()
 302             return 'SUCCESS'
 303
 304         # rewrite '-' into '_' in step names
 305         self.options.steps   = [ step.replace('-', '_') for step in self.options.steps ]
 306         self.options.exclude = [ step.replace('-', '_') for step in self.options.exclude ]
 307         self.options.ignore  = [ step.replace('-', '_') for step in self.options.ignore ]
 308
 309         # technicality, decorate known steps to produce the '_ignore' version
 310         TestPlc.create_ignore_steps()
 311
 312         # exclude
 313         selected = []
 314         for step in self.options.steps:
 315             keep = True
 316             for exclude in self.options.exclude:
 317                 if utils.match(step, exclude):
 318                     keep = False
 319                     break
 320             if keep:
 321                 selected.append(step)
 322
 323         # ignore
 324         selected = [ step if step not in self.options.ignore else step + "_ignore"
 325                      for step in selected ]
 326
 327         self.options.steps = selected
 328
 329         # this is useful when propagating on host boxes, to avoid conflicts
 330         self.options.buildname = os.path.basename(os.path.abspath(self.path))
 331
 332         if self.options.verbose:
 333             self.show_env(self.options, "Verbose")
 334
 335         # load configs
 336         all_plc_specs = []
 337         for config in self.options.config:
 338             modulename = 'config_' + config
 339             try:
 340                 m = __import__(modulename)
 341                 all_plc_specs = m.config(all_plc_specs, self.options)
 342             except :
 343                 traceback.print_exc()
 344                 print 'Cannot load config %s -- ignored' % modulename
 345                 raise
 346
 347         # provision on local substrate
 348         all_plc_specs = LocalSubstrate.local_substrate.provision(all_plc_specs, self.options)
 349
 350         # remember substrate IP address(es) for next run
 351         ips_bplc_file = open('arg-ips-bplc', 'w')
 352         for plc_spec in all_plc_specs:
 353             ips_bplc_file.write("%s\n" % plc_spec['host_box'])
 354         ips_bplc_file.close()
 355         ips_vplc_file = open('arg-ips-vplc', 'w')
 356         for plc_spec in all_plc_specs:
 357             ips_vplc_file.write("%s\n" % plc_spec['settings']['PLC_API_HOST'])
 358         ips_vplc_file.close()
 359         # ditto for nodes
 360         ips_bnode_file = open('arg-ips-bnode', 'w')
 361         for plc_spec in all_plc_specs:
 362             for site_spec in plc_spec['sites']:
 363                 for node_spec in site_spec['nodes']:
 364                     ips_bnode_file.write("%s\n" % node_spec['host_box'])
 365         ips_bnode_file.close()
 366         ips_vnode_file = open('arg-ips-vnode','w')
 367         for plc_spec in all_plc_specs:
 368             for site_spec in plc_spec['sites']:
 369                 for node_spec in site_spec['nodes']:
 370                     # back to normal (unqualified) form
 371                     stripped = node_spec['node_fields']['hostname'].split('.')[0]
 372                     ips_vnode_file.write("%s\n" % stripped)
 373         ips_vnode_file.close()
 374
 375         # build a TestPlc object from the result, passing options
 376         for spec in all_plc_specs:
 377             spec['failed_step'] = False
 378         all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs]
 379
 380         # pass options to utils as well
 381         utils.init_options(self.options)
 382
 383         # populate TestBonding objects
 384         # need to wait until here as we need all_plcs
 385         if self.options.bonding:
 386             ## allow to pass -g ../2015.03.15--f18 so we can use bash completion
 387             self.options.bonding = os.path.basename(self.options.bonding)
 388             # this will fail if ../{bonding} has not the right arg- files
 389             for spec, test_plc in all_plcs:
 390                 test_plc.test_bonding = TestBonding (test_plc,
 391                                                      onelab_bonding_spec(self.options.bonding),
 392                                                      self.options)
 393
 394         overall_result = 'SUCCESS'
 395         all_step_infos = []
 396         for step in self.options.steps:
 397             if not TestPlc.valid_step(step):
 398                 continue
 399             # some steps need to be done regardless of the previous ones: we force them
 400             force = False
 401             if step.endswith("_force"):
 402                 step = step.replace("_force", "")
 403                 force = True
 404             # allow for steps to specify an index like in
 405             # run checkslice@2
 406             try:
 407                 step, qualifier = step.split('@')
 408             except:
 409                 qualifier = self.options.qualifier
 410
 411             try:
 412                 stepobj = Step (step)
 413                 for substep, method in stepobj.tuples():
 414                     # a cross step will run a method on TestPlc that has a signature like
 415                     # def cross_foo (self, all_test_plcs)
 416                     cross = False
 417                     if substep.find("cross_") == 0:
 418                         cross = True
 419                     all_step_infos.append ( (substep, method, force, cross, qualifier, ) )
 420             except :
 421                 utils.header("********** FAILED step %s (NOT FOUND) -- won't be run" % step)
 422                 traceback.print_exc()
 423                 overall_result = 'FAILURE'
 424
 425         if self.options.dry_run:
 426             self.show_env(self.options, "Dry run")
 427
 428         # init & open trace file if provided
 429         if self.options.trace_file and not self.options.dry_run:
 430             # create dir if needed
 431             trace_dir = os.path.dirname(self.options.trace_file)
 432             if trace_dir and not os.path.isdir(trace_dir):
 433                 os.makedirs(trace_dir)
 434             trace = open(self.options.trace_file,"w")
 435
 436         # do all steps on all plcs
 437         TIME_FORMAT = "%H-%M-%S"
 438         TRACE_FORMAT = "TRACE: %(plc_counter)d %(begin)s->%(seconds)ss=%(duration)s " + \
 439                        "status=%(status)s step=%(stepname)s plc=%(plcname)s force=%(force)s\n"
 440         for stepname, method, force, cross, qualifier in all_step_infos:
 441             plc_counter = 0
 442             for spec, plc_obj in all_plcs:
 443                 plc_counter += 1
 444                 # skip this step if we have specified a plc_explicit
 445                 if qualifier and plc_counter != int(qualifier):
 446                     continue
 447
 448                 plcname = spec['name']
 449                 across_plcs = [ o for (s,o) in all_plcs if o!=plc_obj ]
 450
 451                 # run the step
 452                 beg_time = datetime.now()
 453                 begin = beg_time.strftime(TIME_FORMAT)
 454                 if not spec['failed_step'] or force or self.options.interactive or self.options.keep_going:
 455                     skip_step = False
 456                     if self.options.interactive:
 457                         prompting = True
 458                         while prompting:
 459                             msg="%d Run step %s on %s [r](un)/d(ry_run)/p(roceed)/s(kip)/q(uit) ? " % \
 460                                 (plc_counter,stepname,plcname)
 461                             answer = raw_input(msg).strip().lower() or "r"
 462                             answer = answer[0]
 463                             if answer in ['s','n']:     # skip/no/next
 464                                 print '%s on %s skipped' % (stepname, plcname)
 465                                 prompting = False
 466                                 skip_step = True
 467                             elif answer in ['q','b']:   # quit/bye
 468                                 print 'Exiting'
 469                                 return 'FAILURE'
 470                             elif answer in ['d']:       # dry_run
 471                                 dry_run = self.options.dry_run
 472                                 self.options.dry_run = True
 473                                 plc_obj.options.dry_run = True
 474                                 plc_obj.apiserver.set_dry_run(True)
 475                                 if not cross:
 476                                     step_result=method(plc_obj)
 477                                 else:
 478                                     step_result=method(plc_obj, across_plcs)
 479                                 print 'dry_run step ->', step_result
 480                                 self.options.dry_run = dry_run
 481                                 plc_obj.options.dry_run = dry_run
 482                                 plc_obj.apiserver.set_dry_run(dry_run)
 483                             elif answer in ['p']:
 484                                 # take it as a yes and leave interactive mode
 485                                 prompting = False
 486                                 self.options.interactive = False
 487                             elif answer in ['r','y']:   # run/yes
 488                                 prompting = False
 489                     if skip_step:
 490                         continue
 491                     try:
 492                         force_msg = ""
 493                         if force and spec['failed_step']:
 494                             force_msg=" (forced after %s has failed)" % spec['failed_step']
 495                         utils.header("********** %d RUNNING step %s%s on plc %s" % \
 496                                      (plc_counter, stepname, force_msg, plcname))
 497                         if not cross:
 498                             step_result = method(plc_obj)
 499                         else:
 500                             step_result = method(plc_obj, across_plcs)
 501                         if isinstance (step_result, Ignored):
 502                             step_result = step_result.result
 503                             if step_result:
 504                                 msg = "OK"
 505                             else:
 506                                 msg = "KO"
 507                                 # do not overwrite if FAILURE
 508                                 if overall_result == 'SUCCESS':
 509                                     overall_result = 'IGNORED'
 510                             utils.header('********** %d IGNORED (%s) step %s on %s' % \
 511                                          (plc_counter, msg, stepname, plcname))
 512                             status="%s[I]" % msg
 513                         elif step_result:
 514                             utils.header('********** %d SUCCESSFUL step %s on %s' % \
 515                                          (plc_counter, stepname, plcname))
 516                             status = "OK"
 517                         else:
 518                             overall_result = 'FAILURE'
 519                             spec['failed_step'] = stepname
 520                             utils.header('********** %d FAILED step %s on %s (discarded from further steps)' % \
 521                                          (plc_counter, stepname, plcname))
 522                             status = "KO"
 523                     except:
 524                         overall_result = 'FAILURE'
 525                         spec['failed_step'] = stepname
 526                         traceback.print_exc()
 527                         utils.header ('********** %d FAILED (exception) step %s on %s (discarded from further steps)' % \
 528                                       (plc_counter, stepname, plcname))
 529                         status = "KO"
 530
 531                 # do not run, just display it's skipped
 532                 else:
 533                     why = "has failed %s" % spec['failed_step']
 534                     utils.header("********** %d SKIPPED Step %s on %s (%s)" % \
 535                                  (plc_counter, stepname, plcname, why))
 536                     status = "UNDEF"
 537                 if not self.options.dry_run:
 538                     delay = datetime.now()-beg_time
 539                     seconds = int(delay.total_seconds())
 540                     duration = str(delay)
 541                     # always do this on stdout
 542                     print TRACE_FORMAT % locals()
 543                     # duplicate on trace_file if provided
 544                     if self.options.trace_file:
 545                         trace.write(TRACE_FORMAT % locals())
 546                         trace.flush()
 547
 548         if self.options.trace_file and not self.options.dry_run:
 549             trace.close()
 550
 551         # free local substrate
 552         LocalSubstrate.local_substrate.release(self.options)
 553
 554         return overall_result
 555
 556     # wrapper to run, returns a shell-compatible result
 557     # retcod:
 558     # 0: SUCCESS
 559     # 1: FAILURE
 560     # 2: SUCCESS but some ignored steps failed
 561     # 3: OTHER ERROR
 562     def main(self):
 563         try:
 564             success = self.run()
 565             if success == 'SUCCESS':
 566                 return 0
 567             elif success == 'IGNORED':
 568                 return 2
 569             else:
 570                 return 1
 571         except SystemExit:
 572             print 'Caught SystemExit'
 573             return 3
 574         except:
 575             traceback.print_exc()
 576             return 3
 577
 578 if __name__ == "__main__":
 579     exit_code = TestMain().main()
 580     print "TestMain exit code", exit_code
 581     sys.exit(exit_code)