system/TestMain.py

   1 #!/usr/bin/python -u
   2
   3 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
   4 # Copyright (C) 2010 INRIA
   5 #
   6 import sys, os, os.path
   7 from argparse import ArgumentParser
   8 import traceback
   9 import readline
  10 import glob
  11 from datetime import datetime
  12
  13 import utils
  14 from TestPlc import TestPlc, Ignored
  15 from TestSite import TestSite
  16 from TestNode import TestNode
  17 from macros import sequences
  18
  19 # add $HOME in PYTHONPATH so we can import LocalSubstrate.py
  20 sys.path.append(os.environ['HOME'])
  21 import LocalSubstrate
  22
  23 class Step:
  24
  25     natives = TestPlc.__dict__
  26
  27     def display (self):
  28         return self.name.replace('_', '-')
  29     def internal (self):
  30         return self.name.replace('-', '_')
  31
  32     def __init__ (self, name):
  33         self.name = name
  34         # a native step is implemented as a method on TestPlc
  35         self.native = self.internal() in Step.natives
  36         if self.native:
  37             self.method = Step.natives[self.internal()]
  38         else:
  39             try:
  40                 self.substeps = sequences[self.internal()]
  41             except Exception,e:
  42                 print "macro step %s not found in macros.py (%s) - exiting" % (self.display(),e)
  43                 raise
  44
  45     def print_doc (self, level=0):
  46         tab = 32
  47         trail = 8
  48         if self.native:
  49             start = level*' ' + '* '
  50             # 2 is the len of '* '
  51             width = tab - level - 2
  52             format = "%%-%ds" % width
  53             line = start + format % self.display()
  54             print line,
  55             try:
  56                 print self.method.__doc__
  57             except:
  58                 print "*** no doc found"
  59         else:
  60             beg_start = level*' ' + '>>> '
  61             end_start = level*' ' + '<<< '
  62             trailer = trail * '-'
  63             # 4 is the len of '>>> '
  64             width = tab - level - 4 - trail
  65             format = "%%-%ds" % width
  66             beg_line = beg_start + format % self.display() + trail*'>'
  67             end_line = end_start + format % self.display() + trail*'<'
  68             print beg_line
  69             for step in self.substeps:
  70                 Step(step).print_doc(level+1)
  71             print end_line
  72
  73     # return a list of (name, method) for all native steps involved
  74     def tuples (self):
  75         if self.native:
  76             return [ (self.internal(), self.method,) ]
  77         else:
  78             result = []
  79             for substep in [ Step(name) for name in self.substeps ] :
  80                 result += substep.tuples()
  81             return result
  82
  83     # convenience for listing macros
  84     # just do a listdir, hoping we're in the right directory...
  85     @staticmethod
  86     def list_macros ():
  87         names= sequences.keys()
  88         names.sort()
  89         return names
  90
  91 class TestMain:
  92
  93     default_config = [ 'default' ]
  94 #    default_rspec_styles = [ 'pl', 'pg' ]
  95     default_rspec_styles = [ 'pg' ]
  96
  97     default_build_url = "git://git.onelab.eu/tests"
  98
  99     def __init__(self):
 100         self.path = os.path.dirname(sys.argv[0]) or "."
 101         os.chdir(self.path)
 102
 103     def show_env(self, options, message):
 104         if self.options.verbose:
 105             utils.header(message)
 106             utils.show_options("main options", options)
 107
 108     def init_steps(self):
 109         self.steps_message  = 20*'x' + " Defaut steps are\n" + \
 110                               TestPlc.printable_steps(TestPlc.default_steps)
 111         self.steps_message += 20*'x' + " Other useful steps are\n" + \
 112                               TestPlc.printable_steps(TestPlc.other_steps)
 113         self.steps_message += 20*'x' + " Macro steps are\n" + \
 114                               " ".join(Step.list_macros())
 115
 116     def list_steps(self):
 117         if not self.options.verbose:
 118             print self.steps_message
 119         else:
 120             # steps mentioned on the command line
 121             if self.options.args:
 122                 scopes = [("Argument steps",self.options.args)]
 123             else:
 124                 scopes = [("Default steps", TestPlc.default_steps)]
 125                 if self.options.all_steps:
 126                     scopes.append ( ("Other steps", TestPlc.other_steps) )
 127                     # try to list macro steps as well
 128                     scopes.append ( ("Macro steps", Step.list_macros()) )
 129             for (scope, steps) in scopes:
 130                 print '--------------------', scope
 131                 for step in [step for step in steps if TestPlc.valid_step(step)]:
 132                     try:
 133                         (step, qualifier) = step.split('@')
 134                     except:
 135                         pass
 136                     stepname = step
 137                     for special in ['force', 'ignore']:
 138                         stepname = stepname.replace('_'+special, "")
 139                     Step(stepname).print_doc()
 140
 141     def run (self):
 142         self.init_steps()
 143         usage = """usage: %%prog [options] steps
 144 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
 145    no default
 146 config defaults to the last value used, as stored in arg-config,
 147    or %r
 148 ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode},
 149    default is to use IP scanning
 150 steps refer to a method in TestPlc or to a step_* module
 151 ===
 152 """%(TestMain.default_config)
 153         usage += self.steps_message
 154
 155         parser = ArgumentParser(usage = usage)
 156         parser.add_argument("-u", "--url", action="store",  dest="arch_rpms_url",
 157                             help="URL of the arch-dependent RPMS area - for locating what to test")
 158         parser.add_argument("-b", "--build", action="store", dest="build_url",
 159                             help="ignored, for legacy only")
 160         parser.add_argument("-c", "--config", action="append", dest="config", default=[],
 161                             help="Config module - can be set multiple times, or use quotes")
 162         parser.add_argument("-p", "--personality", action="store", dest="personality",
 163                             help="personality - as in vbuild-nightly")
 164         parser.add_argument("-d", "--pldistro", action="store", dest="pldistro",
 165                             help="pldistro - as in vbuild-nightly")
 166         parser.add_argument("-f", "--fcdistro", action="store", dest="fcdistro",
 167                             help="fcdistro - as in vbuild-nightly")
 168         parser.add_argument("-e", "--exclude", action="append", dest="exclude", default=[],
 169                             help="steps to exclude - can be set multiple times, or use quotes")
 170         parser.add_argument("-i", "--ignore", action="append", dest="ignore", default=[],
 171                             help="steps to run but ignore - can be set multiple times, or use quotes")
 172         parser.add_argument("-a", "--all", action="store_true", dest="all_steps", default=False,
 173                             help="Run all default steps")
 174         parser.add_argument("-l", "--list", action="store_true", dest="list_steps", default=False,
 175                             help="List known steps")
 176         parser.add_argument("-V", "--vserver", action="append", dest="ips_bplc", default=[],
 177                             help="Specify the set of hostnames for the boxes that host the plcs")
 178         parser.add_argument("-P", "--plcs", action="append", dest="ips_vplc", default=[],
 179                             help="Specify the set of hostname/IP's to use for vplcs")
 180         parser.add_argument("-Q", "--qemus", action="append", dest="ips_bnode", default=[],
 181                             help="Specify the set of hostnames for the boxes that host the nodes")
 182         parser.add_argument("-N", "--nodes", action="append", dest="ips_vnode", default=[],
 183                             help="Specify the set of hostname/IP's to use for vnodes")
 184         parser.add_argument("-s", "--size", action="store", dest="size", default=1,
 185                             type=int,
 186                             help="set test size in # of plcs - default is 1")
 187         parser.add_argument("-q", "--qualifier", action="store", dest="qualifier", default=None,
 188                             type=int,
 189                             help="run steps only on plc numbered <qualifier>, starting at 1")
 190         parser.add_argument("-y", "--rspec-style", action="append", dest="rspec_styles", default=[],
 191                             help="pl is for planetlab rspecs, pg is for protogeni")
 192         parser.add_argument("-k", "--keep-going", action="store", dest="keep_going", default=False,
 193                             help="proceeds even if some steps are failing")
 194         parser.add_argument("-D", "--dbname", action="store", dest="dbname", default=None,
 195                             help="Used by plc_db_dump and plc_db_restore")
 196         parser.add_argument("-v", "--verbose", action="store_true", dest="verbose", default=False,
 197                             help="Run in verbose mode")
 198         parser.add_argument("-I", "--interactive", action="store_true", dest="interactive", default=False,
 199                             help="prompts before each step")
 200         parser.add_argument("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
 201                             help="Show environment and exits")
 202         parser.add_argument("-t", "--trace", action="store", dest="trace_file", default=None,
 203                             help="Trace file location")
 204 #        parser.add_argument("-g", "--bonding", action='store', dest='bonding', default=None,
 205 #                            help="specify build to bond with")
 206         parser.add_argument("steps", nargs='*')
 207         self.options = parser.parse_args()
 208
 209         # allow things like "run -c 'c1 c2' -c c3"
 210         def flatten (x):
 211             result = []
 212             for el in x:
 213                 if hasattr(el, "__iter__") and not isinstance(el, basestring):
 214                     result.extend(flatten(el))
 215                 else:
 216                     result.append(el)
 217             return result
 218         # flatten relevant options
 219         for optname in ['config', 'exclude', 'ignore', 'ips_bplc', 'ips_vplc', 'ips_bnode', 'ips_vnode']:
 220             setattr(self.options, optname,
 221                     flatten([arg.split() for arg in getattr(self.options, optname)]))
 222
 223         if not self.options.rspec_styles:
 224             self.options.rspec_styles = TestMain.default_rspec_styles
 225
 226         # handle defaults and option persistence
 227         for recname, filename, default, need_reverse in (
 228             ('build_url', 'arg-build-url', TestMain.default_build_url, None),
 229             ('ips_bplc', 'arg-ips-bplc', [], True),
 230             ('ips_vplc', 'arg-ips-vplc', [], True),
 231             ('ips_bnode', 'arg-ips-bnode', [], True),
 232             ('ips_vnode', 'arg-ips-vnode', [], True),
 233             ('config', 'arg-config', TestMain.default_config, False),
 234             ('arch_rpms_url', 'arg-arch-rpms-url', "", None),
 235             ('personality', 'arg-personality', "linux64", None),
 236             ('pldistro', 'arg-pldistro', "onelab", None),
 237             ('fcdistro', 'arg-fcdistro', 'f14', None),
 238             ):
 239 #            print 'handling',recname
 240             path = filename
 241             is_list = isinstance(default, list)
 242             is_bool = isinstance(default, bool)
 243             if not getattr(self.options, recname):
 244                 try:
 245                     parsed = file(path).readlines()
 246                     if is_list:         # lists
 247                         parsed = [x.strip() for x in parsed]
 248                     else:               # strings and booleans
 249                         if len(parsed) != 1:
 250                             print "%s - error when parsing %s" % (sys.argv[1],path)
 251                             sys.exit(1)
 252                         parsed = parsed[0].strip()
 253                         if is_bool:
 254                             parsed = parsed.lower() == 'true'
 255                     setattr(self.options, recname, parsed)
 256                 except:
 257                     if default != "":
 258                         setattr(self.options, recname, default)
 259                     else:
 260                         print "Cannot determine", recname
 261                         print "Run %s --help for help" % sys.argv[0]
 262                         sys.exit(1)
 263
 264             # save for next run
 265             fsave = open(path, "w")
 266             if is_list:                 # lists
 267                 for value in getattr(self.options, recname):
 268                     fsave.write(value + "\n")
 269             else:                       # strings and booleans - just call str()
 270                 fsave.write(str(getattr(self.options, recname)) + "\n")
 271             fsave.close()
 272 #            utils.header('Saved %s into %s'%(recname,filename))
 273
 274             # lists need be reversed
 275             # I suspect this is useful for the various pools but for config, it's painful
 276             if isinstance(getattr(self.options, recname), list) and need_reverse:
 277                 getattr(self.options, recname).reverse()
 278
 279             if self.options.verbose:
 280                 utils.header('* Using %s = %s' % (recname, getattr(self.options, recname)))
 281
 282         # hack : if sfa is not among the published rpms, skip these tests
 283         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
 284
 285         # use the default list of steps if unspecified
 286         if len(self.options.steps) == 0:
 287             self.options.steps = TestPlc.default_steps
 288
 289         if self.options.list_steps:
 290             self.init_steps()
 291             self.list_steps()
 292             return 'SUCCESS'
 293
 294         # steps
 295         if not self.options.steps:
 296             #default (all) steps
 297             #self.options.steps=['dump','clean','install','populate']
 298             self.options.steps = TestPlc.default_steps
 299
 300         # rewrite '-' into '_' in step names
 301         self.options.steps   = [ step.replace('-', '_') for step in self.options.steps ]
 302         self.options.exclude = [ step.replace('-', '_') for step in self.options.exclude ]
 303         self.options.ignore  = [ step.replace('-', '_') for step in self.options.ignore ]
 304
 305         # technicality, decorate known steps to produce the '_ignore' version
 306         TestPlc.create_ignore_steps()
 307
 308         # exclude
 309         selected = []
 310         for step in self.options.steps:
 311             keep = True
 312             for exclude in self.options.exclude:
 313                 if utils.match(step, exclude):
 314                     keep = False
 315                     break
 316             if keep:
 317                 selected.append(step)
 318
 319         # ignore
 320         selected = [ step if step not in self.options.ignore else step + "_ignore"
 321                      for step in selected ]
 322
 323         self.options.steps = selected
 324
 325         # this is useful when propagating on host boxes, to avoid conflicts
 326         self.options.buildname = os.path.basename(os.path.abspath(self.path))
 327
 328         if self.options.verbose:
 329             self.show_env(self.options, "Verbose")
 330
 331         # load configs
 332         all_plc_specs = []
 333         for config in self.options.config:
 334             modulename = 'config_' + config
 335             try:
 336                 m = __import__(modulename)
 337                 all_plc_specs = m.config(all_plc_specs, self.options)
 338             except :
 339                 traceback.print_exc()
 340                 print 'Cannot load config %s -- ignored' % modulename
 341                 raise
 342
 343         # provision on local substrate
 344         all_plc_specs = LocalSubstrate.local_substrate.provision(all_plc_specs, self.options)
 345
 346         # remember substrate IP address(es) for next run
 347         ips_bplc_file = open('arg-ips-bplc', 'w')
 348         for plc_spec in all_plc_specs:
 349             ips_bplc_file.write("%s\n" % plc_spec['host_box'])
 350         ips_bplc_file.close()
 351         ips_vplc_file = open('arg-ips-vplc', 'w')
 352         for plc_spec in all_plc_specs:
 353             ips_vplc_file.write("%s\n" % plc_spec['settings']['PLC_API_HOST'])
 354         ips_vplc_file.close()
 355         # ditto for nodes
 356         ips_bnode_file = open('arg-ips-bnode', 'w')
 357         for plc_spec in all_plc_specs:
 358             for site_spec in plc_spec['sites']:
 359                 for node_spec in site_spec['nodes']:
 360                     ips_bnode_file.write("%s\n" % node_spec['host_box'])
 361         ips_bnode_file.close()
 362         ips_vnode_file = open('arg-ips-vnode','w')
 363         for plc_spec in all_plc_specs:
 364             for site_spec in plc_spec['sites']:
 365                 for node_spec in site_spec['nodes']:
 366                     # back to normal (unqualified) form
 367                     stripped = node_spec['node_fields']['hostname'].split('.')[0]
 368                     ips_vnode_file.write("%s\n" % stripped)
 369         ips_vnode_file.close()
 370
 371         # build a TestPlc object from the result, passing options
 372         for spec in all_plc_specs:
 373             spec['failed_step'] = False
 374         all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs]
 375
 376         # pass options to utils as well
 377         utils.init_options(self.options)
 378
 379         overall_result = 'SUCCESS'
 380         all_step_infos = []
 381         for step in self.options.steps:
 382             if not TestPlc.valid_step(step):
 383                 continue
 384             # some steps need to be done regardless of the previous ones: we force them
 385             force = False
 386             if step.endswith("_force"):
 387                 step = step.replace("_force", "")
 388                 force = True
 389             # allow for steps to specify an index like in
 390             # run checkslice@2
 391             try:
 392                 step, qualifier = step.split('@')
 393             except:
 394                 qualifier = self.options.qualifier
 395
 396             try:
 397                 stepobj = Step (step)
 398                 for substep, method in stepobj.tuples():
 399                     # a cross step will run a method on TestPlc that has a signature like
 400                     # def cross_foo (self, all_test_plcs)
 401                     cross = False
 402                     if substep.find("cross_") == 0:
 403                         cross = True
 404                     all_step_infos.append ( (substep, method, force, cross, qualifier, ) )
 405             except :
 406                 utils.header("********** FAILED step %s (NOT FOUND) -- won't be run" % step)
 407                 traceback.print_exc()
 408                 overall_result = 'FAILURE'
 409
 410         if self.options.dry_run:
 411             self.show_env(self.options, "Dry run")
 412
 413         # init & open trace file if provided
 414         if self.options.trace_file and not self.options.dry_run:
 415             # create dir if needed
 416             trace_dir = os.path.dirname(self.options.trace_file)
 417             if trace_dir and not os.path.isdir(trace_dir):
 418                 os.makedirs(trace_dir)
 419             trace = open(self.options.trace_file,"w")
 420
 421         # do all steps on all plcs
 422         TIME_FORMAT = "%H-%M-%S"
 423         TRACE_FORMAT = "TRACE: %(plc_counter)d %(begin)s->%(seconds)ss=%(duration)s " + \
 424                        "status=%(status)s step=%(stepname)s plc=%(plcname)s force=%(force)s\n"
 425         for stepname, method, force, cross, qualifier in all_step_infos:
 426             plc_counter = 0
 427             for spec, plc_obj in all_plcs:
 428                 plc_counter += 1
 429                 # skip this step if we have specified a plc_explicit
 430                 if qualifier and plc_counter != int(qualifier):
 431                     continue
 432
 433                 plcname = spec['name']
 434                 across_plcs = [ o for (s,o) in all_plcs if o!=plc_obj ]
 435
 436                 # run the step
 437                 beg_time = datetime.now()
 438                 begin = beg_time.strftime(TIME_FORMAT)
 439                 if not spec['failed_step'] or force or self.options.interactive or self.options.keep_going:
 440                     skip_step = False
 441                     if self.options.interactive:
 442                         prompting = True
 443                         while prompting:
 444                             msg="%d Run step %s on %s [r](un)/d(ry_run)/p(roceed)/s(kip)/q(uit) ? " % \
 445                                 (plc_counter,stepname,plcname)
 446                             answer = raw_input(msg).strip().lower() or "r"
 447                             answer = answer[0]
 448                             if answer in ['s','n']:     # skip/no/next
 449                                 print '%s on %s skipped' % (stepname, plcname)
 450                                 prompting = False
 451                                 skip_step = True
 452                             elif answer in ['q','b']:   # quit/bye
 453                                 print 'Exiting'
 454                                 return 'FAILURE'
 455                             elif answer in ['d']:       # dry_run
 456                                 dry_run = self.options.dry_run
 457                                 self.options.dry_run = True
 458                                 plc_obj.options.dry_run = True
 459                                 plc_obj.apiserver.set_dry_run(True)
 460                                 if not cross:
 461                                     step_result=method(plc_obj)
 462                                 else:
 463                                     step_result=method(plc_obj, across_plcs)
 464                                 print 'dry_run step ->', step_result
 465                                 self.options.dry_run = dry_run
 466                                 plc_obj.options.dry_run = dry_run
 467                                 plc_obj.apiserver.set_dry_run(dry_run)
 468                             elif answer in ['p']:
 469                                 # take it as a yes and leave interactive mode
 470                                 prompting = False
 471                                 self.options.interactive = False
 472                             elif answer in ['r','y']:   # run/yes
 473                                 prompting = False
 474                     if skip_step:
 475                         continue
 476                     try:
 477                         force_msg = ""
 478                         if force and spec['failed_step']:
 479                             force_msg=" (forced after %s has failed)" % spec['failed_step']
 480                         utils.header("********** %d RUNNING step %s%s on plc %s" % \
 481                                      (plc_counter, stepname, force_msg, plcname))
 482                         if not cross:
 483                             step_result = method(plc_obj)
 484                         else:
 485                             step_result = method(plc_obj, across_plcs)
 486                         if isinstance (step_result, Ignored):
 487                             step_result = step_result.result
 488                             if step_result:
 489                                 msg = "OK"
 490                             else:
 491                                 msg = "KO"
 492                                 # do not overwrite if FAILURE
 493                                 if overall_result == 'SUCCESS':
 494                                     overall_result = 'IGNORED'
 495                             utils.header('********** %d IGNORED (%s) step %s on %s' % \
 496                                          (plc_counter, msg, stepname, plcname))
 497                             status="%s[I]" % msg
 498                         elif step_result:
 499                             utils.header('********** %d SUCCESSFUL step %s on %s' % \
 500                                          (plc_counter, stepname, plcname))
 501                             status = "OK"
 502                         else:
 503                             overall_result = 'FAILURE'
 504                             spec['failed_step'] = stepname
 505                             utils.header('********** %d FAILED step %s on %s (discarded from further steps)' % \
 506                                          (plc_counter, stepname, plcname))
 507                             status = "KO"
 508                     except:
 509                         overall_result = 'FAILURE'
 510                         spec['failed_step'] = stepname
 511                         traceback.print_exc()
 512                         utils.header ('********** %d FAILED (exception) step %s on %s (discarded from further steps)' % \
 513                                       (plc_counter, stepname, plcname))
 514                         status = "KO"
 515
 516                 # do not run, just display it's skipped
 517                 else:
 518                     why = "has failed %s" % spec['failed_step']
 519                     utils.header("********** %d SKIPPED Step %s on %s (%s)" % \
 520                                  (plc_counter, stepname, plcname, why))
 521                     status = "UNDEF"
 522                 if not self.options.dry_run:
 523                     delay = datetime.now()-beg_time
 524                     seconds = int(delay.total_seconds())
 525                     duration = str(delay)
 526                     # always do this on stdout
 527                     print TRACE_FORMAT % locals()
 528                     # duplicate on trace_file if provided
 529                     if self.options.trace_file:
 530                         trace.write(TRACE_FORMAT % locals())
 531                         trace.flush()
 532
 533         if self.options.trace_file and not self.options.dry_run:
 534             trace.close()
 535
 536         # free local substrate
 537         LocalSubstrate.local_substrate.release(self.options)
 538
 539         return overall_result
 540
 541     # wrapper to run, returns a shell-compatible result
 542     # retcod:
 543     # 0: SUCCESS
 544     # 1: FAILURE
 545     # 2: SUCCESS but some ignored steps failed
 546     # 3: OTHER ERROR
 547     def main(self):
 548         try:
 549             success = self.run()
 550             if success == 'SUCCESS':
 551                 return 0
 552             elif success == 'IGNORED':
 553                 return 2
 554             else:
 555                 return 1
 556         except SystemExit:
 557             print 'Caught SystemExit'
 558             return 3
 559         except:
 560             traceback.print_exc()
 561             return 3
 562
 563 if __name__ == "__main__":
 564     exit_code = TestMain().main()
 565     print "TestMain exit code", exit_code
 566     sys.exit(exit_code)