system/TestMain.py

   1 #!/usr/bin/python -u
   2
   3 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
   4 # Copyright (C) 2010 INRIA
   5 #
   6 import sys, os, os.path
   7 from optparse import OptionParser
   8 import traceback
   9 import readline
  10 import glob
  11 from datetime import datetime
  12
  13 import utils
  14 from TestPlc import TestPlc, Ignored
  15 from TestSite import TestSite
  16 from TestNode import TestNode
  17 from macros import sequences
  18
  19 # add $HOME in PYTHONPATH so we can import LocalSubstrate.py
  20 sys.path.append(os.environ['HOME'])
  21 import LocalSubstrate
  22
  23 class Step:
  24
  25     natives = TestPlc.__dict__
  26
  27     def display (self):
  28         return self.name.replace('_', '-')
  29     def internal (self):
  30         return self.name.replace('-', '_')
  31
  32     def __init__ (self, name):
  33         self.name = name
  34         # a native step is implemented as a method on TestPlc
  35         self.native = self.internal() in Step.natives
  36         if self.native:
  37             self.method = Step.natives[self.internal()]
  38         else:
  39             try:
  40                 self.substeps = sequences[self.internal()]
  41             except Exception,e:
  42                 print "macro step %s not found in macros.py (%s) - exiting" % (self.display(),e)
  43                 raise
  44
  45     def print_doc (self, level=0):
  46         tab = 32
  47         trail = 8
  48         if self.native:
  49             start = level*' ' + '* '
  50             # 2 is the len of '* '
  51             width = tab - level - 2
  52             format = "%%-%ds" % width
  53             line = start + format % self.display()
  54             print line,
  55             try:
  56                 print self.method.__doc__
  57             except:
  58                 print "*** no doc found"
  59         else:
  60             beg_start = level*' ' + '>>> '
  61             end_start = level*' ' + '<<< '
  62             trailer = trail * '-'
  63             # 4 is the len of '>>> '
  64             width = tab - level - 4 - trail
  65             format = "%%-%ds" % width
  66             beg_line = beg_start + format % self.display() + trail*'>'
  67             end_line = end_start + format % self.display() + trail*'<'
  68             print beg_line
  69             for step in self.substeps:
  70                 Step(step).print_doc(level+1)
  71             print end_line
  72
  73     # return a list of (name, method) for all native steps involved
  74     def tuples (self):
  75         if self.native:
  76             return [ (self.internal(), self.method,) ]
  77         else:
  78             result = []
  79             for substep in [ Step(name) for name in self.substeps ] :
  80                 result += substep.tuples()
  81             return result
  82
  83     # convenience for listing macros
  84     # just do a listdir, hoping we're in the right directory...
  85     @staticmethod
  86     def list_macros ():
  87         names= sequences.keys()
  88         names.sort()
  89         return names
  90
  91 class TestMain:
  92
  93     subversion_id = "Now using git -- version tracker broken"
  94
  95     default_config = [ 'default' ]
  96 #    default_rspec_styles = [ 'pl', 'pg' ]
  97     default_rspec_styles = [ 'pg' ]
  98
  99     default_build_url = "git://git.onelab.eu/tests"
 100
 101     def __init__(self):
 102         self.path = os.path.dirname(sys.argv[0]) or "."
 103         os.chdir(self.path)
 104
 105     def show_env(self, options, message):
 106         if self.options.verbose:
 107             utils.header(message)
 108             utils.show_options("main options", options)
 109
 110     def init_steps(self):
 111         self.steps_message  = 20*'x' + " Defaut steps are\n" + \
 112                               TestPlc.printable_steps(TestPlc.default_steps)
 113         self.steps_message += 20*'x' + " Other useful steps are\n" + \
 114                               TestPlc.printable_steps(TestPlc.other_steps)
 115         self.steps_message += 20*'x' + " Macro steps are\n" + \
 116                               " ".join(Step.list_macros())
 117
 118     def list_steps(self):
 119         if not self.options.verbose:
 120             print self.steps_message,
 121         else:
 122             # steps mentioned on the command line
 123             if self.options.args:
 124                 scopes = [("Argument steps",self.options.args)]
 125             else:
 126                 scopes = [("Default steps", TestPlc.default_steps)]
 127                 if self.options.all_steps:
 128                     scopes.append ( ("Other steps", TestPlc.other_steps) )
 129                     # try to list macro steps as well
 130                     scopes.append ( ("Macro steps", Step.list_macros()) )
 131             for (scope, steps) in scopes:
 132                 print '--------------------', scope
 133                 for step in [step for step in steps if TestPlc.valid_step(step)]:
 134                     try:
 135                         (step, qualifier) = step.split('@')
 136                     except:
 137                         pass
 138                     stepname = step
 139                     for special in ['force', 'ignore']:
 140                         stepname = stepname.replace('_'+special, "")
 141                     Step(stepname).print_doc()
 142
 143     def run (self):
 144         self.init_steps()
 145         usage = """usage: %%prog [options] steps
 146 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
 147    no default
 148 config defaults to the last value used, as stored in arg-config,
 149    or %r
 150 ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode},
 151    default is to use IP scanning
 152 steps refer to a method in TestPlc or to a step_* module
 153 ===
 154 """%(TestMain.default_config)
 155         usage += self.steps_message
 156
 157         parser = OptionParser(usage=usage, version=self.subversion_id)
 158         parser.add_option("-u", "--url", action="store",  dest="arch_rpms_url",
 159                           help="URL of the arch-dependent RPMS area - for locating what to test")
 160         parser.add_option("-b", "--build", action="store", dest="build_url",
 161                           help="ignored, for legacy only")
 162         parser.add_option("-c", "--config", action="append", dest="config", default=[],
 163                           help="Config module - can be set multiple times, or use quotes")
 164         parser.add_option("-p", "--personality", action="store", dest="personality",
 165                           help="personality - as in vbuild-nightly")
 166         parser.add_option("-d", "--pldistro", action="store", dest="pldistro",
 167                           help="pldistro - as in vbuild-nightly")
 168         parser.add_option("-f", "--fcdistro", action="store", dest="fcdistro",
 169                           help="fcdistro - as in vbuild-nightly")
 170         parser.add_option("-e", "--exclude", action="append", dest="exclude", default=[],
 171                           help="steps to exclude - can be set multiple times, or use quotes")
 172         parser.add_option("-i", "--ignore", action="append", dest="ignore", default=[],
 173                           help="steps to run but ignore - can be set multiple times, or use quotes")
 174         parser.add_option("-a", "--all", action="store_true", dest="all_steps", default=False,
 175                           help="Run all default steps")
 176         parser.add_option("-l", "--list", action="store_true", dest="list_steps", default=False,
 177                           help="List known steps")
 178         parser.add_option("-V", "--vserver", action="append", dest="ips_bplc", default=[],
 179                           help="Specify the set of hostnames for the boxes that host the plcs")
 180         parser.add_option("-P", "--plcs", action="append", dest="ips_vplc", default=[],
 181                           help="Specify the set of hostname/IP's to use for vplcs")
 182         parser.add_option("-Q", "--qemus", action="append", dest="ips_bnode", default=[],
 183                           help="Specify the set of hostnames for the boxes that host the nodes")
 184         parser.add_option("-N", "--nodes", action="append", dest="ips_vnode", default=[],
 185                           help="Specify the set of hostname/IP's to use for vnodes")
 186         parser.add_option("-s", "--size", action="store", type="int", dest="size", default=1,
 187                           help="sets test size in # of plcs - default is 1")
 188         parser.add_option("-q", "--qualifier", action="store", dest="qualifier", default=None,
 189                           type="int",
 190                           help="run steps only on plc numbered <qualifier>, starting at 1")
 191         parser.add_option("-y", "--rspec-style", action="append", dest="rspec_styles", default=[],
 192                           help="pl is for planetlab rspecs, pg is for protogeni")
 193         parser.add_option("-k", "--keep-going", action="store", dest="keep_going", default=False,
 194                           help="proceeds even if some steps are failing")
 195         parser.add_option("-D", "--dbname", action="store", dest="dbname", default=None,
 196                            help="Used by plc_db_dump and plc_db_restore")
 197         parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
 198                           help="Run in verbose mode")
 199         parser.add_option("-I", "--interactive", action="store_true", dest="interactive", default=False,
 200                           help="prompts before each step")
 201         parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
 202                           help="Show environment and exits")
 203         parser.add_option("-t", "--trace", action="store", dest="trace_file", default=None,
 204                           help="Trace file location")
 205         self.options, self.args = parser.parse_args()
 206
 207         # allow things like "run -c 'c1 c2' -c c3"
 208         def flatten (x):
 209             result = []
 210             for el in x:
 211                 if hasattr(el, "__iter__") and not isinstance(el, basestring):
 212                     result.extend(flatten(el))
 213                 else:
 214                     result.append(el)
 215             return result
 216         # flatten relevant options
 217         for optname in ['config', 'exclude', 'ignore', 'ips_bplc', 'ips_vplc', 'ips_bnode', 'ips_vnode']:
 218             setattr(self.options, optname,
 219                     flatten([arg.split() for arg in getattr(self.options, optname)]))
 220
 221         if not self.options.rspec_styles:
 222             self.options.rspec_styles = TestMain.default_rspec_styles
 223
 224         # handle defaults and option persistence
 225         for recname, filename, default, need_reverse in (
 226             ('build_url', 'arg-build-url', TestMain.default_build_url, None),
 227             ('ips_bplc', 'arg-ips-bplc', [], True),
 228             ('ips_vplc', 'arg-ips-vplc', [], True),
 229             ('ips_bnode', 'arg-ips-bnode', [], True),
 230             ('ips_vnode', 'arg-ips-vnode', [], True),
 231             ('config', 'arg-config', TestMain.default_config, False),
 232             ('arch_rpms_url', 'arg-arch-rpms-url', "", None),
 233             ('personality', 'arg-personality', "linux64", None),
 234             ('pldistro', 'arg-pldistro', "onelab", None),
 235             ('fcdistro', 'arg-fcdistro', 'f14', None),
 236             ):
 237 #            print 'handling',recname
 238             path = filename
 239             is_list = isinstance(default, list)
 240             is_bool = isinstance(default, bool)
 241             if not getattr(self.options, recname):
 242                 try:
 243                     parsed = file(path).readlines()
 244                     if is_list:         # lists
 245                         parsed = [x.strip() for x in parsed]
 246                     else:               # strings and booleans
 247                         if len(parsed) != 1:
 248                             print "%s - error when parsing %s" % (sys.argv[1],path)
 249                             sys.exit(1)
 250                         parsed = parsed[0].strip()
 251                         if is_bool:
 252                             parsed = parsed.lower() == 'true'
 253                     setattr(self.options, recname, parsed)
 254                 except:
 255                     if default != "":
 256                         setattr(self.options, recname, default)
 257                     else:
 258                         print "Cannot determine", recname
 259                         print "Run %s --help for help" % sys.argv[0]
 260                         sys.exit(1)
 261
 262             # save for next run
 263             fsave = open(path, "w")
 264             if is_list:                 # lists
 265                 for value in getattr(self.options, recname):
 266                     fsave.write(value + "\n")
 267             else:                       # strings and booleans - just call str()
 268                 fsave.write(str(getattr(self.options, recname)) + "\n")
 269             fsave.close()
 270 #            utils.header('Saved %s into %s'%(recname,filename))
 271
 272             # lists need be reversed
 273             # I suspect this is useful for the various pools but for config, it's painful
 274             if isinstance(getattr(self.options, recname), list) and need_reverse:
 275                 getattr(self.options, recname).reverse()
 276
 277             if self.options.verbose:
 278                 utils.header('* Using %s = %s' % (recname, getattr(self.options, recname)))
 279
 280         # hack : if sfa is not among the published rpms, skip these tests
 281         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
 282
 283         # no step specified
 284         self.options.args = self.args
 285         if len(self.args) == 0:
 286             self.options.steps = TestPlc.default_steps
 287         else:
 288             self.options.steps = self.args
 289
 290         if self.options.list_steps:
 291             self.init_steps()
 292             self.list_steps()
 293             return 'SUCCESS'
 294
 295         # steps
 296         if not self.options.steps:
 297             #default (all) steps
 298             #self.options.steps=['dump','clean','install','populate']
 299             self.options.steps = TestPlc.default_steps
 300
 301         # rewrite '-' into '_' in step names
 302         self.options.steps = [ step.replace('-', '_') for step in self.options.steps ]
 303         self.options.exclude = [ step.replace('-', '_') for step in self.options.exclude ]
 304         self.options.ignore = [ step.replace('-', '_') for step in self.options.ignore ]
 305
 306         # technicality, decorate known steps to produce the '_ignore' version
 307         TestPlc.create_ignore_steps()
 308
 309         # exclude
 310         selected = []
 311         for step in self.options.steps:
 312             keep = True
 313             for exclude in self.options.exclude:
 314                 if utils.match(step, exclude):
 315                     keep = False
 316                     break
 317             if keep:
 318                 selected.append(step)
 319
 320         # ignore
 321         selected = [ step if step not in self.options.ignore else step + "_ignore"
 322                      for step in selected ]
 323
 324         self.options.steps = selected
 325
 326         # this is useful when propagating on host boxes, to avoid conflicts
 327         self.options.buildname = os.path.basename(os.path.abspath(self.path))
 328
 329         if self.options.verbose:
 330             self.show_env(self.options, "Verbose")
 331
 332         # load configs
 333         all_plc_specs = []
 334         for config in self.options.config:
 335             modulename = 'config_' + config
 336             try:
 337                 m = __import__(modulename)
 338                 all_plc_specs = m.config(all_plc_specs, self.options)
 339             except :
 340                 traceback.print_exc()
 341                 print 'Cannot load config %s -- ignored' % modulename
 342                 raise
 343
 344         # provision on local substrate
 345         all_plc_specs = LocalSubstrate.local_substrate.provision(all_plc_specs, self.options)
 346
 347         # remember substrate IP address(es) for next run
 348         ips_bplc_file = open('arg-ips-bplc', 'w')
 349         for plc_spec in all_plc_specs:
 350             ips_bplc_file.write("%s\n" % plc_spec['host_box'])
 351         ips_bplc_file.close()
 352         ips_vplc_file = open('arg-ips-vplc', 'w')
 353         for plc_spec in all_plc_specs:
 354             ips_vplc_file.write("%s\n" % plc_spec['settings']['PLC_API_HOST'])
 355         ips_vplc_file.close()
 356         # ditto for nodes
 357         ips_bnode_file = open('arg-ips-bnode', 'w')
 358         for plc_spec in all_plc_specs:
 359             for site_spec in plc_spec['sites']:
 360                 for node_spec in site_spec['nodes']:
 361                     ips_bnode_file.write("%s\n" % node_spec['host_box'])
 362         ips_bnode_file.close()
 363         ips_vnode_file = open('arg-ips-vnode','w')
 364         for plc_spec in all_plc_specs:
 365             for site_spec in plc_spec['sites']:
 366                 for node_spec in site_spec['nodes']:
 367                     # back to normal (unqualified) form
 368                     stripped = node_spec['node_fields']['hostname'].split('.')[0]
 369                     ips_vnode_file.write("%s\n" % stripped)
 370         ips_vnode_file.close()
 371
 372         # build a TestPlc object from the result, passing options
 373         for spec in all_plc_specs:
 374             spec['failed_step'] = False
 375         all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs]
 376
 377         # pass options to utils as well
 378         utils.init_options(self.options)
 379
 380         overall_result = 'SUCCESS'
 381         all_step_infos = []
 382         for step in self.options.steps:
 383             if not TestPlc.valid_step(step):
 384                 continue
 385             # some steps need to be done regardless of the previous ones: we force them
 386             force = False
 387             if step.endswith("_force"):
 388                 step = step.replace("_force", "")
 389                 force = True
 390             # allow for steps to specify an index like in
 391             # run checkslice@2
 392             try:
 393                 step, qualifier = step.split('@')
 394             except:
 395                 qualifier = self.options.qualifier
 396
 397             try:
 398                 stepobj = Step (step)
 399                 for substep, method in stepobj.tuples():
 400                     # a cross step will run a method on TestPlc that has a signature like
 401                     # def cross_foo (self, all_test_plcs)
 402                     cross = False
 403                     if substep.find("cross_") == 0:
 404                         cross = True
 405                     all_step_infos.append ( (substep, method, force, cross, qualifier, ) )
 406             except :
 407                 utils.header("********** FAILED step %s (NOT FOUND) -- won't be run" % step)
 408                 traceback.print_exc()
 409                 overall_result = 'FAILURE'
 410
 411         if self.options.dry_run:
 412             self.show_env(self.options, "Dry run")
 413
 414         # init & open trace file if provided
 415         if self.options.trace_file and not self.options.dry_run:
 416             # create dir if needed
 417             trace_dir = os.path.dirname(self.options.trace_file)
 418             if trace_dir and not os.path.isdir(trace_dir):
 419                 os.makedirs(trace_dir)
 420             trace = open(self.options.trace_file,"w")
 421
 422         # do all steps on all plcs
 423         TIME_FORMAT = "%H-%M-%S"
 424         TRACE_FORMAT = "TRACE: %(plc_counter)d %(begin)s->%(seconds)ss=%(duration)s " + \
 425                        "status=%(status)s step=%(stepname)s plc=%(plcname)s force=%(force)s\n"
 426         for stepname, method, force, cross, qualifier in all_step_infos:
 427             plc_counter = 0
 428             for spec, plc_obj in all_plcs:
 429                 plc_counter += 1
 430                 # skip this step if we have specified a plc_explicit
 431                 if qualifier and plc_counter != int(qualifier):
 432                     continue
 433
 434                 plcname = spec['name']
 435                 across_plcs = [ o for (s,o) in all_plcs if o!=plc_obj ]
 436
 437                 # run the step
 438                 beg_time = datetime.now()
 439                 begin = beg_time.strftime(TIME_FORMAT)
 440                 if not spec['failed_step'] or force or self.options.interactive or self.options.keep_going:
 441                     skip_step = False
 442                     if self.options.interactive:
 443                         prompting = True
 444                         while prompting:
 445                             msg="%d Run step %s on %s [r](un)/d(ry_run)/p(roceed)/s(kip)/q(uit) ? " % \
 446                                 (plc_counter,stepname,plcname)
 447                             answer = raw_input(msg).strip().lower() or "r"
 448                             answer = answer[0]
 449                             if answer in ['s','n']:     # skip/no/next
 450                                 print '%s on %s skipped' % (stepname, plcname)
 451                                 prompting = False
 452                                 skip_step = True
 453                             elif answer in ['q','b']:   # quit/bye
 454                                 print 'Exiting'
 455                                 return 'FAILURE'
 456                             elif answer in ['d']:       # dry_run
 457                                 dry_run = self.options.dry_run
 458                                 self.options.dry_run = True
 459                                 plc_obj.options.dry_run = True
 460                                 plc_obj.apiserver.set_dry_run(True)
 461                                 if not cross:
 462                                     step_result=method(plc_obj)
 463                                 else:
 464                                     step_result=method(plc_obj, across_plcs)
 465                                 print 'dry_run step ->', step_result
 466                                 self.options.dry_run = dry_run
 467                                 plc_obj.options.dry_run = dry_run
 468                                 plc_obj.apiserver.set_dry_run(dry_run)
 469                             elif answer in ['p']:
 470                                 # take it as a yes and leave interactive mode
 471                                 prompting = False
 472                                 self.options.interactive = False
 473                             elif answer in ['r','y']:   # run/yes
 474                                 prompting = False
 475                     if skip_step:
 476                         continue
 477                     try:
 478                         force_msg = ""
 479                         if force and spec['failed_step']:
 480                             force_msg=" (forced after %s has failed)" % spec['failed_step']
 481                         utils.header("********** %d RUNNING step %s%s on plc %s" % \
 482                                      (plc_counter, stepname, force_msg, plcname))
 483                         if not cross:
 484                             step_result = method(plc_obj)
 485                         else:
 486                             step_result = method(plc_obj, across_plcs)
 487                         if isinstance (step_result, Ignored):
 488                             step_result = step_result.result
 489                             if step_result:
 490                                 msg = "OK"
 491                             else:
 492                                 msg = "KO"
 493                                 # do not overwrite if FAILURE
 494                                 if overall_result == 'SUCCESS':
 495                                     overall_result = 'IGNORED'
 496                             utils.header('********** %d IGNORED (%s) step %s on %s' % \
 497                                          (plc_counter, msg, stepname, plcname))
 498                             status="%s[I]" % msg
 499                         elif step_result:
 500                             utils.header('********** %d SUCCESSFUL step %s on %s' % \
 501                                          (plc_counter, stepname, plcname))
 502                             status = "OK"
 503                         else:
 504                             overall_result = 'FAILURE'
 505                             spec['failed_step'] = stepname
 506                             utils.header('********** %d FAILED step %s on %s (discarded from further steps)' % \
 507                                          (plc_counter, stepname, plcname))
 508                             status = "KO"
 509                     except:
 510                         overall_result = 'FAILURE'
 511                         spec['failed_step'] = stepname
 512                         traceback.print_exc()
 513                         utils.header ('********** %d FAILED (exception) step %s on %s (discarded from further steps)' % \
 514                                       (plc_counter, stepname, plcname))
 515                         status = "KO"
 516
 517                 # do not run, just display it's skipped
 518                 else:
 519                     why = "has failed %s" % spec['failed_step']
 520                     utils.header("********** %d SKIPPED Step %s on %s (%s)" % \
 521                                  (plc_counter, stepname, plcname, why))
 522                     status = "UNDEF"
 523                 if not self.options.dry_run:
 524                     delay = datetime.now()-beg_time
 525                     seconds = int(delay.total_seconds())
 526                     duration = str(delay)
 527                     # always do this on stdout
 528                     print TRACE_FORMAT % locals()
 529                     # duplicate on trace_file if provided
 530                     if self.options.trace_file:
 531                         trace.write(TRACE_FORMAT % locals())
 532                         trace.flush()
 533
 534         if self.options.trace_file and not self.options.dry_run:
 535             trace.close()
 536
 537         # free local substrate
 538         LocalSubstrate.local_substrate.release(self.options)
 539
 540         return overall_result
 541
 542     # wrapper to run, returns a shell-compatible result
 543     # retcod:
 544     # 0: SUCCESS
 545     # 1: FAILURE
 546     # 2: SUCCESS but some ignored steps failed
 547     # 3: OTHER ERROR
 548     def main(self):
 549         try:
 550             success = self.run()
 551             if success == 'SUCCESS':
 552                 return 0
 553             elif success == 'IGNORED':
 554                 return 2
 555             else:
 556                 return 1
 557         except SystemExit:
 558             print 'Caught SystemExit'
 559             return 3
 560         except:
 561             traceback.print_exc()
 562             return 3
 563
 564 if __name__ == "__main__":
 565     exit_code = TestMain().main()
 566     print "TestMain exit code", exit_code
 567     sys.exit(exit_code)