system/TestMain.py

   1 #!/usr/bin/python3 -u
   2
   3 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
   4 # Copyright (C) 2010 INRIA
   5 #
   6 import sys, os, os.path
   7 from argparse import ArgumentParser
   8 import traceback
   9 import readline
  10 import glob
  11 from datetime import datetime
  12
  13 import utils
  14 from TestPlc import TestPlc, Ignored
  15 from TestBonding import TestBonding, onelab_bonding_spec
  16 from TestSite import TestSite
  17 from TestNode import TestNode
  18 from macros import sequences
  19
  20 # add $HOME in PYTHONPATH so we can import LocalSubstrate.py
  21 sys.path.append(os.environ['HOME'])
  22 import LocalSubstrate
  23
  24 class Step:
  25
  26     natives = TestPlc.__dict__
  27
  28     def display (self):
  29         return self.name.replace('_', '-')
  30     def internal (self):
  31         return self.name.replace('-', '_')
  32
  33     def __init__ (self, name):
  34         self.name = name
  35         # a native step is implemented as a method on TestPlc
  36         self.native = self.internal() in Step.natives
  37         if self.native:
  38             self.method = Step.natives[self.internal()]
  39         else:
  40             try:
  41                 self.substeps = sequences[self.internal()]
  42             except Exception as e:
  43                 print("macro step {} not found in macros.py ({}) - exiting".format(self.display(),e))
  44                 raise
  45
  46     def print_doc (self, level=0):
  47         tab = 32
  48         trail = 8
  49         if self.native:
  50             start = level*' ' + '* '
  51             # 2 is the len of '* '
  52             width = tab - level - 2
  53             format = "%%-%ds" % width
  54             line = start + format % self.display()
  55             print(line, end=' ')
  56             try:
  57                 print(self.method.__doc__)
  58             except:
  59                 print("*** no doc found")
  60         else:
  61             beg_start = level*' ' + '>>> '
  62             end_start = level*' ' + '<<< '
  63             trailer = trail * '-'
  64             # 4 is the len of '>>> '
  65             width = tab - level - 4 - trail
  66             format = "%%-%ds" % width
  67             beg_line = beg_start + format % self.display() + trail*'>'
  68             end_line = end_start + format % self.display() + trail*'<'
  69             print(beg_line)
  70             for step in self.substeps:
  71                 Step(step).print_doc(level+1)
  72             print(end_line)
  73
  74     # return a list of (name, method) for all native steps involved
  75     def tuples (self):
  76         if self.native:
  77             return [ (self.internal(), self.method,) ]
  78         else:
  79             result = []
  80             for substep in [ Step(name) for name in self.substeps ] :
  81                 result += substep.tuples()
  82             return result
  83
  84     # convenience for listing macros
  85     # just do a listdir, hoping we're in the right directory...
  86     @staticmethod
  87     def list_macros ():
  88         names= list(sequences.keys())
  89         names.sort()
  90         return names
  91
  92 class TestMain:
  93
  94     default_config = [ 'default' ]
  95 #    default_rspec_styles = [ 'pl', 'pg' ]
  96     default_rspec_styles = [ 'pg' ]
  97
  98     default_build_url = "git://git.onelab.eu/tests"
  99
 100     def __init__(self):
 101         self.path = os.path.dirname(sys.argv[0]) or "."
 102         os.chdir(self.path)
 103
 104     def show_env(self, options, message):
 105         if self.options.verbose:
 106             utils.header(message)
 107             utils.show_options("main options", options)
 108
 109     def init_steps(self):
 110         self.steps_message  = ""
 111         if not self.options.bonding_build:
 112             self.steps_message += 20*'x' + " Defaut steps are\n" + \
 113                                   TestPlc.printable_steps(TestPlc.default_steps)
 114             self.steps_message += 20*'x' + " Other useful steps are\n" + \
 115                                   TestPlc.printable_steps(TestPlc.other_steps)
 116             self.steps_message += 20*'x' + " Macro steps are\n" + \
 117                                   " ".join(Step.list_macros())
 118         else:
 119             self.steps_message += 20*'x' + " Default steps with bonding build are\n" + \
 120                                   TestPlc.printable_steps(TestPlc.default_bonding_steps)
 121
 122     def list_steps(self):
 123         if not self.options.verbose:
 124             print(self.steps_message)
 125         else:
 126             # steps mentioned on the command line
 127             if self.options.steps:
 128                 scopes = [("Argument steps",self.options.steps)]
 129             else:
 130                 scopes = [("Default steps", TestPlc.default_steps)]
 131                 if self.options.all_steps:
 132                     scopes.append ( ("Other steps", TestPlc.other_steps) )
 133                     # try to list macro steps as well
 134                     scopes.append ( ("Macro steps", Step.list_macros()) )
 135             for (scope, steps) in scopes:
 136                 print('--------------------', scope)
 137                 for step in [step for step in steps if TestPlc.valid_step(step)]:
 138                     try:
 139                         (step, qualifier) = step.split('@')
 140                     except:
 141                         pass
 142                     stepname = step
 143                     for special in ['force', 'ignore']:
 144                         stepname = stepname.replace('_'+special, "")
 145                     Step(stepname).print_doc()
 146
 147     def run (self):
 148         usage = """usage: %%prog [options] steps
 149 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
 150    no default
 151 config defaults to the last value used, as stored in arg-config,
 152    or {}
 153 ips_vnode, ips_vplc and ips_qemu defaults to the last value used,
 154    as stored in arg-ips-{{bplc,vplc,bnode,vnode}},
 155    default is to use IP scanning
 156 steps refer to a method in TestPlc or to a step_* module
 157
 158 run with -l to see a list of available steps
 159 ===
 160 """.format(TestMain.default_config)
 161
 162         parser = ArgumentParser(usage = usage)
 163         parser.add_argument("-u", "--url", action="store",  dest="arch_rpms_url",
 164                             help="URL of the arch-dependent RPMS area - for locating what to test")
 165         parser.add_argument("-b", "--build", action="store", dest="build_url",
 166                             help="ignored, for legacy only")
 167         parser.add_argument("-c", "--config", action="append", dest="config", default=[],
 168                             help="Config module - can be set multiple times, or use quotes")
 169         parser.add_argument("-p", "--personality", action="store", dest="personality",
 170                             help="personality - as in vbuild-nightly")
 171         parser.add_argument("-d", "--pldistro", action="store", dest="pldistro",
 172                             help="pldistro - as in vbuild-nightly")
 173         parser.add_argument("-f", "--fcdistro", action="store", dest="fcdistro",
 174                             help="fcdistro - as in vbuild-nightly")
 175         parser.add_argument("-e", "--exclude", action="append", dest="exclude", default=[],
 176                             help="steps to exclude - can be set multiple times, or use quotes")
 177         parser.add_argument("-i", "--ignore", action="append", dest="ignore", default=[],
 178                             help="steps to run but ignore - can be set multiple times, or use quotes")
 179         parser.add_argument("-a", "--all", action="store_true", dest="all_steps", default=False,
 180                             help="Run all default steps")
 181         parser.add_argument("-l", "--list", action="store_true", dest="list_steps", default=False,
 182                             help="List known steps")
 183         parser.add_argument("-V", "--vserver", action="append", dest="ips_bplc", default=[],
 184                             help="Specify the set of hostnames for the boxes that host the plcs")
 185         parser.add_argument("-P", "--plcs", action="append", dest="ips_vplc", default=[],
 186                             help="Specify the set of hostname/IP's to use for vplcs")
 187         parser.add_argument("-Q", "--qemus", action="append", dest="ips_bnode", default=[],
 188                             help="Specify the set of hostnames for the boxes that host the nodes")
 189         parser.add_argument("-N", "--nodes", action="append", dest="ips_vnode", default=[],
 190                             help="Specify the set of hostname/IP's to use for vnodes")
 191         parser.add_argument("-s", "--size", action="store", dest="size", default=1,
 192                             type=int,
 193                             help="set test size in # of plcs - default is 1")
 194         parser.add_argument("-q", "--qualifier", action="store", dest="qualifier", default=None,
 195                             type=int,
 196                             help="run steps only on plc numbered <qualifier>, starting at 1")
 197         parser.add_argument("-y", "--rspec-style", action="append", dest="rspec_styles", default=[],
 198                             help="pl is for planetlab rspecs, pg is for protogeni")
 199         parser.add_argument("-k", "--keep-going", action="store", dest="keep_going", default=False,
 200                             help="proceeds even if some steps are failing")
 201         parser.add_argument("-D", "--dbname", action="store", dest="dbname", default=None,
 202                             help="Used by plc_db_dump and plc_db_restore")
 203         parser.add_argument("-v", "--verbose", action="store_true", dest="verbose", default=False,
 204                             help="Run in verbose mode")
 205         parser.add_argument("-I", "--interactive", action="store_true", dest="interactive", default=False,
 206                             help="prompts before each step")
 207         parser.add_argument("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
 208                             help="Show environment and exits")
 209         parser.add_argument("-t", "--trace", action="store", dest="trace_file", default=None,
 210                             help="Trace file location")
 211         parser.add_argument("-g", "--bonding", action='store', dest='bonding_build', default=None,
 212                             help="specify build to bond with")
 213         # if we call symlink 'rung' instead of just run this is equivalent to run -G
 214         bonding_default = 'rung' in sys.argv[0]
 215         parser.add_argument("-G", "--bonding-env", action='store_true', dest='bonding_env', default=bonding_default,
 216                             help="get bonding build from env. variable $bonding")
 217         parser.add_argument("steps", nargs='*')
 218         self.options = parser.parse_args()
 219
 220         # handle -G/-g options
 221         if self.options.bonding_env:
 222             if 'bonding' not in os.environ:
 223                 print("env. variable $bonding must be set with --bonding-env")
 224                 sys.exit(1)
 225             self.options.bonding_build = os.environ['bonding']
 226
 227         if self.options.bonding_build:
 228             ## allow to pass -g ../2015.03.15--f18 so we can use bash completion
 229             self.options.bonding_build = os.path.basename(self.options.bonding_build)
 230             if not os.path.isdir("../{}".format(self.options.bonding_build)):
 231                 print("could not find test dir for bonding build {}".format(self.options.bonding_build))
 232                 sys.exit(1)
 233
 234         # allow things like "run -c 'c1 c2' -c c3"
 235         def flatten (x):
 236             result = []
 237             for el in x:
 238                 if hasattr(el, "__iter__") and not isinstance(el, str):
 239                     result.extend(flatten(el))
 240                 else:
 241                     result.append(el)
 242             return result
 243         # flatten relevant options
 244         for optname in ['config', 'exclude', 'ignore', 'ips_bplc', 'ips_vplc', 'ips_bnode', 'ips_vnode']:
 245             setattr(self.options, optname,
 246                     flatten([arg.split() for arg in getattr(self.options, optname)]))
 247
 248         if not self.options.rspec_styles:
 249             self.options.rspec_styles = TestMain.default_rspec_styles
 250
 251         # handle defaults and option persistence
 252         for recname, filename, default, need_reverse in (
 253             ('build_url', 'arg-build-url', TestMain.default_build_url, None),
 254             ('ips_bplc', 'arg-ips-bplc', [], True),
 255             ('ips_vplc', 'arg-ips-vplc', [], True),
 256             ('ips_bnode', 'arg-ips-bnode', [], True),
 257             ('ips_vnode', 'arg-ips-vnode', [], True),
 258             ('config', 'arg-config', TestMain.default_config, False),
 259             ('arch_rpms_url', 'arg-arch-rpms-url', "", None),
 260             ('personality', 'arg-personality', "linux64", None),
 261             ('pldistro', 'arg-pldistro', "onelab", None),
 262             ('fcdistro', 'arg-fcdistro', 'f14', None),
 263             ):
 264 #            print('handling', recname)
 265             path = filename
 266             is_list = isinstance(default, list)
 267             is_bool = isinstance(default, bool)
 268             if not getattr(self.options, recname):
 269                 try:
 270                     with open(path) as file:
 271                         parsed = file.readlines()
 272                     if is_list:         # lists
 273                         parsed = [x.strip() for x in parsed]
 274                     else:               # strings and booleans
 275                         if len(parsed) != 1:
 276                             print("{} - error when parsing {}".format(sys.argv[1], path))
 277                             sys.exit(1)
 278                         parsed = parsed[0].strip()
 279                         if is_bool:
 280                             parsed = parsed.lower() == 'true'
 281                     setattr(self.options, recname, parsed)
 282                 except  Exception as e:
 283                     if default != "":
 284                         setattr(self.options, recname, default)
 285                     else:
 286                         print("Cannot determine", recname, e)
 287                         print("Run {} --help for help".format(sys.argv[0]))
 288                         sys.exit(1)
 289
 290             # save for next run
 291             fsave = open(path, "w")
 292             if is_list:                 # lists
 293                 for value in getattr(self.options, recname):
 294                     fsave.write(value + "\n")
 295             else:                       # strings and booleans - just call str()
 296                 fsave.write(str(getattr(self.options, recname)) + "\n")
 297             fsave.close()
 298 #            utils.header('Saved {} into {}'.format(recname, filename))
 299
 300             # lists need be reversed
 301             # I suspect this is useful for the various pools but for config, it's painful
 302             if isinstance(getattr(self.options, recname), list) and need_reverse:
 303                 getattr(self.options, recname).reverse()
 304
 305             if self.options.verbose:
 306                 utils.header('* Using {} = {}'.format(recname, getattr(self.options, recname)))
 307
 308         # hack : if sfa is not among the published rpms, skip these tests
 309         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
 310
 311         # initialize steps
 312         if not self.options.steps:
 313             # defaults, depends on using bonding or not
 314             if self.options.bonding_build:
 315                 self.options.steps = TestPlc.default_bonding_steps
 316             else:
 317                 self.options.steps = TestPlc.default_steps
 318
 319         if self.options.list_steps:
 320             self.init_steps()
 321             self.list_steps()
 322             return 'SUCCESS'
 323
 324         # rewrite '-' into '_' in step names
 325         self.options.steps   = [ step.replace('-', '_') for step in self.options.steps ]
 326         self.options.exclude = [ step.replace('-', '_') for step in self.options.exclude ]
 327         self.options.ignore  = [ step.replace('-', '_') for step in self.options.ignore ]
 328
 329         # technicality, decorate known steps to produce the '_ignore' version
 330         TestPlc.create_ignore_steps()
 331
 332         # exclude
 333         selected = []
 334         for step in self.options.steps:
 335             keep = True
 336             for exclude in self.options.exclude:
 337                 if utils.match(step, exclude):
 338                     keep = False
 339                     break
 340             if keep:
 341                 selected.append(step)
 342
 343         # ignore
 344         selected = [ step if step not in self.options.ignore else step + "_ignore"
 345                      for step in selected ]
 346
 347         self.options.steps = selected
 348
 349         # this is useful when propagating on host boxes, to avoid conflicts
 350         self.options.buildname = os.path.basename(os.path.abspath(self.path))
 351
 352         if self.options.verbose:
 353             self.show_env(self.options, "Verbose")
 354
 355         # load configs
 356         all_plc_specs = []
 357         for config in self.options.config:
 358             modulename = 'config_' + config
 359             try:
 360                 m = __import__(modulename)
 361                 all_plc_specs = m.config(all_plc_specs, self.options)
 362             except :
 363                 traceback.print_exc()
 364                 print('Cannot load config {} -- ignored'.format(modulename))
 365                 raise
 366
 367         # provision on local substrate
 368         all_plc_specs = LocalSubstrate.local_substrate.provision(all_plc_specs, self.options)
 369
 370         # remember substrate IP address(es) for next run
 371         with open('arg-ips-bplc', 'w') as ips_bplc_file:
 372             for plc_spec in all_plc_specs:
 373                 ips_bplc_file.write("{}\n".format(plc_spec['host_box']))
 374         with open('arg-ips-vplc', 'w') as ips_vplc_file:
 375             for plc_spec in all_plc_specs:
 376                 ips_vplc_file.write("{}\n".format(plc_spec['settings']['PLC_API_HOST']))
 377         # ditto for nodes
 378         with open('arg-ips-bnode', 'w') as ips_bnode_file:
 379             for plc_spec in all_plc_specs:
 380                 for site_spec in plc_spec['sites']:
 381                     for node_spec in site_spec['nodes']:
 382                         ips_bnode_file.write("{}\n".format(node_spec['host_box']))
 383         with open('arg-ips-vnode','w') as ips_vnode_file:
 384             for plc_spec in all_plc_specs:
 385                 for site_spec in plc_spec['sites']:
 386                     for node_spec in site_spec['nodes']:
 387                         # back to normal (unqualified) form
 388                         stripped = node_spec['node_fields']['hostname'].split('.')[0]
 389                         ips_vnode_file.write("{}\n".format(stripped))
 390
 391         # build a TestPlc object from the result, passing options
 392         for spec in all_plc_specs:
 393             spec['failed_step'] = False
 394         all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs]
 395
 396         # pass options to utils as well
 397         utils.init_options(self.options)
 398
 399         # populate TestBonding objects
 400         # need to wait until here as we need all_plcs
 401         if self.options.bonding_build:
 402             # this will fail if ../{bonding_build} has not the right arg- files
 403             for spec, test_plc in all_plcs:
 404                 test_plc.test_bonding = TestBonding (test_plc,
 405                                                      onelab_bonding_spec(self.options.bonding_build),
 406                                                      LocalSubstrate.local_substrate,
 407                                                      self.options)
 408
 409         overall_result = 'SUCCESS'
 410         all_step_infos = []
 411         for step in self.options.steps:
 412             if not TestPlc.valid_step(step):
 413                 continue
 414             # some steps need to be done regardless of the previous ones: we force them
 415             force = False
 416             if step.endswith("_force"):
 417                 step = step.replace("_force", "")
 418                 force = True
 419             # allow for steps to specify an index like in
 420             # run checkslice@2
 421             try:
 422                 step, qualifier = step.split('@')
 423             except:
 424                 qualifier = self.options.qualifier
 425
 426             try:
 427                 stepobj = Step (step)
 428                 for substep, method in stepobj.tuples():
 429                     # a cross step will run a method on TestPlc that has a signature like
 430                     # def cross_foo (self, all_test_plcs)
 431                     cross = False
 432                     if substep.find("cross_") == 0:
 433                         cross = True
 434                     all_step_infos.append ( (substep, method, force, cross, qualifier, ) )
 435             except :
 436                 utils.header("********** FAILED step {} (NOT FOUND) -- won't be run".format(step))
 437                 traceback.print_exc()
 438                 overall_result = 'FAILURE'
 439
 440         if self.options.dry_run:
 441             self.show_env(self.options, "Dry run")
 442
 443         # init & open trace file if provided
 444         if self.options.trace_file and not self.options.dry_run:
 445             # create dir if needed
 446             trace_dir = os.path.dirname(self.options.trace_file)
 447             if trace_dir and not os.path.isdir(trace_dir):
 448                 os.makedirs(trace_dir)
 449             trace = open(self.options.trace_file,"w")
 450
 451         # do all steps on all plcs
 452         TIME_FORMAT = "%H-%M-%S"
 453         TRACE_FORMAT = "TRACE: {plc_counter:d} {begin}->{seconds}s={duration}s " + \
 454                        "status={status} step={stepname} plc={plcname} force={force}\n"
 455         for stepname, method, force, cross, qualifier in all_step_infos:
 456             plc_counter = 0
 457             for spec, plc_obj in all_plcs:
 458                 plc_counter += 1
 459                 # skip this step if we have specified a plc_explicit
 460                 if qualifier and plc_counter != int(qualifier):
 461                     continue
 462
 463                 plcname = spec['name']
 464                 across_plcs = [ o for (s,o) in all_plcs if o!=plc_obj ]
 465
 466                 # run the step
 467                 beg_time = datetime.now()
 468                 begin = beg_time.strftime(TIME_FORMAT)
 469                 if not spec['failed_step'] or force or self.options.interactive or self.options.keep_going:
 470                     skip_step = False
 471                     if self.options.interactive:
 472                         prompting = True
 473                         while prompting:
 474                             msg="{:d} Run step {} on {} [r](un)/d(ry_run)/p(roceed)/s(kip)/q(uit) ? "\
 475                                 .format(plc_counter, stepname, plcname)
 476                             answer = input(msg).strip().lower() or "r"
 477                             answer = answer[0]
 478                             if answer in ['s','n']:     # skip/no/next
 479                                 print('{} on {} skipped'.format(stepname, plcname))
 480                                 prompting = False
 481                                 skip_step = True
 482                             elif answer in ['q','b']:   # quit/bye
 483                                 print('Exiting')
 484                                 return 'FAILURE'
 485                             elif answer in ['d']:       # dry_run
 486                                 dry_run = self.options.dry_run
 487                                 self.options.dry_run = True
 488                                 plc_obj.options.dry_run = True
 489                                 plc_obj.apiserver.set_dry_run(True)
 490                                 if not cross:
 491                                     step_result=method(plc_obj)
 492                                 else:
 493                                     step_result=method(plc_obj, across_plcs)
 494                                 print('dry_run step ->', step_result)
 495                                 self.options.dry_run = dry_run
 496                                 plc_obj.options.dry_run = dry_run
 497                                 plc_obj.apiserver.set_dry_run(dry_run)
 498                             elif answer in ['p']:
 499                                 # take it as a yes and leave interactive mode
 500                                 prompting = False
 501                                 self.options.interactive = False
 502                             elif answer in ['r','y']:   # run/yes
 503                                 prompting = False
 504                     if skip_step:
 505                         continue
 506                     try:
 507                         force_msg = ""
 508                         if force and spec['failed_step']:
 509                             force_msg=" (forced after {} has failed)".format(spec['failed_step'])
 510                         utils.header("********** {:d} RUNNING step {}{} on plc {}"\
 511                                      .format(plc_counter, stepname, force_msg, plcname))
 512                         if not cross:
 513                             step_result = method(plc_obj)
 514                         else:
 515                             step_result = method(plc_obj, across_plcs)
 516                         if isinstance (step_result, Ignored):
 517                             step_result = step_result.result
 518                             if step_result:
 519                                 msg = "OK"
 520                             else:
 521                                 msg = "KO"
 522                                 # do not overwrite if FAILURE
 523                                 if overall_result == 'SUCCESS':
 524                                     overall_result = 'IGNORED'
 525                             utils.header('********** {} IGNORED ({}) step {} on {}'\
 526                                          .format(plc_counter, msg, stepname, plcname))
 527                             status="{}[I]".format(msg)
 528                         elif step_result:
 529                             utils.header('********** {:d} SUCCESSFUL step {} on {}'\
 530                                          .format(plc_counter, stepname, plcname))
 531                             status = "OK"
 532                         else:
 533                             overall_result = 'FAILURE'
 534                             spec['failed_step'] = stepname
 535                             utils.header('********** {:d} FAILED step {} on {} (discarded from further steps)'\
 536                                          .format(plc_counter, stepname, plcname))
 537                             status = "KO"
 538                     except:
 539                         overall_result = 'FAILURE'
 540                         spec['failed_step'] = stepname
 541                         traceback.print_exc()
 542                         utils.header ('********** {} FAILED (exception) step {} on {} (discarded from further steps)'\
 543                                       .format(plc_counter, stepname, plcname))
 544                         status = "KO"
 545
 546                 # do not run, just display it's skipped
 547                 else:
 548                     why = "has failed {}".format(spec['failed_step'])
 549                     utils.header("********** {} SKIPPED Step {} on {} ({})"\
 550                                  .format(plc_counter, stepname, plcname, why))
 551                     status = "UNDEF"
 552                 if not self.options.dry_run:
 553                     delay = datetime.now()-beg_time
 554                     seconds = int(delay.total_seconds())
 555                     duration = str(delay)
 556                     # always do this on stdout
 557                     print(TRACE_FORMAT.format(**locals()))
 558                     # duplicate on trace_file if provided
 559                     if self.options.trace_file:
 560                         trace.write(TRACE_FORMAT.format(**locals()))
 561                         trace.flush()
 562
 563         if self.options.trace_file and not self.options.dry_run:
 564             trace.close()
 565
 566         # free local substrate
 567         LocalSubstrate.local_substrate.release(self.options)
 568
 569         return overall_result
 570
 571     # wrapper to run, returns a shell-compatible result
 572     # retcod:
 573     # 0: SUCCESS
 574     # 1: FAILURE
 575     # 2: SUCCESS but some ignored steps failed
 576     # 3: OTHER ERROR
 577     def main(self):
 578         try:
 579             success = self.run()
 580             if success == 'SUCCESS':
 581                 return 0
 582             elif success == 'IGNORED':
 583                 return 2
 584             else:
 585                 return 1
 586         except SystemExit:
 587             print('Caught SystemExit')
 588             return 3
 589         except:
 590             traceback.print_exc()
 591             return 3
 592
 593 if __name__ == "__main__":
 594     exit_code = TestMain().main()
 595     print("TestMain exit code", exit_code)
 596     sys.exit(exit_code)