system/TestMain.py

   1 #!/usr/bin/python -u
   2
   3 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
   4 # Copyright (C) 2010 INRIA
   5 #
   6 import sys, os, os.path
   7 from optparse import OptionParser
   8 import traceback
   9 from time import strftime
  10 import readline
  11 import glob
  12
  13 import utils
  14 from TestPlc import TestPlc, Ignored
  15 from TestSite import TestSite
  16 from TestNode import TestNode
  17 from macros import sequences
  18
  19 # add $HOME in PYTHONPATH so we can import LocalSubstrate.py
  20 sys.path.append(os.environ['HOME'])
  21 import LocalSubstrate
  22
  23 class Step:
  24
  25     natives=TestPlc.__dict__
  26
  27     def display (self): return self.name.replace('_','-')
  28     def internal (self): return self.name.replace('-','_')
  29
  30     def __init__ (self, name):
  31         self.name=name
  32         # a native step is implemented as a method on TestPlc
  33         self.native = self.internal() in Step.natives
  34         if self.native:
  35             self.method=Step.natives[self.internal()]
  36         else:
  37             try:
  38                 self.substeps=sequences[self.internal()]
  39             except Exception,e:
  40                 print "macro step %s not found in macros.py (%s) - exiting"%(self.display(),e)
  41                 raise
  42
  43     def print_doc (self,level=0):
  44         tab=32
  45         trail=8
  46         if self.native:
  47             start=level*' '+'* '
  48             # 2 is the len of '* '
  49             width=tab-level-2
  50             format="%%-%ds"%width
  51             line=start+format%self.display()
  52             print line,
  53             try:
  54                 print self.method.__doc__
  55             except:
  56                 print "*** no doc found"
  57         else:
  58             beg_start=level*' '+'>>> '
  59             end_start=level*' '+'<<< '
  60             trailer=trail*'-'
  61             # 4 is the len of '>>> '
  62             width=tab-level-4-trail
  63             format=("%%-%ds"%width)
  64             beg_line=beg_start+format%self.display()+trail*'>'
  65             end_line=end_start+format%self.display()+trail*'<'
  66             print beg_line
  67             for step in self.substeps:
  68                 Step(step).print_doc(level+1)
  69             print end_line
  70
  71     # return a list of (name, method) for all native steps involved
  72     def tuples (self):
  73         if self.native: return [ (self.internal(), self.method,) ]
  74         else:
  75             result=[]
  76             for substep in [ Step(name) for name in self.substeps ] :
  77                 result += substep.tuples()
  78             return result
  79
  80     # convenience for listing macros
  81     # just do a listdir, hoping we're in the right directory...
  82     @staticmethod
  83     def list_macros ():
  84         names= sequences.keys()
  85         names.sort()
  86         return names
  87
  88 class TestMain:
  89
  90     subversion_id = "Now using git -- version tracker broken"
  91
  92     default_config = [ 'default' ]
  93 #    default_rspec_styles = [ 'pl', 'pg' ]
  94     default_rspec_styles = [ 'pg' ]
  95
  96     default_build_url = "git://git.onelab.eu/tests"
  97
  98     def __init__ (self):
  99         self.path=os.path.dirname(sys.argv[0]) or "."
 100         os.chdir(self.path)
 101
 102     def show_env (self,options, message):
 103         if self.options.verbose:
 104             utils.header (message)
 105             utils.show_options("main options",options)
 106
 107     def init_steps(self):
 108         self.steps_message  = 20*'x'+" Defaut steps are\n"+TestPlc.printable_steps(TestPlc.default_steps)
 109         self.steps_message += 20*'x'+" Other useful steps are\n"+TestPlc.printable_steps(TestPlc.other_steps)
 110         self.steps_message += 20*'x'+" Macro steps are\n"+" ".join(Step.list_macros())
 111
 112     def list_steps(self):
 113         if not self.options.verbose:
 114             print self.steps_message,
 115         else:
 116             # steps mentioned on the command line
 117             if self.options.args:
 118                 scopes = [("Argument steps",self.options.args)]
 119             else:
 120                 scopes = [("Default steps",TestPlc.default_steps)]
 121                 if self.options.all_steps:
 122                     scopes.append ( ("Other steps",TestPlc.other_steps) )
 123                     # try to list macro steps as well
 124                     scopes.append ( ("Macro steps", Step.list_macros()) )
 125             for (scope,steps) in scopes:
 126                 print '--------------------',scope
 127                 for step in [step for step in steps if TestPlc.valid_step(step)]:
 128                     try:        (step,qualifier)=step.split('@')
 129                     except:     pass
 130                     stepname=step
 131                     for special in ['force','ignore']:
 132                         stepname = stepname.replace('_'+special,"")
 133                     Step(stepname).print_doc()
 134
 135     def run (self):
 136         self.init_steps()
 137         usage = """usage: %%prog [options] steps
 138 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
 139    no default
 140 config defaults to the last value used, as stored in arg-config,
 141    or %r
 142 ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode},
 143    default is to use IP scanning
 144 steps refer to a method in TestPlc or to a step_* module
 145 ===
 146 """%(TestMain.default_config)
 147         usage += self.steps_message
 148         parser=OptionParser(usage=usage,version=self.subversion_id)
 149         parser.add_option("-u","--url",action="store", dest="arch_rpms_url",
 150                           help="URL of the arch-dependent RPMS area - for locating what to test")
 151         parser.add_option("-b","--build",action="store", dest="build_url",
 152                           help="ignored, for legacy only")
 153         parser.add_option("-c","--config",action="append", dest="config", default=[],
 154                           help="Config module - can be set multiple times, or use quotes")
 155         parser.add_option("-p","--personality",action="store", dest="personality",
 156                           help="personality - as in vbuild-nightly")
 157         parser.add_option("-d","--pldistro",action="store", dest="pldistro",
 158                           help="pldistro - as in vbuild-nightly")
 159         parser.add_option("-f","--fcdistro",action="store", dest="fcdistro",
 160                           help="fcdistro - as in vbuild-nightly")
 161         parser.add_option("-e","--exclude",action="append", dest="exclude", default=[],
 162                           help="steps to exclude - can be set multiple times, or use quotes")
 163         parser.add_option("-i","--ignore",action="append", dest="ignore", default=[],
 164                           help="steps to run but ignore - can be set multiple times, or use quotes")
 165         parser.add_option("-a","--all",action="store_true",dest="all_steps", default=False,
 166                           help="Run all default steps")
 167         parser.add_option("-l","--list",action="store_true",dest="list_steps", default=False,
 168                           help="List known steps")
 169         parser.add_option("-V","--vserver",action="append", dest="ips_bplc", default=[],
 170                           help="Specify the set of hostnames for the boxes that host the plcs")
 171         parser.add_option("-P","--plcs",action="append", dest="ips_vplc", default=[],
 172                           help="Specify the set of hostname/IP's to use for vplcs")
 173         parser.add_option("-Q","--qemus",action="append", dest="ips_bnode", default=[],
 174                           help="Specify the set of hostnames for the boxes that host the nodes")
 175         parser.add_option("-N","--nodes",action="append", dest="ips_vnode", default=[],
 176                           help="Specify the set of hostname/IP's to use for vnodes")
 177         parser.add_option("-s","--size",action="store",type="int",dest="size",default=1,
 178                           help="sets test size in # of plcs - default is 1")
 179         parser.add_option("-q","--qualifier",action="store",type="int",dest="qualifier",default=None,
 180                           help="run steps only on plc numbered <qualifier>, starting at 1")
 181         parser.add_option("-y","--rspec-style",action="append",dest="rspec_styles",default=[],
 182                           help="pl is for planetlab rspecs, pg is for protogeni")
 183         parser.add_option("-k","--keep-going",action="store",dest="keep_going",default=False,
 184                           help="proceeds even if some steps are failing")
 185         parser.add_option("-D","--dbname",action="store",dest="dbname",default=None,
 186                            help="Used by plc_db_dump and plc_db_restore")
 187         parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
 188                           help="Run in verbose mode")
 189         parser.add_option("-I","--interactive",action="store_true",dest="interactive",default=False,
 190                           help="prompts before each step")
 191         parser.add_option("-n","--dry-run", action="store_true", dest="dry_run", default=False,
 192                           help="Show environment and exits")
 193 # dropped when added Completer.py
 194 #        parser.add_option("-r","--restart-nm", action="store_true", dest="forcenm", default=False,
 195 #                          help="Force the NM to restart in ssh_slices step")
 196         parser.add_option("-t","--trace", action="store", dest="trace_file", default=None,
 197                           help="Trace file location")
 198         (self.options, self.args) = parser.parse_args()
 199
 200         # allow things like "run -c 'c1 c2' -c c3"
 201         def flatten (x):
 202             result = []
 203             for el in x:
 204                 if hasattr(el, "__iter__") and not isinstance(el, basestring):
 205                     result.extend(flatten(el))
 206                 else:
 207                     result.append(el)
 208             return result
 209         # flatten relevant options
 210         for optname in ['config','exclude','ignore','ips_bplc','ips_vplc','ips_bnode','ips_vnode']:
 211             setattr(self.options,optname, flatten ( [ arg.split() for arg in getattr(self.options,optname) ] ))
 212
 213         if not self.options.rspec_styles:
 214             self.options.rspec_styles=TestMain.default_rspec_styles
 215
 216         # handle defaults and option persistence
 217         for (recname,filename,default,need_reverse) in (
 218             ('build_url','arg-build-url',TestMain.default_build_url,None) ,
 219             ('ips_bplc','arg-ips-bplc',[],True),
 220             ('ips_vplc','arg-ips-vplc',[],True) ,
 221             ('ips_bnode','arg-ips-bnode',[],True),
 222             ('ips_vnode','arg-ips-vnode',[],True) ,
 223             ('config','arg-config',TestMain.default_config,False) ,
 224             ('arch_rpms_url','arg-arch-rpms-url',"",None) ,
 225             ('personality','arg-personality',"linux64",None),
 226             ('pldistro','arg-pldistro',"onelab",None),
 227             ('fcdistro','arg-fcdistro','f14',None),
 228             ) :
 229 #            print 'handling',recname
 230             path=filename
 231             is_list = isinstance(default,list)
 232             is_bool = isinstance(default,bool)
 233             if not getattr(self.options,recname):
 234                 try:
 235                     parsed=file(path).readlines()
 236                     if is_list:         # lists
 237                         parsed=[x.strip() for x in parsed]
 238                     else:               # strings and booleans
 239                         if len(parsed) != 1:
 240                             print "%s - error when parsing %s"%(sys.argv[1],path)
 241                             sys.exit(1)
 242                         parsed=parsed[0].strip()
 243                         if is_bool:
 244                             parsed = parsed.lower()=='true'
 245                     setattr(self.options,recname,parsed)
 246                 except:
 247                     if default != "":
 248                         setattr(self.options,recname,default)
 249                     else:
 250                         print "Cannot determine",recname
 251                         print "Run %s --help for help"%sys.argv[0]
 252                         sys.exit(1)
 253
 254             # save for next run
 255             fsave=open(path,"w")
 256             if is_list:                 # lists
 257                 for value in getattr(self.options,recname):
 258                     fsave.write(value + "\n")
 259             else:                       # strings and booleans - just call str()
 260                 fsave.write(str(getattr(self.options,recname)) + "\n")
 261             fsave.close()
 262 #            utils.header('Saved %s into %s'%(recname,filename))
 263
 264             # lists need be reversed
 265             # I suspect this is useful for the various pools but for config, it's painful
 266             if isinstance(getattr(self.options,recname),list) and need_reverse:
 267                 getattr(self.options,recname).reverse()
 268
 269             if self.options.verbose:
 270                 utils.header('* Using %s = %s'%(recname,getattr(self.options,recname)))
 271
 272         # hack : if sfa is not among the published rpms, skip these tests
 273         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
 274
 275         # no step specified
 276         self.options.args = self.args
 277         if len(self.args) == 0:
 278             self.options.steps=TestPlc.default_steps
 279         else:
 280             self.options.steps = self.args
 281
 282         if self.options.list_steps:
 283             self.init_steps()
 284             self.list_steps()
 285             return 'SUCCESS'
 286
 287         # steps
 288         if not self.options.steps:
 289             #default (all) steps
 290             #self.options.steps=['dump','clean','install','populate']
 291             self.options.steps=TestPlc.default_steps
 292
 293         # rewrite '-' into '_' in step names
 294         self.options.steps = [ step.replace('-','_') for step in self.options.steps ]
 295         self.options.exclude = [ step.replace('-','_') for step in self.options.exclude ]
 296         self.options.ignore = [ step.replace('-','_') for step in self.options.ignore ]
 297
 298         # technicality, decorate known steps to produce the '_ignore' version
 299         TestPlc.create_ignore_steps()
 300
 301         # exclude
 302         selected=[]
 303         for step in self.options.steps:
 304             keep=True
 305             for exclude in self.options.exclude:
 306                 if utils.match(step,exclude):
 307                     keep=False
 308                     break
 309             if keep: selected.append(step)
 310
 311         # ignore
 312         selected = [ step if step not in self.options.ignore else step+"_ignore"
 313                      for step in selected ]
 314
 315         self.options.steps=selected
 316
 317         # this is useful when propagating on host boxes, to avoid conflicts
 318         self.options.buildname = os.path.basename (os.path.abspath (self.path))
 319
 320         if self.options.verbose:
 321             self.show_env(self.options,"Verbose")
 322
 323         # load configs
 324         all_plc_specs = []
 325         for config in self.options.config:
 326             modulename='config_'+config
 327             try:
 328                 m = __import__(modulename)
 329                 all_plc_specs = m.config(all_plc_specs,self.options)
 330             except :
 331                 traceback.print_exc()
 332                 print 'Cannot load config %s -- ignored'%modulename
 333                 raise
 334
 335         # provision on local substrate
 336         all_plc_specs = LocalSubstrate.local_substrate.provision(all_plc_specs,self.options)
 337
 338         # remember substrate IP address(es) for next run
 339         ips_bplc_file=open('arg-ips-bplc','w')
 340         for plc_spec in all_plc_specs:
 341             ips_bplc_file.write("%s\n"%plc_spec['host_box'])
 342         ips_bplc_file.close()
 343         ips_vplc_file=open('arg-ips-vplc','w')
 344         for plc_spec in all_plc_specs:
 345             ips_vplc_file.write("%s\n"%plc_spec['settings']['PLC_API_HOST'])
 346         ips_vplc_file.close()
 347         # ditto for nodes
 348         ips_bnode_file=open('arg-ips-bnode','w')
 349         for plc_spec in all_plc_specs:
 350             for site_spec in plc_spec['sites']:
 351                 for node_spec in site_spec['nodes']:
 352                     ips_bnode_file.write("%s\n"%node_spec['host_box'])
 353         ips_bnode_file.close()
 354         ips_vnode_file=open('arg-ips-vnode','w')
 355         for plc_spec in all_plc_specs:
 356             for site_spec in plc_spec['sites']:
 357                 for node_spec in site_spec['nodes']:
 358                     # back to normal (unqualified) form
 359                     stripped=node_spec['node_fields']['hostname'].split('.')[0]
 360                     ips_vnode_file.write("%s\n"%stripped)
 361         ips_vnode_file.close()
 362
 363         # build a TestPlc object from the result, passing options
 364         for spec in all_plc_specs:
 365             spec['failed_step'] = False
 366         all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs]
 367
 368         # pass options to utils as well
 369         utils.init_options(self.options)
 370
 371         overall_result = 'SUCCESS'
 372         all_step_infos=[]
 373         for step in self.options.steps:
 374             if not TestPlc.valid_step(step):
 375                 continue
 376             # some steps need to be done regardless of the previous ones: we force them
 377             force=False
 378             if step.endswith("_force"):
 379                 step=step.replace("_force","")
 380                 force=True
 381             # allow for steps to specify an index like in
 382             # run checkslice@2
 383             try:        (step,qualifier)=step.split('@')
 384             except:     qualifier=self.options.qualifier
 385
 386             try:
 387                 stepobj = Step (step)
 388                 for (substep, method) in stepobj.tuples():
 389                     # a cross step will run a method on TestPlc that has a signature like
 390                     # def cross_foo (self, all_test_plcs)
 391                     cross=False
 392                     if substep.find("cross_") == 0:
 393                         cross=True
 394                     all_step_infos.append ( (substep, method, force, cross, qualifier, ) )
 395             except :
 396                 utils.header("********** FAILED step %s (NOT FOUND) -- won't be run"%step)
 397                 traceback.print_exc()
 398                 overall_result = 'FAILURE'
 399
 400         if self.options.dry_run:
 401             self.show_env(self.options,"Dry run")
 402
 403         # init & open trace file if provided
 404         if self.options.trace_file and not self.options.dry_run:
 405             # create dir if needed
 406             trace_dir=os.path.dirname(self.options.trace_file)
 407             if trace_dir and not os.path.isdir(trace_dir):
 408                 os.makedirs(trace_dir)
 409             trace=open(self.options.trace_file,"w")
 410
 411         # do all steps on all plcs
 412         TIME_FORMAT="%H-%M-%S"
 413         TRACE_FORMAT="TRACE: %(plc_counter)d %(beg)s->%(end)s status=%(status)s step=%(stepname)s plc=%(plcname)s force=%(force)s\n"
 414         for (stepname,method,force,cross,qualifier) in all_step_infos:
 415             plc_counter=0
 416             for (spec,plc_obj) in all_plcs:
 417                 plc_counter+=1
 418                 # skip this step if we have specified a plc_explicit
 419                 if qualifier and plc_counter!=int(qualifier): continue
 420
 421                 plcname=spec['name']
 422                 across_plcs = [ o for (s,o) in all_plcs if o!=plc_obj ]
 423
 424                 # run the step
 425                 beg=strftime(TIME_FORMAT)
 426                 if not spec['failed_step'] or force or self.options.interactive or self.options.keep_going:
 427                     skip_step=False
 428                     if self.options.interactive:
 429                         prompting=True
 430                         while prompting:
 431                             msg="%d Run step %s on %s [r](un)/d(ry_run)/p(roceed)/s(kip)/q(uit) ? "%(plc_counter,stepname,plcname)
 432                             answer=raw_input(msg).strip().lower() or "r"
 433                             answer=answer[0]
 434                             if answer in ['s','n']:     # skip/no/next
 435                                 print '%s on %s skipped'%(stepname,plcname)
 436                                 prompting=False
 437                                 skip_step=True
 438                             elif answer in ['q','b']:   # quit/bye
 439                                 print 'Exiting'
 440                                 return 'FAILURE'
 441                             elif answer in ['d']:       # dry_run
 442                                 dry_run=self.options.dry_run
 443                                 self.options.dry_run=True
 444                                 plc_obj.options.dry_run=True
 445                                 plc_obj.apiserver.set_dry_run(True)
 446                                 if not cross:   step_result=method(plc_obj)
 447                                 else:           step_result=method(plc_obj,across_plcs)
 448                                 print 'dry_run step ->',step_result
 449                                 self.options.dry_run=dry_run
 450                                 plc_obj.options.dry_run=dry_run
 451                                 plc_obj.apiserver.set_dry_run(dry_run)
 452                             elif answer in ['p']:
 453                                 # take it as a yes and leave interactive mode
 454                                 prompting=False
 455                                 self.options.interactive=False
 456                             elif answer in ['r','y']:   # run/yes
 457                                 prompting=False
 458                     if skip_step:
 459                         continue
 460                     try:
 461                         force_msg=""
 462                         if force and spec['failed_step']: force_msg=" (forced after %s has failed)"%spec['failed_step']
 463                         utils.header("********** %d RUNNING step %s%s on plc %s"%(plc_counter,stepname,force_msg,plcname))
 464                         if not cross:   step_result = method(plc_obj)
 465                         else:           step_result = method(plc_obj,across_plcs)
 466                         if isinstance (step_result,Ignored):
 467                             step_result=step_result.result
 468                             if step_result:
 469                                 msg="OK"
 470                             else:
 471                                 msg="KO"
 472                                 # do not overwrite if FAILURE
 473                                 if overall_result=='SUCCESS':
 474                                     overall_result='IGNORED'
 475                             utils.header('********** %d IGNORED (%s) step %s on %s'%(plc_counter,msg,stepname,plcname))
 476                             status="%s[I]"%msg
 477                         elif step_result:
 478                             utils.header('********** %d SUCCESSFUL step %s on %s'%(plc_counter,stepname,plcname))
 479                             status="OK"
 480                         else:
 481                             overall_result = 'FAILURE'
 482                             spec['failed_step'] = stepname
 483                             utils.header('********** %d FAILED Step %s on %s (discarded from further steps)'\
 484                                              %(plc_counter,stepname,plcname))
 485                             status="KO"
 486                     except:
 487                         overall_result='FAILURE'
 488                         spec['failed_step'] = stepname
 489                         traceback.print_exc()
 490                         utils.header ('********** %d FAILED (exception) Step %s on %s (discarded from further steps)'\
 491                                           %(plc_counter,stepname,plcname))
 492                         status="KO"
 493
 494                 # do not run, just display it's skipped
 495                 else:
 496                     why="has failed %s"%spec['failed_step']
 497                     utils.header("********** %d SKIPPED Step %s on %s (%s)"%(plc_counter,stepname,plcname,why))
 498                     status="UNDEF"
 499                 if not self.options.dry_run:
 500                     end=strftime(TIME_FORMAT)
 501                     # always do this on stdout
 502                     print TRACE_FORMAT%locals()
 503                     # duplicate on trace_file if provided
 504                     if self.options.trace_file:
 505                         trace.write(TRACE_FORMAT%locals())
 506                         trace.flush()
 507
 508         if self.options.trace_file and not self.options.dry_run:
 509             trace.close()
 510
 511         # free local substrate
 512         LocalSubstrate.local_substrate.release(self.options)
 513
 514         return overall_result
 515
 516     # wrapper to run, returns a shell-compatible result
 517     # retcod:
 518     # 0: SUCCESS
 519     # 1: FAILURE
 520     # 2: SUCCESS but some ignored steps failed
 521     # 3: OTHER ERROR
 522     def main(self):
 523         try:
 524             success=self.run()
 525             if success == 'SUCCESS':    return 0
 526             elif success == 'IGNORED':  return 2
 527             else:                       return 1
 528         except SystemExit:
 529             print 'Caught SystemExit'
 530             return 3
 531         except:
 532             traceback.print_exc()
 533             return 3
 534
 535 if __name__ == "__main__":
 536     exit_code = TestMain().main()
 537     print "TestMain exit code",exit_code
 538     sys.exit(exit_code)