system/TestMain.py

   1 #!/usr/bin/python -u
   2
   3 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
   4 # Copyright (C) 2010 INRIA
   5 #
   6 import sys, os, os.path
   7 from optparse import OptionParser
   8 import traceback
   9 import readline
  10 import glob
  11 from datetime import datetime
  12
  13 import utils
  14 from TestPlc import TestPlc, Ignored
  15 from TestSite import TestSite
  16 from TestNode import TestNode
  17 from macros import sequences
  18
  19 # add $HOME in PYTHONPATH so we can import LocalSubstrate.py
  20 sys.path.append(os.environ['HOME'])
  21 import LocalSubstrate
  22
  23 class Step:
  24
  25     natives=TestPlc.__dict__
  26
  27     def display (self): return self.name.replace('_','-')
  28     def internal (self): return self.name.replace('-','_')
  29
  30     def __init__ (self, name):
  31         self.name=name
  32         # a native step is implemented as a method on TestPlc
  33         self.native = self.internal() in Step.natives
  34         if self.native:
  35             self.method=Step.natives[self.internal()]
  36         else:
  37             try:
  38                 self.substeps=sequences[self.internal()]
  39             except Exception,e:
  40                 print "macro step %s not found in macros.py (%s) - exiting"%(self.display(),e)
  41                 raise
  42
  43     def print_doc (self,level=0):
  44         tab=32
  45         trail=8
  46         if self.native:
  47             start=level*' '+'* '
  48             # 2 is the len of '* '
  49             width=tab-level-2
  50             format="%%-%ds"%width
  51             line=start+format%self.display()
  52             print line,
  53             try:
  54                 print self.method.__doc__
  55             except:
  56                 print "*** no doc found"
  57         else:
  58             beg_start=level*' '+'>>> '
  59             end_start=level*' '+'<<< '
  60             trailer=trail*'-'
  61             # 4 is the len of '>>> '
  62             width=tab-level-4-trail
  63             format=("%%-%ds"%width)
  64             beg_line=beg_start+format%self.display()+trail*'>'
  65             end_line=end_start+format%self.display()+trail*'<'
  66             print beg_line
  67             for step in self.substeps:
  68                 Step(step).print_doc(level+1)
  69             print end_line
  70
  71     # return a list of (name, method) for all native steps involved
  72     def tuples (self):
  73         if self.native: return [ (self.internal(), self.method,) ]
  74         else:
  75             result=[]
  76             for substep in [ Step(name) for name in self.substeps ] :
  77                 result += substep.tuples()
  78             return result
  79
  80     # convenience for listing macros
  81     # just do a listdir, hoping we're in the right directory...
  82     @staticmethod
  83     def list_macros ():
  84         names= sequences.keys()
  85         names.sort()
  86         return names
  87
  88 class TestMain:
  89
  90     subversion_id = "Now using git -- version tracker broken"
  91
  92     default_config = [ 'default' ]
  93 #    default_rspec_styles = [ 'pl', 'pg' ]
  94     default_rspec_styles = [ 'pg' ]
  95
  96     default_build_url = "git://git.onelab.eu/tests"
  97
  98     def __init__ (self):
  99         self.path=os.path.dirname(sys.argv[0]) or "."
 100         os.chdir(self.path)
 101
 102     def show_env (self,options, message):
 103         if self.options.verbose:
 104             utils.header (message)
 105             utils.show_options("main options",options)
 106
 107     def init_steps(self):
 108         self.steps_message  = 20*'x'+" Defaut steps are\n"+TestPlc.printable_steps(TestPlc.default_steps)
 109         self.steps_message += 20*'x'+" Other useful steps are\n"+TestPlc.printable_steps(TestPlc.other_steps)
 110         self.steps_message += 20*'x'+" Macro steps are\n"+" ".join(Step.list_macros())
 111
 112     def list_steps(self):
 113         if not self.options.verbose:
 114             print self.steps_message,
 115         else:
 116             # steps mentioned on the command line
 117             if self.options.args:
 118                 scopes = [("Argument steps",self.options.args)]
 119             else:
 120                 scopes = [("Default steps",TestPlc.default_steps)]
 121                 if self.options.all_steps:
 122                     scopes.append ( ("Other steps",TestPlc.other_steps) )
 123                     # try to list macro steps as well
 124                     scopes.append ( ("Macro steps", Step.list_macros()) )
 125             for (scope,steps) in scopes:
 126                 print '--------------------',scope
 127                 for step in [step for step in steps if TestPlc.valid_step(step)]:
 128                     try:        (step,qualifier)=step.split('@')
 129                     except:     pass
 130                     stepname=step
 131                     for special in ['force','ignore']:
 132                         stepname = stepname.replace('_'+special,"")
 133                     Step(stepname).print_doc()
 134
 135     def run (self):
 136         self.init_steps()
 137         usage = """usage: %%prog [options] steps
 138 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
 139    no default
 140 config defaults to the last value used, as stored in arg-config,
 141    or %r
 142 ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode},
 143    default is to use IP scanning
 144 steps refer to a method in TestPlc or to a step_* module
 145 ===
 146 """%(TestMain.default_config)
 147         usage += self.steps_message
 148         parser=OptionParser(usage=usage,version=self.subversion_id)
 149         parser.add_option("-u","--url",action="store", dest="arch_rpms_url",
 150                           help="URL of the arch-dependent RPMS area - for locating what to test")
 151         parser.add_option("-b","--build",action="store", dest="build_url",
 152                           help="ignored, for legacy only")
 153         parser.add_option("-c","--config",action="append", dest="config", default=[],
 154                           help="Config module - can be set multiple times, or use quotes")
 155         parser.add_option("-p","--personality",action="store", dest="personality",
 156                           help="personality - as in vbuild-nightly")
 157         parser.add_option("-d","--pldistro",action="store", dest="pldistro",
 158                           help="pldistro - as in vbuild-nightly")
 159         parser.add_option("-f","--fcdistro",action="store", dest="fcdistro",
 160                           help="fcdistro - as in vbuild-nightly")
 161         parser.add_option("-e","--exclude",action="append", dest="exclude", default=[],
 162                           help="steps to exclude - can be set multiple times, or use quotes")
 163         parser.add_option("-i","--ignore",action="append", dest="ignore", default=[],
 164                           help="steps to run but ignore - can be set multiple times, or use quotes")
 165         parser.add_option("-a","--all",action="store_true",dest="all_steps", default=False,
 166                           help="Run all default steps")
 167         parser.add_option("-l","--list",action="store_true",dest="list_steps", default=False,
 168                           help="List known steps")
 169         parser.add_option("-V","--vserver",action="append", dest="ips_bplc", default=[],
 170                           help="Specify the set of hostnames for the boxes that host the plcs")
 171         parser.add_option("-P","--plcs",action="append", dest="ips_vplc", default=[],
 172                           help="Specify the set of hostname/IP's to use for vplcs")
 173         parser.add_option("-Q","--qemus",action="append", dest="ips_bnode", default=[],
 174                           help="Specify the set of hostnames for the boxes that host the nodes")
 175         parser.add_option("-N","--nodes",action="append", dest="ips_vnode", default=[],
 176                           help="Specify the set of hostname/IP's to use for vnodes")
 177         parser.add_option("-s","--size",action="store",type="int",dest="size",default=1,
 178                           help="sets test size in # of plcs - default is 1")
 179         parser.add_option("-q","--qualifier",action="store",type="int",dest="qualifier",default=None,
 180                           help="run steps only on plc numbered <qualifier>, starting at 1")
 181         parser.add_option("-y","--rspec-style",action="append",dest="rspec_styles",default=[],
 182                           help="pl is for planetlab rspecs, pg is for protogeni")
 183         parser.add_option("-k","--keep-going",action="store",dest="keep_going",default=False,
 184                           help="proceeds even if some steps are failing")
 185         parser.add_option("-D","--dbname",action="store",dest="dbname",default=None,
 186                            help="Used by plc_db_dump and plc_db_restore")
 187         parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
 188                           help="Run in verbose mode")
 189         parser.add_option("-I","--interactive",action="store_true",dest="interactive",default=False,
 190                           help="prompts before each step")
 191         parser.add_option("-n","--dry-run", action="store_true", dest="dry_run", default=False,
 192                           help="Show environment and exits")
 193         parser.add_option("-t","--trace", action="store", dest="trace_file", default=None,
 194                           help="Trace file location")
 195         (self.options, self.args) = parser.parse_args()
 196
 197         # allow things like "run -c 'c1 c2' -c c3"
 198         def flatten (x):
 199             result = []
 200             for el in x:
 201                 if hasattr(el, "__iter__") and not isinstance(el, basestring):
 202                     result.extend(flatten(el))
 203                 else:
 204                     result.append(el)
 205             return result
 206         # flatten relevant options
 207         for optname in ['config','exclude','ignore','ips_bplc','ips_vplc','ips_bnode','ips_vnode']:
 208             setattr(self.options,optname, flatten ( [ arg.split() for arg in getattr(self.options,optname) ] ))
 209
 210         if not self.options.rspec_styles:
 211             self.options.rspec_styles=TestMain.default_rspec_styles
 212
 213         # handle defaults and option persistence
 214         for (recname,filename,default,need_reverse) in (
 215             ('build_url','arg-build-url',TestMain.default_build_url,None) ,
 216             ('ips_bplc','arg-ips-bplc',[],True),
 217             ('ips_vplc','arg-ips-vplc',[],True) ,
 218             ('ips_bnode','arg-ips-bnode',[],True),
 219             ('ips_vnode','arg-ips-vnode',[],True) ,
 220             ('config','arg-config',TestMain.default_config,False) ,
 221             ('arch_rpms_url','arg-arch-rpms-url',"",None) ,
 222             ('personality','arg-personality',"linux64",None),
 223             ('pldistro','arg-pldistro',"onelab",None),
 224             ('fcdistro','arg-fcdistro','f14',None),
 225             ) :
 226 #            print 'handling',recname
 227             path=filename
 228             is_list = isinstance(default,list)
 229             is_bool = isinstance(default,bool)
 230             if not getattr(self.options,recname):
 231                 try:
 232                     parsed=file(path).readlines()
 233                     if is_list:         # lists
 234                         parsed=[x.strip() for x in parsed]
 235                     else:               # strings and booleans
 236                         if len(parsed) != 1:
 237                             print "%s - error when parsing %s"%(sys.argv[1],path)
 238                             sys.exit(1)
 239                         parsed=parsed[0].strip()
 240                         if is_bool:
 241                             parsed = parsed.lower()=='true'
 242                     setattr(self.options,recname,parsed)
 243                 except:
 244                     if default != "":
 245                         setattr(self.options,recname,default)
 246                     else:
 247                         print "Cannot determine",recname
 248                         print "Run %s --help for help"%sys.argv[0]
 249                         sys.exit(1)
 250
 251             # save for next run
 252             fsave=open(path,"w")
 253             if is_list:                 # lists
 254                 for value in getattr(self.options,recname):
 255                     fsave.write(value + "\n")
 256             else:                       # strings and booleans - just call str()
 257                 fsave.write(str(getattr(self.options,recname)) + "\n")
 258             fsave.close()
 259 #            utils.header('Saved %s into %s'%(recname,filename))
 260
 261             # lists need be reversed
 262             # I suspect this is useful for the various pools but for config, it's painful
 263             if isinstance(getattr(self.options,recname),list) and need_reverse:
 264                 getattr(self.options,recname).reverse()
 265
 266             if self.options.verbose:
 267                 utils.header('* Using %s = %s'%(recname,getattr(self.options,recname)))
 268
 269         # hack : if sfa is not among the published rpms, skip these tests
 270         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
 271
 272         # no step specified
 273         self.options.args = self.args
 274         if len(self.args) == 0:
 275             self.options.steps=TestPlc.default_steps
 276         else:
 277             self.options.steps = self.args
 278
 279         if self.options.list_steps:
 280             self.init_steps()
 281             self.list_steps()
 282             return 'SUCCESS'
 283
 284         # steps
 285         if not self.options.steps:
 286             #default (all) steps
 287             #self.options.steps=['dump','clean','install','populate']
 288             self.options.steps=TestPlc.default_steps
 289
 290         # rewrite '-' into '_' in step names
 291         self.options.steps = [ step.replace('-','_') for step in self.options.steps ]
 292         self.options.exclude = [ step.replace('-','_') for step in self.options.exclude ]
 293         self.options.ignore = [ step.replace('-','_') for step in self.options.ignore ]
 294
 295         # technicality, decorate known steps to produce the '_ignore' version
 296         TestPlc.create_ignore_steps()
 297
 298         # exclude
 299         selected=[]
 300         for step in self.options.steps:
 301             keep=True
 302             for exclude in self.options.exclude:
 303                 if utils.match(step,exclude):
 304                     keep=False
 305                     break
 306             if keep: selected.append(step)
 307
 308         # ignore
 309         selected = [ step if step not in self.options.ignore else step+"_ignore"
 310                      for step in selected ]
 311
 312         self.options.steps=selected
 313
 314         # this is useful when propagating on host boxes, to avoid conflicts
 315         self.options.buildname = os.path.basename (os.path.abspath (self.path))
 316
 317         if self.options.verbose:
 318             self.show_env(self.options,"Verbose")
 319
 320         # load configs
 321         all_plc_specs = []
 322         for config in self.options.config:
 323             modulename='config_'+config
 324             try:
 325                 m = __import__(modulename)
 326                 all_plc_specs = m.config(all_plc_specs,self.options)
 327             except :
 328                 traceback.print_exc()
 329                 print 'Cannot load config %s -- ignored'%modulename
 330                 raise
 331
 332         # provision on local substrate
 333         all_plc_specs = LocalSubstrate.local_substrate.provision(all_plc_specs,self.options)
 334
 335         # remember substrate IP address(es) for next run
 336         ips_bplc_file=open('arg-ips-bplc','w')
 337         for plc_spec in all_plc_specs:
 338             ips_bplc_file.write("%s\n"%plc_spec['host_box'])
 339         ips_bplc_file.close()
 340         ips_vplc_file=open('arg-ips-vplc','w')
 341         for plc_spec in all_plc_specs:
 342             ips_vplc_file.write("%s\n"%plc_spec['settings']['PLC_API_HOST'])
 343         ips_vplc_file.close()
 344         # ditto for nodes
 345         ips_bnode_file=open('arg-ips-bnode','w')
 346         for plc_spec in all_plc_specs:
 347             for site_spec in plc_spec['sites']:
 348                 for node_spec in site_spec['nodes']:
 349                     ips_bnode_file.write("%s\n"%node_spec['host_box'])
 350         ips_bnode_file.close()
 351         ips_vnode_file=open('arg-ips-vnode','w')
 352         for plc_spec in all_plc_specs:
 353             for site_spec in plc_spec['sites']:
 354                 for node_spec in site_spec['nodes']:
 355                     # back to normal (unqualified) form
 356                     stripped=node_spec['node_fields']['hostname'].split('.')[0]
 357                     ips_vnode_file.write("%s\n"%stripped)
 358         ips_vnode_file.close()
 359
 360         # build a TestPlc object from the result, passing options
 361         for spec in all_plc_specs:
 362             spec['failed_step'] = False
 363         all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs]
 364
 365         # pass options to utils as well
 366         utils.init_options(self.options)
 367
 368         overall_result = 'SUCCESS'
 369         all_step_infos=[]
 370         for step in self.options.steps:
 371             if not TestPlc.valid_step(step):
 372                 continue
 373             # some steps need to be done regardless of the previous ones: we force them
 374             force=False
 375             if step.endswith("_force"):
 376                 step=step.replace("_force","")
 377                 force=True
 378             # allow for steps to specify an index like in
 379             # run checkslice@2
 380             try:        (step,qualifier)=step.split('@')
 381             except:     qualifier=self.options.qualifier
 382
 383             try:
 384                 stepobj = Step (step)
 385                 for (substep, method) in stepobj.tuples():
 386                     # a cross step will run a method on TestPlc that has a signature like
 387                     # def cross_foo (self, all_test_plcs)
 388                     cross=False
 389                     if substep.find("cross_") == 0:
 390                         cross=True
 391                     all_step_infos.append ( (substep, method, force, cross, qualifier, ) )
 392             except :
 393                 utils.header("********** FAILED step %s (NOT FOUND) -- won't be run"%step)
 394                 traceback.print_exc()
 395                 overall_result = 'FAILURE'
 396
 397         if self.options.dry_run:
 398             self.show_env(self.options,"Dry run")
 399
 400         # init & open trace file if provided
 401         if self.options.trace_file and not self.options.dry_run:
 402             # create dir if needed
 403             trace_dir=os.path.dirname(self.options.trace_file)
 404             if trace_dir and not os.path.isdir(trace_dir):
 405                 os.makedirs(trace_dir)
 406             trace=open(self.options.trace_file,"w")
 407
 408         # do all steps on all plcs
 409         TIME_FORMAT="%H-%M-%S"
 410         TRACE_FORMAT="TRACE: %(plc_counter)d %(begin)s->%(seconds)ss=%(duration)s status=%(status)s step=%(stepname)s plc=%(plcname)s force=%(force)s\n"
 411         for (stepname,method,force,cross,qualifier) in all_step_infos:
 412             plc_counter=0
 413             for (spec,plc_obj) in all_plcs:
 414                 plc_counter+=1
 415                 # skip this step if we have specified a plc_explicit
 416                 if qualifier and plc_counter!=int(qualifier): continue
 417
 418                 plcname=spec['name']
 419                 across_plcs = [ o for (s,o) in all_plcs if o!=plc_obj ]
 420
 421                 # run the step
 422                 beg_time = datetime.now()
 423                 begin = beg_time.strftime(TIME_FORMAT)
 424                 if not spec['failed_step'] or force or self.options.interactive or self.options.keep_going:
 425                     skip_step=False
 426                     if self.options.interactive:
 427                         prompting=True
 428                         while prompting:
 429                             msg="%d Run step %s on %s [r](un)/d(ry_run)/p(roceed)/s(kip)/q(uit) ? "%(plc_counter,stepname,plcname)
 430                             answer=raw_input(msg).strip().lower() or "r"
 431                             answer=answer[0]
 432                             if answer in ['s','n']:     # skip/no/next
 433                                 print '%s on %s skipped'%(stepname,plcname)
 434                                 prompting=False
 435                                 skip_step=True
 436                             elif answer in ['q','b']:   # quit/bye
 437                                 print 'Exiting'
 438                                 return 'FAILURE'
 439                             elif answer in ['d']:       # dry_run
 440                                 dry_run=self.options.dry_run
 441                                 self.options.dry_run=True
 442                                 plc_obj.options.dry_run=True
 443                                 plc_obj.apiserver.set_dry_run(True)
 444                                 if not cross:   step_result=method(plc_obj)
 445                                 else:           step_result=method(plc_obj,across_plcs)
 446                                 print 'dry_run step ->',step_result
 447                                 self.options.dry_run=dry_run
 448                                 plc_obj.options.dry_run=dry_run
 449                                 plc_obj.apiserver.set_dry_run(dry_run)
 450                             elif answer in ['p']:
 451                                 # take it as a yes and leave interactive mode
 452                                 prompting=False
 453                                 self.options.interactive=False
 454                             elif answer in ['r','y']:   # run/yes
 455                                 prompting=False
 456                     if skip_step:
 457                         continue
 458                     try:
 459                         force_msg=""
 460                         if force and spec['failed_step']: force_msg=" (forced after %s has failed)"%spec['failed_step']
 461                         utils.header("********** %d RUNNING step %s%s on plc %s"%(plc_counter,stepname,force_msg,plcname))
 462                         if not cross:   step_result = method(plc_obj)
 463                         else:           step_result = method(plc_obj,across_plcs)
 464                         if isinstance (step_result,Ignored):
 465                             step_result=step_result.result
 466                             if step_result:
 467                                 msg="OK"
 468                             else:
 469                                 msg="KO"
 470                                 # do not overwrite if FAILURE
 471                                 if overall_result=='SUCCESS':
 472                                     overall_result='IGNORED'
 473                             utils.header('********** %d IGNORED (%s) step %s on %s'%(plc_counter,msg,stepname,plcname))
 474                             status="%s[I]"%msg
 475                         elif step_result:
 476                             utils.header('********** %d SUCCESSFUL step %s on %s'%(plc_counter,stepname,plcname))
 477                             status="OK"
 478                         else:
 479                             overall_result = 'FAILURE'
 480                             spec['failed_step'] = stepname
 481                             utils.header('********** %d FAILED step %s on %s (discarded from further steps)'\
 482                                              %(plc_counter,stepname,plcname))
 483                             status="KO"
 484                     except:
 485                         overall_result='FAILURE'
 486                         spec['failed_step'] = stepname
 487                         traceback.print_exc()
 488                         utils.header ('********** %d FAILED (exception) step %s on %s (discarded from further steps)'\
 489                                           %(plc_counter,stepname,plcname))
 490                         status="KO"
 491
 492                 # do not run, just display it's skipped
 493                 else:
 494                     why="has failed %s"%spec['failed_step']
 495                     utils.header("********** %d SKIPPED Step %s on %s (%s)"%(plc_counter,stepname,plcname,why))
 496                     status="UNDEF"
 497                 if not self.options.dry_run:
 498                     delay = datetime.now()-beg_time
 499                     seconds = int(delay.total_seconds())
 500                     duration = str(delay)
 501                     # always do this on stdout
 502                     print TRACE_FORMAT%locals()
 503                     # duplicate on trace_file if provided
 504                     if self.options.trace_file:
 505                         trace.write(TRACE_FORMAT%locals())
 506                         trace.flush()
 507
 508         if self.options.trace_file and not self.options.dry_run:
 509             trace.close()
 510
 511         # free local substrate
 512         LocalSubstrate.local_substrate.release(self.options)
 513
 514         return overall_result
 515
 516     # wrapper to run, returns a shell-compatible result
 517     # retcod:
 518     # 0: SUCCESS
 519     # 1: FAILURE
 520     # 2: SUCCESS but some ignored steps failed
 521     # 3: OTHER ERROR
 522     def main(self):
 523         try:
 524             success=self.run()
 525             if success == 'SUCCESS':    return 0
 526             elif success == 'IGNORED':  return 2
 527             else:                       return 1
 528         except SystemExit:
 529             print 'Caught SystemExit'
 530             return 3
 531         except:
 532             traceback.print_exc()
 533             return 3
 534
 535 if __name__ == "__main__":
 536     exit_code = TestMain().main()
 537     print "TestMain exit code",exit_code
 538     sys.exit(exit_code)