system/TestMain.py

   1 #!/usr/bin/python -u
   2
   3 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
   4 # Copyright (C) 2010 INRIA
   5 #
   6 import sys, os, os.path
   7 from optparse import OptionParser
   8 import traceback
   9 from time import strftime
  10 import readline
  11 import glob
  12
  13 import utils
  14 from TestPlc import TestPlc, Ignored
  15 from TestSite import TestSite
  16 from TestNode import TestNode
  17 from macros import sequences
  18
  19 # add $HOME in PYTHONPATH so we can import LocalSubstrate.py
  20 sys.path.append(os.environ['HOME'])
  21 import LocalSubstrate
  22
  23 class Step:
  24
  25     natives=TestPlc.__dict__
  26
  27     def display (self): return self.name.replace('_','-')
  28     def internal (self): return self.name.replace('-','_')
  29
  30     def __init__ (self, name):
  31         self.name=name
  32         # a native step is implemented as a method on TestPlc
  33         self.native = self.internal() in Step.natives
  34         if self.native:
  35             self.method=Step.natives[self.internal()]
  36         else:
  37             try:
  38                 self.substeps=sequences[self.internal()]
  39             except Exception,e:
  40                 print "macro step %s not found in macros.py (%s) - exiting"%(self.display(),e)
  41                 raise
  42
  43     def print_doc (self,level=0):
  44         tab=32
  45         trail=8
  46         if self.native:
  47             start=level*' '+'* '
  48             # 2 is the len of '* '
  49             width=tab-level-2
  50             format="%%-%ds"%width
  51             line=start+format%self.display()
  52             print line,
  53             try:
  54                 print self.method.__doc__
  55             except:
  56                 print "*** no doc found"
  57         else:
  58             beg_start=level*' '+'>>> '
  59             end_start=level*' '+'<<< '
  60             trailer=trail*'-'
  61             # 4 is the len of '>>> '
  62             width=tab-level-4-trail
  63             format=("%%-%ds"%width)
  64             beg_line=beg_start+format%self.display()+trail*'>'
  65             end_line=end_start+format%self.display()+trail*'<'
  66             print beg_line
  67             for step in self.substeps:
  68                 Step(step).print_doc(level+1)
  69             print end_line
  70
  71     # return a list of (name, method) for all native steps involved
  72     def tuples (self):
  73         if self.native: return [ (self.internal(), self.method,) ]
  74         else:
  75             result=[]
  76             for substep in [ Step(name) for name in self.substeps ] :
  77                 result += substep.tuples()
  78             return result
  79
  80     # convenience for listing macros
  81     # just do a listdir, hoping we're in the right directory...
  82     @staticmethod
  83     def list_macros ():
  84         names= sequences.keys()
  85         names.sort()
  86         return names
  87
  88 class TestMain:
  89
  90     subversion_id = "Now using git -- version tracker broken"
  91
  92     default_config = [ 'default' ]
  93     default_rspec_styles = [ 'pl', 'pg' ]
  94
  95     default_build_url = "git://git.onelab.eu/tests"
  96
  97     def __init__ (self):
  98         self.path=os.path.dirname(sys.argv[0]) or "."
  99         os.chdir(self.path)
 100
 101     def show_env (self,options, message):
 102         if self.options.verbose:
 103             utils.header (message)
 104             utils.show_options("main options",options)
 105
 106     def init_steps(self):
 107         self.steps_message  = 20*'x'+" Defaut steps are\n"+TestPlc.printable_steps(TestPlc.default_steps)
 108         self.steps_message += 20*'x'+" Other useful steps are\n"+TestPlc.printable_steps(TestPlc.other_steps)
 109         self.steps_message += 20*'x'+" Macro steps are\n"+" ".join(Step.list_macros())
 110
 111     def list_steps(self):
 112         if not self.options.verbose:
 113             print self.steps_message,
 114         else:
 115             # steps mentioned on the command line
 116             if self.options.args:
 117                 scopes = [("Argument steps",self.options.args)]
 118             else:
 119                 scopes = [("Default steps",TestPlc.default_steps)]
 120                 if self.options.all_steps:
 121                     scopes.append ( ("Other steps",TestPlc.other_steps) )
 122                     # try to list macro steps as well
 123                     scopes.append ( ("Macro steps", Step.list_macros()) )
 124             for (scope,steps) in scopes:
 125                 print '--------------------',scope
 126                 for step in [step for step in steps if TestPlc.valid_step(step)]:
 127                     try:        (step,qualifier)=step.split('@')
 128                     except:     pass
 129                     stepname=step
 130                     for special in ['force','ignore']:
 131                         stepname = stepname.replace('_'+special,"")
 132                     Step(stepname).print_doc()
 133
 134     def run (self):
 135         self.init_steps()
 136         usage = """usage: %%prog [options] steps
 137 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
 138    no default
 139 config defaults to the last value used, as stored in arg-config,
 140    or %r
 141 ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode},
 142    default is to use IP scanning
 143 steps refer to a method in TestPlc or to a step_* module
 144 ===
 145 """%(TestMain.default_config)
 146         usage += self.steps_message
 147         parser=OptionParser(usage=usage,version=self.subversion_id)
 148         parser.add_option("-u","--url",action="store", dest="arch_rpms_url",
 149                           help="URL of the arch-dependent RPMS area - for locating what to test")
 150         parser.add_option("-b","--build",action="store", dest="build_url",
 151                           help="ignored, for legacy only")
 152         parser.add_option("-c","--config",action="append", dest="config", default=[],
 153                           help="Config module - can be set multiple times, or use quotes")
 154         parser.add_option("-p","--personality",action="store", dest="personality",
 155                           help="personality - as in vbuild-nightly")
 156         parser.add_option("-d","--pldistro",action="store", dest="pldistro",
 157                           help="pldistro - as in vbuild-nightly")
 158         parser.add_option("-f","--fcdistro",action="store", dest="fcdistro",
 159                           help="fcdistro - as in vbuild-nightly")
 160         parser.add_option("-e","--exclude",action="append", dest="exclude", default=[],
 161                           help="steps to exclude - can be set multiple times, or use quotes")
 162         parser.add_option("-i","--ignore",action="append", dest="ignore", default=[],
 163                           help="steps to run but ignore - can be set multiple times, or use quotes")
 164         parser.add_option("-a","--all",action="store_true",dest="all_steps", default=False,
 165                           help="Run all default steps")
 166         parser.add_option("-l","--list",action="store_true",dest="list_steps", default=False,
 167                           help="List known steps")
 168         parser.add_option("-V","--vserver",action="append", dest="ips_bplc", default=[],
 169                           help="Specify the set of hostnames for the boxes that host the plcs")
 170         parser.add_option("-P","--plcs",action="append", dest="ips_vplc", default=[],
 171                           help="Specify the set of hostname/IP's to use for vplcs")
 172         parser.add_option("-Q","--qemus",action="append", dest="ips_bnode", default=[],
 173                           help="Specify the set of hostnames for the boxes that host the nodes")
 174         parser.add_option("-N","--nodes",action="append", dest="ips_vnode", default=[],
 175                           help="Specify the set of hostname/IP's to use for vnodes")
 176         parser.add_option("-s","--size",action="store",type="int",dest="size",default=1,
 177                           help="sets test size in # of plcs - default is 1")
 178         parser.add_option("-q","--qualifier",action="store",type="int",dest="qualifier",default=None,
 179                           help="run steps only on plc numbered <qualifier>, starting at 1")
 180         parser.add_option("-y","--rspec-style",action="append",dest="rspec_styles",default=[],
 181                           help="pl is for planetlab rspecs, pg is for protogeni")
 182         parser.add_option("-k","--keep-going",action="store",dest="keep_going",default=False,
 183                           help="proceeds even if some steps are failing")
 184         parser.add_option("-D","--dbname",action="store",dest="dbname",default=None,
 185                            help="Used by plc_db_dump and plc_db_restore")
 186         parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
 187                           help="Run in verbose mode")
 188         parser.add_option("-I","--interactive",action="store_true",dest="interactive",default=False,
 189                           help="prompts before each step")
 190         parser.add_option("-n","--dry-run", action="store_true", dest="dry_run", default=False,
 191                           help="Show environment and exits")
 192 # dropped when added Completer.py
 193 #        parser.add_option("-r","--restart-nm", action="store_true", dest="forcenm", default=False,
 194 #                          help="Force the NM to restart in ssh_slices step")
 195         parser.add_option("-t","--trace", action="store", dest="trace_file", default=None,
 196                           help="Trace file location")
 197         (self.options, self.args) = parser.parse_args()
 198
 199         # allow things like "run -c 'c1 c2' -c c3"
 200         def flatten (x):
 201             result = []
 202             for el in x:
 203                 if hasattr(el, "__iter__") and not isinstance(el, basestring):
 204                     result.extend(flatten(el))
 205                 else:
 206                     result.append(el)
 207             return result
 208         # flatten relevant options
 209         for optname in ['config','exclude','ignore','ips_bplc','ips_vplc','ips_bnode','ips_vnode']:
 210             setattr(self.options,optname, flatten ( [ arg.split() for arg in getattr(self.options,optname) ] ))
 211
 212         if not self.options.rspec_styles:
 213             self.options.rspec_styles=TestMain.default_rspec_styles
 214
 215         # handle defaults and option persistence
 216         for (recname,filename,default,need_reverse) in (
 217             ('build_url','arg-build-url',TestMain.default_build_url,None) ,
 218             ('ips_bplc','arg-ips-bplc',[],True),
 219             ('ips_vplc','arg-ips-vplc',[],True) ,
 220             ('ips_bnode','arg-ips-bnode',[],True),
 221             ('ips_vnode','arg-ips-vnode',[],True) ,
 222             ('config','arg-config',TestMain.default_config,False) ,
 223             ('arch_rpms_url','arg-arch-rpms-url',"",None) ,
 224             ('personality','arg-personality',"linux64",None),
 225             ('pldistro','arg-pldistro',"onelab",None),
 226             ('fcdistro','arg-fcdistro','f14',None),
 227             ) :
 228 #            print 'handling',recname
 229             path=filename
 230             is_list = isinstance(default,list)
 231             is_bool = isinstance(default,bool)
 232             if not getattr(self.options,recname):
 233                 try:
 234                     parsed=file(path).readlines()
 235                     if is_list:         # lists
 236                         parsed=[x.strip() for x in parsed]
 237                     else:               # strings and booleans
 238                         if len(parsed) != 1:
 239                             print "%s - error when parsing %s"%(sys.argv[1],path)
 240                             sys.exit(1)
 241                         parsed=parsed[0].strip()
 242                         if is_bool:
 243                             parsed = parsed.lower()=='true'
 244                     setattr(self.options,recname,parsed)
 245                 except:
 246                     if default != "":
 247                         setattr(self.options,recname,default)
 248                     else:
 249                         print "Cannot determine",recname
 250                         print "Run %s --help for help"%sys.argv[0]
 251                         sys.exit(1)
 252
 253             # save for next run
 254             fsave=open(path,"w")
 255             if is_list:                 # lists
 256                 for value in getattr(self.options,recname):
 257                     fsave.write(value + "\n")
 258             else:                       # strings and booleans - just call str()
 259                 fsave.write(str(getattr(self.options,recname)) + "\n")
 260             fsave.close()
 261 #            utils.header('Saved %s into %s'%(recname,filename))
 262
 263             # lists need be reversed
 264             # I suspect this is useful for the various pools but for config, it's painful
 265             if isinstance(getattr(self.options,recname),list) and need_reverse:
 266                 getattr(self.options,recname).reverse()
 267
 268             if self.options.verbose:
 269                 utils.header('* Using %s = %s'%(recname,getattr(self.options,recname)))
 270
 271         # hack : if sfa is not among the published rpms, skip these tests
 272         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
 273
 274         # no step specified
 275         self.options.args = self.args
 276         if len(self.args) == 0:
 277             self.options.steps=TestPlc.default_steps
 278         else:
 279             self.options.steps = self.args
 280
 281         if self.options.list_steps:
 282             self.init_steps()
 283             self.list_steps()
 284             return 'SUCCESS'
 285
 286         # steps
 287         if not self.options.steps:
 288             #default (all) steps
 289             #self.options.steps=['dump','clean','install','populate']
 290             self.options.steps=TestPlc.default_steps
 291
 292         # rewrite '-' into '_' in step names
 293         self.options.steps = [ step.replace('-','_') for step in self.options.steps ]
 294         self.options.exclude = [ step.replace('-','_') for step in self.options.exclude ]
 295         self.options.ignore = [ step.replace('-','_') for step in self.options.ignore ]
 296
 297         # technicality, decorate known steps to produce the '_ignore' version
 298         TestPlc.create_ignore_steps()
 299
 300         # exclude
 301         selected=[]
 302         for step in self.options.steps:
 303             keep=True
 304             for exclude in self.options.exclude:
 305                 if utils.match(step,exclude):
 306                     keep=False
 307                     break
 308             if keep: selected.append(step)
 309
 310         # ignore
 311         selected = [ step if step not in self.options.ignore else step+"_ignore"
 312                      for step in selected ]
 313
 314         self.options.steps=selected
 315
 316         # this is useful when propagating on host boxes, to avoid conflicts
 317         self.options.buildname = os.path.basename (os.path.abspath (self.path))
 318
 319         if self.options.verbose:
 320             self.show_env(self.options,"Verbose")
 321
 322         # load configs
 323         all_plc_specs = []
 324         for config in self.options.config:
 325             modulename='config_'+config
 326             try:
 327                 m = __import__(modulename)
 328                 all_plc_specs = m.config(all_plc_specs,self.options)
 329             except :
 330                 traceback.print_exc()
 331                 print 'Cannot load config %s -- ignored'%modulename
 332                 raise
 333
 334         # provision on local substrate
 335         all_plc_specs = LocalSubstrate.local_substrate.provision(all_plc_specs,self.options)
 336
 337         # remember substrate IP address(es) for next run
 338         ips_bplc_file=open('arg-ips-bplc','w')
 339         for plc_spec in all_plc_specs:
 340             ips_bplc_file.write("%s\n"%plc_spec['host_box'])
 341         ips_bplc_file.close()
 342         ips_vplc_file=open('arg-ips-vplc','w')
 343         for plc_spec in all_plc_specs:
 344             ips_vplc_file.write("%s\n"%plc_spec['PLC_API_HOST'])
 345         ips_vplc_file.close()
 346         # ditto for nodes
 347         ips_bnode_file=open('arg-ips-bnode','w')
 348         for plc_spec in all_plc_specs:
 349             for site_spec in plc_spec['sites']:
 350                 for node_spec in site_spec['nodes']:
 351                     ips_bnode_file.write("%s\n"%node_spec['host_box'])
 352         ips_bnode_file.close()
 353         ips_vnode_file=open('arg-ips-vnode','w')
 354         for plc_spec in all_plc_specs:
 355             for site_spec in plc_spec['sites']:
 356                 for node_spec in site_spec['nodes']:
 357                     # back to normal (unqualified) form
 358                     stripped=node_spec['node_fields']['hostname'].split('.')[0]
 359                     ips_vnode_file.write("%s\n"%stripped)
 360         ips_vnode_file.close()
 361
 362         # build a TestPlc object from the result, passing options
 363         for spec in all_plc_specs:
 364             spec['failed_step'] = False
 365         all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs]
 366
 367         # pass options to utils as well
 368         utils.init_options(self.options)
 369
 370         overall_result = 'SUCCESS'
 371         all_step_infos=[]
 372         for step in self.options.steps:
 373             if not TestPlc.valid_step(step):
 374                 continue
 375             # some steps need to be done regardless of the previous ones: we force them
 376             force=False
 377             if step.endswith("_force"):
 378                 step=step.replace("_force","")
 379                 force=True
 380             # allow for steps to specify an index like in
 381             # run checkslice@2
 382             try:        (step,qualifier)=step.split('@')
 383             except:     qualifier=self.options.qualifier
 384
 385             try:
 386                 stepobj = Step (step)
 387                 for (substep, method) in stepobj.tuples():
 388                     # a cross step will run a method on TestPlc that has a signature like
 389                     # def cross_foo (self, all_test_plcs)
 390                     cross=False
 391                     if substep.find("cross_") == 0:
 392                         cross=True
 393                     all_step_infos.append ( (substep, method, force, cross, qualifier, ) )
 394             except :
 395                 utils.header("********** FAILED step %s (NOT FOUND) -- won't be run"%step)
 396                 traceback.print_exc()
 397                 overall_result = 'FAILURE'
 398
 399         if self.options.dry_run:
 400             self.show_env(self.options,"Dry run")
 401
 402         # init & open trace file if provided
 403         if self.options.trace_file and not self.options.dry_run:
 404             # create dir if needed
 405             trace_dir=os.path.dirname(self.options.trace_file)
 406             if trace_dir and not os.path.isdir(trace_dir):
 407                 os.makedirs(trace_dir)
 408             trace=open(self.options.trace_file,"w")
 409
 410         # do all steps on all plcs
 411         TIME_FORMAT="%H-%M-%S"
 412         TRACE_FORMAT="TRACE: %(plc_counter)d %(beg)s->%(end)s status=%(status)s step=%(stepname)s plc=%(plcname)s force=%(force)s\n"
 413         for (stepname,method,force,cross,qualifier) in all_step_infos:
 414             plc_counter=0
 415             for (spec,plc_obj) in all_plcs:
 416                 plc_counter+=1
 417                 # skip this step if we have specified a plc_explicit
 418                 if qualifier and plc_counter!=int(qualifier): continue
 419
 420                 plcname=spec['name']
 421                 across_plcs = [ o for (s,o) in all_plcs if o!=plc_obj ]
 422
 423                 # run the step
 424                 beg=strftime(TIME_FORMAT)
 425                 if not spec['failed_step'] or force or self.options.interactive or self.options.keep_going:
 426                     skip_step=False
 427                     if self.options.interactive:
 428                         prompting=True
 429                         while prompting:
 430                             msg="%d Run step %s on %s [r](un)/d(ry_run)/p(roceed)/s(kip)/q(uit) ? "%(plc_counter,stepname,plcname)
 431                             answer=raw_input(msg).strip().lower() or "r"
 432                             answer=answer[0]
 433                             if answer in ['s','n']:     # skip/no/next
 434                                 print '%s on %s skipped'%(stepname,plcname)
 435                                 prompting=False
 436                                 skip_step=True
 437                             elif answer in ['q','b']:   # quit/bye
 438                                 print 'Exiting'
 439                                 return 'FAILURE'
 440                             elif answer in ['d']:       # dry_run
 441                                 dry_run=self.options.dry_run
 442                                 self.options.dry_run=True
 443                                 plc_obj.options.dry_run=True
 444                                 plc_obj.apiserver.set_dry_run(True)
 445                                 if not cross:   step_result=method(plc_obj)
 446                                 else:           step_result=method(plc_obj,across_plcs)
 447                                 print 'dry_run step ->',step_result
 448                                 self.options.dry_run=dry_run
 449                                 plc_obj.options.dry_run=dry_run
 450                                 plc_obj.apiserver.set_dry_run(dry_run)
 451                             elif answer in ['p']:
 452                                 # take it as a yes and leave interactive mode
 453                                 prompting=False
 454                                 self.options.interactive=False
 455                             elif answer in ['r','y']:   # run/yes
 456                                 prompting=False
 457                     if skip_step:
 458                         continue
 459                     try:
 460                         force_msg=""
 461                         if force and spec['failed_step']: force_msg=" (forced after %s has failed)"%spec['failed_step']
 462                         utils.header("********** %d RUNNING step %s%s on plc %s"%(plc_counter,stepname,force_msg,plcname))
 463                         if not cross:   step_result = method(plc_obj)
 464                         else:           step_result = method(plc_obj,across_plcs)
 465                         if isinstance (step_result,Ignored):
 466                             step_result=step_result.result
 467                             if step_result:
 468                                 msg="OK"
 469                             else:
 470                                 msg="KO"
 471                                 # do not overwrite if FAILURE
 472                                 if overall_result=='SUCCESS':
 473                                     overall_result='IGNORED'
 474                             utils.header('********** %d IGNORED (%s) step %s on %s'%(plc_counter,msg,stepname,plcname))
 475                             status="%s[I]"%msg
 476                         elif step_result:
 477                             utils.header('********** %d SUCCESSFUL step %s on %s'%(plc_counter,stepname,plcname))
 478                             status="OK"
 479                         else:
 480                             overall_result = 'FAILURE'
 481                             spec['failed_step'] = stepname
 482                             utils.header('********** %d FAILED Step %s on %s (discarded from further steps)'\
 483                                              %(plc_counter,stepname,plcname))
 484                             status="KO"
 485                     except:
 486                         overall_result='FAILURE'
 487                         spec['failed_step'] = stepname
 488                         traceback.print_exc()
 489                         utils.header ('********** %d FAILED (exception) Step %s on %s (discarded from further steps)'\
 490                                           %(plc_counter,stepname,plcname))
 491                         status="KO"
 492
 493                 # do not run, just display it's skipped
 494                 else:
 495                     why="has failed %s"%spec['failed_step']
 496                     utils.header("********** %d SKIPPED Step %s on %s (%s)"%(plc_counter,stepname,plcname,why))
 497                     status="UNDEF"
 498                 if not self.options.dry_run:
 499                     end=strftime(TIME_FORMAT)
 500                     # always do this on stdout
 501                     print TRACE_FORMAT%locals()
 502                     # duplicate on trace_file if provided
 503                     if self.options.trace_file:
 504                         trace.write(TRACE_FORMAT%locals())
 505                         trace.flush()
 506
 507         if self.options.trace_file and not self.options.dry_run:
 508             trace.close()
 509
 510         # free local substrate
 511         LocalSubstrate.local_substrate.release(self.options)
 512
 513         return overall_result
 514
 515     # wrapper to run, returns a shell-compatible result
 516     # retcod:
 517     # 0: SUCCESS
 518     # 1: FAILURE
 519     # 2: SUCCESS but some ignored steps failed
 520     # 3: OTHER ERROR
 521     def main(self):
 522         try:
 523             success=self.run()
 524             if success == 'SUCCESS':    return 0
 525             elif success == 'IGNORED':  return 2
 526             else:                       return 1
 527         except SystemExit:
 528             print 'Caught SystemExit'
 529             return 3
 530         except:
 531             traceback.print_exc()
 532             return 3
 533
 534 if __name__ == "__main__":
 535     exit_code = TestMain().main()
 536     print "TestMain exit code",exit_code
 537     sys.exit(exit_code)