system/TestMain.py

   1 #!/usr/bin/python -u
   2
   3 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
   4 # Copyright (C) 2010 INRIA
   5 #
   6 import sys, os, os.path
   7 from optparse import OptionParser
   8 import traceback
   9 from time import strftime
  10 import readline
  11 import glob
  12
  13 import utils
  14 from TestPlc import TestPlc, Ignored
  15 from TestSite import TestSite
  16 from TestNode import TestNode
  17 from macros import sequences
  18
  19 # add $HOME in PYTHONPATH so we can import LocalSubstrate.py
  20 sys.path.append(os.environ['HOME'])
  21 import LocalSubstrate
  22
  23 class Step:
  24
  25     natives=TestPlc.__dict__
  26
  27     def display (self): return self.name.replace('_','-')
  28     def internal (self): return self.name.replace('-','_')
  29
  30     def __init__ (self, name):
  31         self.name=name
  32         # a native step is implemented as a method on TestPlc
  33         self.native = self.internal() in Step.natives
  34         if self.native:
  35             self.method=Step.natives[self.internal()]
  36         else:
  37             try:
  38                 self.substeps=sequences[self.internal()]
  39             except Exception,e:
  40                 print "macro step %s not found in macros.py (%s) - exiting"%(self.display(),e)
  41                 raise
  42
  43     def print_doc (self,level=0):
  44         tab=32
  45         trail=8
  46         if self.native:
  47             start=level*' '+'* '
  48             # 2 is the len of '* '
  49             width=tab-level-2
  50             format="%%-%ds"%width
  51             line=start+format%self.display()
  52             print line,
  53             try:
  54                 print self.method.__doc__
  55             except:
  56                 print "*** no doc found"
  57         else:
  58             beg_start=level*' '+'>>> '
  59             end_start=level*' '+'<<< '
  60             trailer=trail*'-'
  61             # 4 is the len of '>>> '
  62             width=tab-level-4-trail
  63             format=("%%-%ds"%width)
  64             beg_line=beg_start+format%self.display()+trail*'>'
  65             end_line=end_start+format%self.display()+trail*'<'
  66             print beg_line
  67             for step in self.substeps:
  68                 Step(step).print_doc(level+1)
  69             print end_line
  70
  71     # return a list of (name, method) for all native steps involved
  72     def tuples (self):
  73         if self.native: return [ (self.internal(), self.method,) ]
  74         else:
  75             result=[]
  76             for substep in [ Step(name) for name in self.substeps ] :
  77                 result += substep.tuples()
  78             return result
  79
  80     # convenience for listing macros
  81     # just do a listdir, hoping we're in the right directory...
  82     @staticmethod
  83     def list_macros ():
  84         names= sequences.keys()
  85         names.sort()
  86         return names
  87
  88 class TestMain:
  89
  90     subversion_id = "Now using git -- version tracker broken"
  91
  92     default_config = [ 'default' ]
  93     default_rspec_styles = [ 'pl', 'pg' ]
  94
  95     default_build_url = "git://git.onelab.eu/tests"
  96
  97     def __init__ (self):
  98         self.path=os.path.dirname(sys.argv[0]) or "."
  99         os.chdir(self.path)
 100
 101     def show_env (self,options, message):
 102         if self.options.verbose:
 103             utils.header (message)
 104             utils.show_options("main options",options)
 105
 106     def init_steps(self):
 107         self.steps_message  = 20*'x'+" Defaut steps are\n"+TestPlc.printable_steps(TestPlc.default_steps)
 108         self.steps_message += 20*'x'+" Other useful steps are\n"+TestPlc.printable_steps(TestPlc.other_steps)
 109         self.steps_message += 20*'x'+" Macro steps are\n"+" ".join(Step.list_macros())
 110
 111     def list_steps(self):
 112         if not self.options.verbose:
 113             print self.steps_message,
 114         else:
 115             # steps mentioned on the command line
 116             if self.options.args:
 117                 scopes = [("Argument steps",self.options.args)]
 118             else:
 119                 scopes = [("Default steps",TestPlc.default_steps)]
 120                 if self.options.all_steps:
 121                     scopes.append ( ("Other steps",TestPlc.other_steps) )
 122                     # try to list macro steps as well
 123                     scopes.append ( ("Macro steps", Step.list_macros()) )
 124             for (scope,steps) in scopes:
 125                 print '--------------------',scope
 126                 for step in [step for step in steps if TestPlc.valid_step(step)]:
 127                     try:        (step,qualifier)=step.split('@')
 128                     except:     pass
 129                     stepname=step
 130                     for special in ['force','ignore']:
 131                         stepname = stepname.replace('_'+special,"")
 132                     Step(stepname).print_doc()
 133
 134     def run (self):
 135         self.init_steps()
 136         usage = """usage: %%prog [options] steps
 137 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
 138    no default
 139 config defaults to the last value used, as stored in arg-config,
 140    or %r
 141 ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode},
 142    default is to use IP scanning
 143 steps refer to a method in TestPlc or to a step_* module
 144 ===
 145 """%(TestMain.default_config)
 146         usage += self.steps_message
 147         parser=OptionParser(usage=usage,version=self.subversion_id)
 148         parser.add_option("-u","--url",action="store", dest="arch_rpms_url",
 149                           help="URL of the arch-dependent RPMS area - for locating what to test")
 150         parser.add_option("-b","--build",action="store", dest="build_url",
 151                           help="ignored, for legacy only")
 152         parser.add_option("-c","--config",action="append", dest="config", default=[],
 153                           help="Config module - can be set multiple times, or use quotes")
 154         parser.add_option("-p","--personality",action="store", dest="personality",
 155                           help="personality - as in vbuild-nightly")
 156         parser.add_option("-d","--pldistro",action="store", dest="pldistro",
 157                           help="pldistro - as in vbuild-nightly")
 158         parser.add_option("-f","--fcdistro",action="store", dest="fcdistro",
 159                           help="fcdistro - as in vbuild-nightly")
 160         parser.add_option("-e","--exclude",action="append", dest="exclude", default=[],
 161                           help="steps to exclude - can be set multiple times, or use quotes")
 162         parser.add_option("-i","--ignore",action="append", dest="ignore", default=[],
 163                           help="steps to run but ignore - can be set multiple times, or use quotes")
 164         parser.add_option("-a","--all",action="store_true",dest="all_steps", default=False,
 165                           help="Run all default steps")
 166         parser.add_option("-l","--list",action="store_true",dest="list_steps", default=False,
 167                           help="List known steps")
 168         parser.add_option("-V","--vserver",action="append", dest="ips_bplc", default=[],
 169                           help="Specify the set of hostnames for the boxes that host the plcs")
 170         parser.add_option("-P","--plcs",action="append", dest="ips_vplc", default=[],
 171                           help="Specify the set of hostname/IP's to use for vplcs")
 172         parser.add_option("-Q","--qemus",action="append", dest="ips_bnode", default=[],
 173                           help="Specify the set of hostnames for the boxes that host the nodes")
 174         parser.add_option("-N","--nodes",action="append", dest="ips_vnode", default=[],
 175                           help="Specify the set of hostname/IP's to use for vnodes")
 176         parser.add_option ('-X', "--lxc",action='store_true',dest='plcs_use_lxc',default=True,
 177                            help='use lxc-enabled plc boxes instead of vs-enabled ones')
 178         parser.add_option ('-S', "--vs",action='store_false',dest='plcs_use_lxc',
 179                            help='use lxc-enabled plc boxes instead of vs-enabled ones')
 180         parser.add_option("-s","--size",action="store",type="int",dest="size",default=1,
 181                           help="sets test size in # of plcs - default is 1")
 182         parser.add_option("-q","--qualifier",action="store",type="int",dest="qualifier",default=None,
 183                           help="run steps only on plc numbered <qualifier>, starting at 1")
 184         parser.add_option("-y","--rspec-style",action="append",dest="rspec_styles",default=[],
 185                           help="pl is for planetlab rspecs, pg is for protogeni")
 186         parser.add_option("-k","--keep-going",action="store",dest="keep_going",default=False,
 187                           help="proceeds even if some steps are failing")
 188         parser.add_option("-D","--dbname",action="store",dest="dbname",default=None,
 189                            help="Used by plc_db_dump and plc_db_restore")
 190         parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
 191                           help="Run in verbose mode")
 192         parser.add_option("-I","--interactive",action="store_true",dest="interactive",default=False,
 193                           help="prompts before each step")
 194         parser.add_option("-n","--dry-run", action="store_true", dest="dry_run", default=False,
 195                           help="Show environment and exits")
 196 # dropped when added Completer.py
 197 #        parser.add_option("-r","--restart-nm", action="store_true", dest="forcenm", default=False,
 198 #                          help="Force the NM to restart in ssh_slices step")
 199         parser.add_option("-t","--trace", action="store", dest="trace_file", default=None,
 200                           help="Trace file location")
 201         (self.options, self.args) = parser.parse_args()
 202
 203         # allow things like "run -c 'c1 c2' -c c3"
 204         def flatten (x):
 205             result = []
 206             for el in x:
 207                 if hasattr(el, "__iter__") and not isinstance(el, basestring):
 208                     result.extend(flatten(el))
 209                 else:
 210                     result.append(el)
 211             return result
 212         # flatten relevant options
 213         for optname in ['config','exclude','ignore','ips_bplc','ips_vplc','ips_bnode','ips_vnode']:
 214             setattr(self.options,optname, flatten ( [ arg.split() for arg in getattr(self.options,optname) ] ))
 215
 216         if not self.options.rspec_styles:
 217             self.options.rspec_styles=TestMain.default_rspec_styles
 218
 219         # handle defaults and option persistence
 220         for (recname,filename,default,need_reverse) in (
 221             ('build_url','arg-build-url',TestMain.default_build_url,None) ,
 222             ('ips_bplc','arg-ips-bplc',[],True),
 223             ('ips_vplc','arg-ips-vplc',[],True) ,
 224             ('ips_bnode','arg-ips-bnode',[],True),
 225             ('ips_vnode','arg-ips-vnode',[],True) ,
 226             ('config','arg-config',TestMain.default_config,False) ,
 227             ('arch_rpms_url','arg-arch-rpms-url',"",None) ,
 228             ('personality','arg-personality',"linux64",None),
 229             ('pldistro','arg-pldistro',"onelab",None),
 230             ('fcdistro','arg-fcdistro','f14',None),
 231             ('plcs_use_lxc','arg-plcs-use-lxc',False,None),
 232             ) :
 233 #            print 'handling',recname
 234             path=filename
 235             is_list = isinstance(default,list)
 236             is_bool = isinstance(default,bool)
 237             if not getattr(self.options,recname):
 238                 try:
 239                     parsed=file(path).readlines()
 240                     if is_list:         # lists
 241                         parsed=[x.strip() for x in parsed]
 242                     else:               # strings and booleans
 243                         if len(parsed) != 1:
 244                             print "%s - error when parsing %s"%(sys.argv[1],path)
 245                             sys.exit(1)
 246                         parsed=parsed[0].strip()
 247                         if is_bool:
 248                             parsed = parsed.lower()=='true'
 249                     setattr(self.options,recname,parsed)
 250                 except:
 251                     if default != "":
 252                         setattr(self.options,recname,default)
 253                     else:
 254                         print "Cannot determine",recname
 255                         print "Run %s --help for help"%sys.argv[0]
 256                         sys.exit(1)
 257
 258             # save for next run
 259             fsave=open(path,"w")
 260             if is_list:                 # lists
 261                 for value in getattr(self.options,recname):
 262                     fsave.write(value + "\n")
 263             else:                       # strings and booleans - just call str()
 264                 fsave.write(str(getattr(self.options,recname)) + "\n")
 265             fsave.close()
 266 #            utils.header('Saved %s into %s'%(recname,filename))
 267
 268             # lists need be reversed
 269             # I suspect this is useful for the various pools but for config, it's painful
 270             if isinstance(getattr(self.options,recname),list) and need_reverse:
 271                 getattr(self.options,recname).reverse()
 272
 273             if self.options.verbose:
 274                 utils.header('* Using %s = %s'%(recname,getattr(self.options,recname)))
 275
 276         # hack : if sfa is not among the published rpms, skip these tests
 277         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
 278
 279         # no step specified
 280         self.options.args = self.args
 281         if len(self.args) == 0:
 282             self.options.steps=TestPlc.default_steps
 283         else:
 284             self.options.steps = self.args
 285
 286         if self.options.list_steps:
 287             self.init_steps()
 288             self.list_steps()
 289             return 'SUCCESS'
 290
 291         # steps
 292         if not self.options.steps:
 293             #default (all) steps
 294             #self.options.steps=['dump','clean','install','populate']
 295             self.options.steps=TestPlc.default_steps
 296
 297         # rewrite '-' into '_' in step names
 298         self.options.steps = [ step.replace('-','_') for step in self.options.steps ]
 299         self.options.exclude = [ step.replace('-','_') for step in self.options.exclude ]
 300         self.options.ignore = [ step.replace('-','_') for step in self.options.ignore ]
 301
 302         # technicality, decorate known steps to produce the '_ignore' version
 303         TestPlc.create_ignore_steps()
 304
 305         # exclude
 306         selected=[]
 307         for step in self.options.steps:
 308             keep=True
 309             for exclude in self.options.exclude:
 310                 if utils.match(step,exclude):
 311                     keep=False
 312                     break
 313             if keep: selected.append(step)
 314
 315         # ignore
 316         selected = [ step if step not in self.options.ignore else step+"_ignore"
 317                      for step in selected ]
 318
 319         self.options.steps=selected
 320
 321         # this is useful when propagating on host boxes, to avoid conflicts
 322         self.options.buildname = os.path.basename (os.path.abspath (self.path))
 323
 324         if self.options.verbose:
 325             self.show_env(self.options,"Verbose")
 326
 327         # load configs
 328         all_plc_specs = []
 329         for config in self.options.config:
 330             modulename='config_'+config
 331             try:
 332                 m = __import__(modulename)
 333                 all_plc_specs = m.config(all_plc_specs,self.options)
 334             except :
 335                 traceback.print_exc()
 336                 print 'Cannot load config %s -- ignored'%modulename
 337                 raise
 338
 339         # provision on local substrate
 340         if self.options.plcs_use_lxc: LocalSubstrate.local_substrate.rescope (plcs_on_vs=False, plcs_on_lxc=True)
 341         all_plc_specs = LocalSubstrate.local_substrate.provision(all_plc_specs,self.options)
 342
 343         # remember substrate IP address(es) for next run
 344         ips_bplc_file=open('arg-ips-bplc','w')
 345         for plc_spec in all_plc_specs:
 346             ips_bplc_file.write("%s\n"%plc_spec['host_box'])
 347         ips_bplc_file.close()
 348         ips_vplc_file=open('arg-ips-vplc','w')
 349         for plc_spec in all_plc_specs:
 350             ips_vplc_file.write("%s\n"%plc_spec['PLC_API_HOST'])
 351         ips_vplc_file.close()
 352         # ditto for nodes
 353         ips_bnode_file=open('arg-ips-bnode','w')
 354         for plc_spec in all_plc_specs:
 355             for site_spec in plc_spec['sites']:
 356                 for node_spec in site_spec['nodes']:
 357                     ips_bnode_file.write("%s\n"%node_spec['host_box'])
 358         ips_bnode_file.close()
 359         ips_vnode_file=open('arg-ips-vnode','w')
 360         for plc_spec in all_plc_specs:
 361             for site_spec in plc_spec['sites']:
 362                 for node_spec in site_spec['nodes']:
 363                     # back to normal (unqualified) form
 364                     stripped=node_spec['node_fields']['hostname'].split('.')[0]
 365                     ips_vnode_file.write("%s\n"%stripped)
 366         ips_vnode_file.close()
 367
 368         # build a TestPlc object from the result, passing options
 369         for spec in all_plc_specs:
 370             spec['failed_step'] = False
 371         all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs]
 372
 373         # pass options to utils as well
 374         utils.init_options(self.options)
 375
 376         overall_result = 'SUCCESS'
 377         all_step_infos=[]
 378         for step in self.options.steps:
 379             if not TestPlc.valid_step(step):
 380                 continue
 381             # some steps need to be done regardless of the previous ones: we force them
 382             force=False
 383             if step.endswith("_force"):
 384                 step=step.replace("_force","")
 385                 force=True
 386             # allow for steps to specify an index like in
 387             # run checkslice@2
 388             try:        (step,qualifier)=step.split('@')
 389             except:     qualifier=self.options.qualifier
 390
 391             try:
 392                 stepobj = Step (step)
 393                 for (substep, method) in stepobj.tuples():
 394                     # a cross step will run a method on TestPlc that has a signature like
 395                     # def cross_foo (self, all_test_plcs)
 396                     cross=False
 397                     if substep.find("cross_") == 0:
 398                         cross=True
 399                     all_step_infos.append ( (substep, method, force, cross, qualifier, ) )
 400             except :
 401                 utils.header("********** FAILED step %s (NOT FOUND) -- won't be run"%step)
 402                 traceback.print_exc()
 403                 overall_result = 'FAILURE'
 404
 405         if self.options.dry_run:
 406             self.show_env(self.options,"Dry run")
 407
 408         # init & open trace file if provided
 409         if self.options.trace_file and not self.options.dry_run:
 410             # create dir if needed
 411             trace_dir=os.path.dirname(self.options.trace_file)
 412             if trace_dir and not os.path.isdir(trace_dir):
 413                 os.makedirs(trace_dir)
 414             trace=open(self.options.trace_file,"w")
 415
 416         # do all steps on all plcs
 417         TIME_FORMAT="%H-%M-%S"
 418         TRACE_FORMAT="TRACE: %(plc_counter)d %(beg)s->%(end)s status=%(status)s step=%(stepname)s plc=%(plcname)s force=%(force)s\n"
 419         for (stepname,method,force,cross,qualifier) in all_step_infos:
 420             plc_counter=0
 421             for (spec,plc_obj) in all_plcs:
 422                 plc_counter+=1
 423                 # skip this step if we have specified a plc_explicit
 424                 if qualifier and plc_counter!=int(qualifier): continue
 425
 426                 plcname=spec['name']
 427                 across_plcs = [ o for (s,o) in all_plcs if o!=plc_obj ]
 428
 429                 # run the step
 430                 beg=strftime(TIME_FORMAT)
 431                 if not spec['failed_step'] or force or self.options.interactive or self.options.keep_going:
 432                     skip_step=False
 433                     if self.options.interactive:
 434                         prompting=True
 435                         while prompting:
 436                             msg="%d Run step %s on %s [r](un)/d(ry_run)/p(roceed)/s(kip)/q(uit) ? "%(plc_counter,stepname,plcname)
 437                             answer=raw_input(msg).strip().lower() or "r"
 438                             answer=answer[0]
 439                             if answer in ['s','n']:     # skip/no/next
 440                                 print '%s on %s skipped'%(stepname,plcname)
 441                                 prompting=False
 442                                 skip_step=True
 443                             elif answer in ['q','b']:   # quit/bye
 444                                 print 'Exiting'
 445                                 return 'FAILURE'
 446                             elif answer in ['d']:       # dry_run
 447                                 dry_run=self.options.dry_run
 448                                 self.options.dry_run=True
 449                                 plc_obj.options.dry_run=True
 450                                 plc_obj.apiserver.set_dry_run(True)
 451                                 if not cross:   step_result=method(plc_obj)
 452                                 else:           step_result=method(plc_obj,across_plcs)
 453                                 print 'dry_run step ->',step_result
 454                                 self.options.dry_run=dry_run
 455                                 plc_obj.options.dry_run=dry_run
 456                                 plc_obj.apiserver.set_dry_run(dry_run)
 457                             elif answer in ['p']:
 458                                 # take it as a yes and leave interactive mode
 459                                 prompting=False
 460                                 self.options.interactive=False
 461                             elif answer in ['r','y']:   # run/yes
 462                                 prompting=False
 463                     if skip_step:
 464                         continue
 465                     try:
 466                         force_msg=""
 467                         if force and spec['failed_step']: force_msg=" (forced after %s has failed)"%spec['failed_step']
 468                         utils.header("********** %d RUNNING step %s%s on plc %s"%(plc_counter,stepname,force_msg,plcname))
 469                         if not cross:   step_result = method(plc_obj)
 470                         else:           step_result = method(plc_obj,across_plcs)
 471                         if isinstance (step_result,Ignored):
 472                             step_result=step_result.result
 473                             if step_result:
 474                                 msg="OK"
 475                             else:
 476                                 msg="KO"
 477                                 # do not overwrite if FAILURE
 478                                 if overall_result=='SUCCESS':
 479                                     overall_result='IGNORED'
 480                             utils.header('********** %d IGNORED (%s) step %s on %s'%(plc_counter,msg,stepname,plcname))
 481                             status="%s[I]"%msg
 482                         elif step_result:
 483                             utils.header('********** %d SUCCESSFUL step %s on %s'%(plc_counter,stepname,plcname))
 484                             status="OK"
 485                         else:
 486                             overall_result = 'FAILURE'
 487                             spec['failed_step'] = stepname
 488                             utils.header('********** %d FAILED Step %s on %s (discarded from further steps)'\
 489                                              %(plc_counter,stepname,plcname))
 490                             status="KO"
 491                     except:
 492                         overall_result='FAILURE'
 493                         spec['failed_step'] = stepname
 494                         traceback.print_exc()
 495                         utils.header ('********** %d FAILED (exception) Step %s on %s (discarded from further steps)'\
 496                                           %(plc_counter,stepname,plcname))
 497                         status="KO"
 498
 499                 # do not run, just display it's skipped
 500                 else:
 501                     why="has failed %s"%spec['failed_step']
 502                     utils.header("********** %d SKIPPED Step %s on %s (%s)"%(plc_counter,stepname,plcname,why))
 503                     status="UNDEF"
 504                 if not self.options.dry_run:
 505                     end=strftime(TIME_FORMAT)
 506                     # always do this on stdout
 507                     print TRACE_FORMAT%locals()
 508                     # duplicate on trace_file if provided
 509                     if self.options.trace_file:
 510                         trace.write(TRACE_FORMAT%locals())
 511                         trace.flush()
 512
 513         if self.options.trace_file and not self.options.dry_run:
 514             trace.close()
 515
 516         # free local substrate
 517         LocalSubstrate.local_substrate.release(self.options)
 518
 519         return overall_result
 520
 521     # wrapper to run, returns a shell-compatible result
 522     # retcod:
 523     # 0: SUCCESS
 524     # 1: FAILURE
 525     # 2: SUCCESS but some ignored steps failed
 526     # 3: OTHER ERROR
 527     def main(self):
 528         try:
 529             success=self.run()
 530             print 'run has returned %s'%success
 531             if success == 'SUCCESS':    return 0
 532             elif success == 'IGNORED':  return 2
 533             else:                       return 1
 534         except SystemExit:
 535             print 'Caught SystemExit'
 536             return 3
 537         except:
 538             traceback.print_exc()
 539             return 3
 540
 541 if __name__ == "__main__":
 542     exit_code = TestMain().main()
 543     print 'run_log is exiting',exit_code
 544     sys.exit(exit_code)