system/TestMain.py

   1 #!/usr/bin/python -u
   2
   3 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
   4 # Copyright (C) 2010 INRIA
   5 #
   6 import sys, os, os.path
   7 from optparse import OptionParser
   8 import traceback
   9 from time import strftime
  10 import readline
  11 import glob
  12
  13 import utils
  14 from TestPlc import TestPlc
  15 from TestSite import TestSite
  16 from TestNode import TestNode
  17 from macros import sequences
  18
  19 # add $HOME in PYTHONPATH so we can import LocalSubstrate.py
  20 sys.path.append(os.environ['HOME'])
  21 import LocalSubstrate
  22
  23 class Step:
  24
  25     natives=TestPlc.__dict__
  26
  27     def __init__ (self, name):
  28         self.name=name.replace('-','_')
  29         # a native step is implemented as a method on TestPlc
  30         self.native = name in Step.natives
  31         if self.native:
  32             self.method=Step.natives[self.name]
  33         else:
  34             try:
  35                 self.substeps=sequences[self.name]
  36             except Exception,e:
  37                 print "macro step %s not found in macros.py (%s) - exiting"%(self.name,e)
  38                 raise
  39
  40     def norm_name (self): return self.name.replace('_','-')
  41
  42     def print_doc (self):
  43         if self.native:
  44             print '*',self.norm_name(),"\r",4*"\t",
  45             try:
  46                 print self.method.__doc__
  47             except:
  48                 print "*** no doc found"
  49         else:
  50             print '*',self.norm_name(),"\r",3*"\t","========== BEG MACRO step"
  51             for step in self.substeps:
  52                 Step(step).print_doc()
  53             print '*',self.norm_name(),"\r",3*"\t","========== END MACRO step"
  54
  55     # return a list of (name, method) for all native steps involved
  56     def tuples (self):
  57         if self.native: return [ (self.name, self.method,) ]
  58         else:
  59             result=[]
  60             for substep in [ Step(name) for name in self.substeps ] :
  61                 result += substep.tuples()
  62             return result
  63
  64     # convenience for listing macros
  65     # just do a listdir, hoping we're in the right directory...
  66     @staticmethod
  67     def list_macros ():
  68         names= sequences.keys()
  69         names.sort()
  70         return names
  71
  72 class TestMain:
  73
  74     subversion_id = "Now using git -- version tracker broken"
  75
  76     default_config = [ 'default' ]
  77     default_rspec_styles = [ 'pl', 'pg' ]
  78
  79     default_build_url = "git://git.onelab.eu/tests"
  80
  81     def __init__ (self):
  82         self.path=os.path.dirname(sys.argv[0]) or "."
  83         os.chdir(self.path)
  84
  85     def show_env (self,options, message):
  86         if self.options.verbose:
  87             utils.header (message)
  88             utils.show_options("main options",options)
  89
  90     def init_steps(self):
  91         self.steps_message  = 20*'x'+" Defaut steps are\n"+TestPlc.printable_steps(TestPlc.default_steps)
  92         self.steps_message += 20*'x'+" Other useful steps are\n"+TestPlc.printable_steps(TestPlc.other_steps)
  93         self.steps_message += 20*'x'+" Macro steps are\n"+" ".join(Step.list_macros())
  94
  95     def list_steps(self):
  96         if not self.options.verbose:
  97             print self.steps_message,
  98         else:
  99             # steps mentioned on the command line
 100             if self.options.args:
 101                 scopes = [("Argument steps",self.options.args)]
 102             else:
 103                 scopes = [("Default steps",TestPlc.default_steps)]
 104                 if self.options.all_steps:
 105                     scopes.append ( ("Other steps",TestPlc.other_steps) )
 106                     # try to list macro steps as well
 107                     scopes.append ( ("Macro steps", Step.list_macros()) )
 108             for (scope,steps) in scopes:
 109                 print '--------------------',scope
 110                 for step in [step for step in steps if TestPlc.valid_step(step)]:
 111                     try:        (step,qualifier)=step.split('@')
 112                     except:     pass
 113                     stepname=step
 114                     for special in ['force']:
 115                         stepname = stepname.replace(special+'_',"")
 116                     Step(stepname).print_doc()
 117
 118     def run (self):
 119         self.init_steps()
 120         usage = """usage: %%prog [options] steps
 121 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
 122    no default
 123 config defaults to the last value used, as stored in arg-config,
 124    or %r
 125 ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode},
 126    default is to use IP scanning
 127 steps refer to a method in TestPlc or to a step_* module
 128 ===
 129 """%(TestMain.default_config)
 130         usage += self.steps_message
 131         parser=OptionParser(usage=usage,version=self.subversion_id)
 132         parser.add_option("-u","--url",action="store", dest="arch_rpms_url",
 133                           help="URL of the arch-dependent RPMS area - for locating what to test")
 134         parser.add_option("-b","--build",action="store", dest="build_url",
 135                           help="ignored, for legacy only")
 136         parser.add_option("-c","--config",action="append", dest="config", default=[],
 137                           help="Config module - can be set multiple times, or use quotes")
 138         parser.add_option("-p","--personality",action="store", dest="personality",
 139                           help="personality - as in vbuild-nightly")
 140         parser.add_option("-d","--pldistro",action="store", dest="pldistro",
 141                           help="pldistro - as in vbuild-nightly")
 142         parser.add_option("-f","--fcdistro",action="store", dest="fcdistro",
 143                           help="fcdistro - as in vbuild-nightly")
 144         parser.add_option("-x","--exclude",action="append", dest="exclude", default=[],
 145                           help="steps to exclude - can be set multiple times, or use quotes")
 146         parser.add_option("-a","--all",action="store_true",dest="all_steps", default=False,
 147                           help="Run all default steps")
 148         parser.add_option("-l","--list",action="store_true",dest="list_steps", default=False,
 149                           help="List known steps")
 150         parser.add_option("-V","--vserver",action="append", dest="ips_bplc", default=[],
 151                           help="Specify the set of hostnames for the boxes that host the plcs")
 152         parser.add_option("-P","--plcs",action="append", dest="ips_vplc", default=[],
 153                           help="Specify the set of hostname/IP's to use for vplcs")
 154         parser.add_option("-Q","--qemus",action="append", dest="ips_bnode", default=[],
 155                           help="Specify the set of hostnames for the boxes that host the nodes")
 156         parser.add_option("-N","--nodes",action="append", dest="ips_vnode", default=[],
 157                           help="Specify the set of hostname/IP's to use for vnodes")
 158         parser.add_option("-s","--size",action="store",type="int",dest="size",default=1,
 159                           help="sets test size in # of plcs - default is 1")
 160         parser.add_option("-q","--qualifier",action="store",type="int",dest="qualifier",default=None,
 161                           help="run steps only on plc numbered <qualifier>, starting at 1")
 162         parser.add_option("-y","--rspec-style",action="append",dest="rspec_styles",default=[],
 163                           help="pl is for planetlab rspecs, pg is for protogeni")
 164         parser.add_option("-k","--keep-going",action="store",dest="keep_going",default=False,
 165                           help="proceeds even if some steps are failing")
 166         parser.add_option("-D","--dbname",action="store",dest="dbname",default=None,
 167                            help="Used by plc_db_dump and plc_db_restore")
 168         parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
 169                           help="Run in verbose mode")
 170         parser.add_option("-i","--interactive",action="store_true",dest="interactive",default=False,
 171                           help="prompts before each step")
 172         parser.add_option("-n","--dry-run", action="store_true", dest="dry_run", default=False,
 173                           help="Show environment and exits")
 174         parser.add_option("-r","--restart-nm", action="store_true", dest="forcenm", default=False,
 175                           help="Force the NM to restart in ssh_slices step")
 176         parser.add_option("-t","--trace", action="store", dest="trace_file", default=None,
 177                           help="Trace file location")
 178         (self.options, self.args) = parser.parse_args()
 179
 180         # allow things like "run -c 'c1 c2' -c c3"
 181         def flatten (x):
 182             result = []
 183             for el in x:
 184                 if hasattr(el, "__iter__") and not isinstance(el, basestring):
 185                     result.extend(flatten(el))
 186                 else:
 187                     result.append(el)
 188             return result
 189         # flatten relevant options
 190         for optname in ['config','exclude','ips_bplc','ips_vplc','ips_bnode','ips_vnode']:
 191             setattr(self.options,optname, flatten ( [ arg.split() for arg in getattr(self.options,optname) ] ))
 192
 193         if not self.options.rspec_styles:
 194             self.options.rspec_styles=TestMain.default_rspec_styles
 195
 196         # handle defaults and option persistence
 197         for (recname,filename,default,need_reverse) in (
 198             ('build_url','arg-build-url',TestMain.default_build_url,None) ,
 199             ('ips_bplc','arg-ips-bplc',[],True),
 200             ('ips_vplc','arg-ips-vplc',[],True) ,
 201             ('ips_bnode','arg-ips-bnode',[],True),
 202             ('ips_vnode','arg-ips-vnode',[],True) ,
 203             ('config','arg-config',TestMain.default_config,False) ,
 204             ('arch_rpms_url','arg-arch-rpms-url',"",None) ,
 205             ('personality','arg-personality',"linux64",None),
 206             ('pldistro','arg-pldistro',"onelab",None),
 207             ('fcdistro','arg-fcdistro','f14',None),
 208             ) :
 209 #            print 'handling',recname
 210             path=filename
 211             is_list = isinstance(default,list)
 212             if not getattr(self.options,recname):
 213                 try:
 214                     parsed=file(path).readlines()
 215                     if not is_list:    # strings
 216                         if len(parsed) != 1:
 217                             print "%s - error when parsing %s"%(sys.argv[1],path)
 218                             sys.exit(1)
 219                         parsed=parsed[0].strip()
 220                     else:              # lists
 221                         parsed=[x.strip() for x in parsed]
 222                     setattr(self.options,recname,parsed)
 223                 except:
 224                     if default != "":
 225                         setattr(self.options,recname,default)
 226                     else:
 227                         print "Cannot determine",recname
 228                         print "Run %s --help for help"%sys.argv[0]
 229                         sys.exit(1)
 230
 231             # save for next run
 232             fsave=open(path,"w")
 233             if not is_list:
 234                 fsave.write(getattr(self.options,recname) + "\n")
 235             else:
 236                 for value in getattr(self.options,recname):
 237                     fsave.write(value + "\n")
 238             fsave.close()
 239 #            utils.header('Saved %s into %s'%(recname,filename))
 240
 241             # lists need be reversed
 242             # I suspect this is useful for the various pools but for config, it's painful
 243             if isinstance(getattr(self.options,recname),list) and need_reverse:
 244                 getattr(self.options,recname).reverse()
 245
 246             if self.options.verbose:
 247                 utils.header('* Using %s = %s'%(recname,getattr(self.options,recname)))
 248
 249         # hack : if sfa is not among the published rpms, skip these tests
 250         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
 251
 252         # no step specified
 253         self.options.args = self.args
 254         if len(self.args) == 0:
 255             self.options.steps=TestPlc.default_steps
 256         else:
 257             self.options.steps = self.args
 258
 259         if self.options.list_steps:
 260             self.init_steps()
 261             self.list_steps()
 262             return True
 263
 264         # steps
 265         if not self.options.steps:
 266             #default (all) steps
 267             #self.options.steps=['dump','clean','install','populate']
 268             self.options.steps=TestPlc.default_steps
 269
 270         # rewrite '-' into '_' in step names
 271         self.options.steps = [ step.replace('-','_') for step in self.options.steps ]
 272
 273         # exclude
 274         selected=[]
 275         for step in self.options.steps:
 276             keep=True
 277             for exclude in self.options.exclude:
 278                 if utils.match(step,exclude):
 279                     keep=False
 280                     break
 281             if keep: selected.append(step)
 282         self.options.steps=selected
 283
 284         # this is useful when propagating on host boxes, to avoid conflicts
 285         self.options.buildname = os.path.basename (os.path.abspath (self.path))
 286
 287         if self.options.verbose:
 288             self.show_env(self.options,"Verbose")
 289
 290         # load configs
 291         all_plc_specs = []
 292         for config in self.options.config:
 293             modulename='config_'+config
 294             try:
 295                 m = __import__(modulename)
 296                 all_plc_specs = m.config(all_plc_specs,self.options)
 297             except :
 298                 traceback.print_exc()
 299                 print 'Cannot load config %s -- ignored'%modulename
 300                 raise
 301
 302         # provision on local substrate
 303         all_plc_specs = LocalSubstrate.local_substrate.provision(all_plc_specs,self.options)
 304
 305         # remember substrate IP address(es) for next run
 306         ips_bplc_file=open('arg-ips-bplc','w')
 307         for plc_spec in all_plc_specs:
 308             ips_bplc_file.write("%s\n"%plc_spec['host_box'])
 309         ips_bplc_file.close()
 310         ips_vplc_file=open('arg-ips-vplc','w')
 311         for plc_spec in all_plc_specs:
 312             ips_vplc_file.write("%s\n"%plc_spec['PLC_API_HOST'])
 313         ips_vplc_file.close()
 314         # ditto for nodes
 315         ips_bnode_file=open('arg-ips-bnode','w')
 316         for plc_spec in all_plc_specs:
 317             for site_spec in plc_spec['sites']:
 318                 for node_spec in site_spec['nodes']:
 319                     ips_bnode_file.write("%s\n"%node_spec['host_box'])
 320         ips_bnode_file.close()
 321         ips_vnode_file=open('arg-ips-vnode','w')
 322         for plc_spec in all_plc_specs:
 323             for site_spec in plc_spec['sites']:
 324                 for node_spec in site_spec['nodes']:
 325                     # back to normal (unqualified) form
 326                     stripped=node_spec['node_fields']['hostname'].split('.')[0]
 327                     ips_vnode_file.write("%s\n"%stripped)
 328         ips_vnode_file.close()
 329
 330         # build a TestPlc object from the result, passing options
 331         for spec in all_plc_specs:
 332             spec['failed_step'] = False
 333         all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs]
 334
 335         # pass options to utils as well
 336         utils.init_options(self.options)
 337
 338         overall_result = True
 339         all_step_infos=[]
 340         for step in self.options.steps:
 341             if not TestPlc.valid_step(step):
 342                 continue
 343             # some steps need to be done regardless of the previous ones: we force them
 344             force=False
 345             if step.find("force_") == 0:
 346                 step=step.replace("force_","")
 347                 force=True
 348             # allow for steps to specify an index like in
 349             # run checkslice@2
 350             try:        (step,qualifier)=step.split('@')
 351             except:     qualifier=self.options.qualifier
 352
 353             try:
 354                 stepobj = Step (step)
 355                 for (substep, method) in stepobj.tuples():
 356                     # a cross step will run a method on TestPlc that has a signature like
 357                     # def cross_foo (self, all_test_plcs)
 358                     cross=False
 359                     if substep.find("cross_") == 0:
 360                         cross=True
 361                     all_step_infos.append ( (substep, method, force, cross, qualifier, ) )
 362             except :
 363                 utils.header("********** FAILED step %s (NOT FOUND) -- won't be run"%step)
 364                 traceback.print_exc()
 365                 overall_result = False
 366
 367         if self.options.dry_run:
 368             self.show_env(self.options,"Dry run")
 369
 370         # init & open trace file if provided
 371         if self.options.trace_file and not self.options.dry_run:
 372             # create dir if needed
 373             trace_dir=os.path.dirname(self.options.trace_file)
 374             if trace_dir and not os.path.isdir(trace_dir):
 375                 os.makedirs(trace_dir)
 376             trace=open(self.options.trace_file,"w")
 377
 378         # do all steps on all plcs
 379         TIME_FORMAT="%H-%M-%S"
 380         TRACE_FORMAT="TRACE: %(plc_counter)d %(beg)s->%(end)s status=%(status)s step=%(stepname)s plc=%(plcname)s force=%(force)s\n"
 381         for (stepname,method,force,cross,qualifier) in all_step_infos:
 382             plc_counter=0
 383             for (spec,plc_obj) in all_plcs:
 384                 plc_counter+=1
 385                 # skip this step if we have specified a plc_explicit
 386                 if qualifier and plc_counter!=int(qualifier): continue
 387
 388                 plcname=spec['name']
 389                 across_plcs = [ o for (s,o) in all_plcs if o!=plc_obj ]
 390
 391                 # run the step
 392                 beg=strftime(TIME_FORMAT)
 393                 if not spec['failed_step'] or force or self.options.interactive or self.options.keep_going:
 394                     skip_step=False
 395                     if self.options.interactive:
 396                         prompting=True
 397                         while prompting:
 398                             msg="%d Run step %s on %s [r](un)/d(ry_run)/p(roceed)/s(kip)/q(uit) ? "%(plc_counter,stepname,plcname)
 399                             answer=raw_input(msg).strip().lower() or "r"
 400                             answer=answer[0]
 401                             if answer in ['s','n']:     # skip/no/next
 402                                 print '%s on %s skipped'%(stepname,plcname)
 403                                 prompting=False
 404                                 skip_step=True
 405                             elif answer in ['q','b']:   # quit/bye
 406                                 print 'Exiting'
 407                                 return
 408                             elif answer in ['d']:       # dry_run
 409                                 dry_run=self.options.dry_run
 410                                 self.options.dry_run=True
 411                                 plc_obj.options.dry_run=True
 412                                 plc_obj.apiserver.set_dry_run(True)
 413                                 if not cross:   step_result=method(plc_obj)
 414                                 else:           step_result=method(plc_obj,across_plcs)
 415                                 print 'dry_run step ->',step_result
 416                                 self.options.dry_run=dry_run
 417                                 plc_obj.options.dry_run=dry_run
 418                                 plc_obj.apiserver.set_dry_run(dry_run)
 419                             elif answer in ['p']:
 420                                 # take it as a yes and leave interactive mode
 421                                 prompting=False
 422                                 self.options.interactive=False
 423                             elif answer in ['r','y']:   # run/yes
 424                                 prompting=False
 425                     if skip_step:
 426                         continue
 427                     try:
 428                         force_msg=""
 429                         if force and spec['failed_step']: force_msg=" (forced after %s has failed)"%spec['failed_step']
 430                         utils.header("********** %d RUNNING step %s%s on plc %s"%(plc_counter,stepname,force_msg,plcname))
 431                         if not cross:   step_result = method(plc_obj)
 432                         else:           step_result = method(plc_obj,across_plcs)
 433                         if step_result:
 434                             utils.header('********** %d SUCCESSFUL step %s on %s'%(plc_counter,stepname,plcname))
 435                             status="OK"
 436                         else:
 437                             overall_result = False
 438                             spec['failed_step'] = stepname
 439                             utils.header('********** %d FAILED Step %s on %s (discarded from further steps)'\
 440                                              %(plc_counter,stepname,plcname))
 441                             status="KO"
 442                     except:
 443                         overall_result=False
 444                         spec['failed_step'] = stepname
 445                         traceback.print_exc()
 446                         utils.header ('********** %d FAILED (exception) Step %s on %s (discarded from further steps)'\
 447                                           %(plc_counter,stepname,plcname))
 448                         status="KO"
 449
 450                 # do not run, just display it's skipped
 451                 else:
 452                     why="has failed %s"%spec['failed_step']
 453                     utils.header("********** %d SKIPPED Step %s on %s (%s)"%(plc_counter,stepname,plcname,why))
 454                     status="UNDEF"
 455                 if not self.options.dry_run:
 456                     end=strftime(TIME_FORMAT)
 457                     # always do this on stdout
 458                     print TRACE_FORMAT%locals()
 459                     # duplicate on trace_file if provided
 460                     if self.options.trace_file:
 461                         trace.write(TRACE_FORMAT%locals())
 462                         trace.flush()
 463
 464         if self.options.trace_file and not self.options.dry_run:
 465             trace.close()
 466
 467         # free local substrate
 468         LocalSubstrate.local_substrate.release(self.options)
 469
 470         return overall_result
 471
 472     # wrapper to run, returns a shell-compatible result
 473     def main(self):
 474         try:
 475             success=self.run()
 476             if success:
 477                 return 0
 478             else:
 479                 return 1
 480         except SystemExit:
 481             print 'Caught SystemExit'
 482             raise
 483         except:
 484             traceback.print_exc()
 485             return 2
 486
 487 if __name__ == "__main__":
 488     sys.exit(TestMain().main())