system/TestMain.py

   1 #!/usr/bin/python -u
   2
   3 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
   4 # Copyright (C) 2010 INRIA
   5 #
   6 import sys, os, os.path
   7 from optparse import OptionParser
   8 import traceback
   9 from time import strftime
  10 import readline
  11 import glob
  12
  13 import utils
  14 from TestPlc import TestPlc
  15 from TestSite import TestSite
  16 from TestNode import TestNode
  17
  18 # add $HOME in PYTHONPATH so we can import LocalSubstrate.py
  19 sys.path.append(os.environ['HOME'])
  20 import LocalSubstrate
  21
  22 class Step:
  23
  24     natives=TestPlc.__dict__
  25
  26     def __init__ (self, name):
  27         self.name=name.replace('-','_')
  28         # a native step is implemented as a method on TestPlc
  29         self.native = name in Step.natives
  30         if self.native:
  31             self.method=Step.natives[self.name]
  32         else:
  33             # non-native steps (macros) are implemented as a 'Step'
  34             try:
  35                 modulename = 'macro_' + self.name
  36                 module = __import__ (modulename)
  37                 self.substeps = module.sequence
  38             except Exception,e:
  39                 print "Cannot load macro step %s (%s) - exiting"%(self.name,e)
  40                 raise
  41
  42     def norm_name (self): return self.name.replace('_','-')
  43
  44     def print_doc (self):
  45         if self.native:
  46             print '*',self.norm_name(),"\r",4*"\t",
  47             try:
  48                 print self.method.__doc__
  49             except:
  50                 print "*** no doc found"
  51         else:
  52             print '*',self.norm_name(),"\r",3*"\t","========== BEG MACRO step"
  53             for step in self.substeps:
  54                 Step(step).print_doc()
  55             print '*',self.norm_name(),"\r",3*"\t","========== END MACRO step"
  56
  57     # return a list of (name, method) for all native steps involved
  58     def tuples (self):
  59         if self.native: return [ (self.name, self.method,) ]
  60         else:
  61             result=[]
  62             for substep in [ Step(name) for name in self.substeps ] :
  63                 result += substep.tuples()
  64             return result
  65
  66     # convenience for listing macros
  67     # just do a listdir, hoping we're in the right directory...
  68     @staticmethod
  69     def list_macros ():
  70         names= [ filename.replace('macro_','').replace('.py','') for filename in glob.glob ('macro_*.py')]
  71         names.sort()
  72         return names
  73
  74 class TestMain:
  75
  76     subversion_id = "Now using git -- version tracker broken"
  77
  78     default_config = [ 'default' ]
  79     default_rspec_styles = [ 'pl', 'pg' ]
  80
  81     default_build_url = "git://git.onelab.eu/tests"
  82
  83     def __init__ (self):
  84         self.path=os.path.dirname(sys.argv[0]) or "."
  85         os.chdir(self.path)
  86
  87     def show_env (self,options, message):
  88         if self.options.verbose:
  89             utils.header (message)
  90             utils.show_options("main options",options)
  91
  92     def init_steps(self):
  93         self.steps_message  = 20*'x'+" Defaut steps are\n"+TestPlc.printable_steps(TestPlc.default_steps)
  94         self.steps_message += 20*'x'+" Other useful steps are\n"+TestPlc.printable_steps(TestPlc.other_steps)
  95         self.steps_message += 20*'x'+" Macro steps are\n"+" ".join(Step.list_macros())
  96
  97     def list_steps(self):
  98         if not self.options.verbose:
  99             print self.steps_message,
 100         else:
 101             # steps mentioned on the command line
 102             if self.options.args:
 103                 scopes = [("Argument steps",self.options.args)]
 104             else:
 105                 scopes = [("Default steps",TestPlc.default_steps)]
 106                 if self.options.all_steps:
 107                     scopes.append ( ("Other steps",TestPlc.other_steps) )
 108                     # try to list macro steps as well
 109                     scopes.append ( ("Macro steps", Step.list_macros()) )
 110             for (scope,steps) in scopes:
 111                 print '--------------------',scope
 112                 for step in [step for step in steps if TestPlc.valid_step(step)]:
 113                     try:        (step,qualifier)=step.split('@')
 114                     except:     pass
 115                     stepname=step
 116                     for special in ['force']:
 117                         stepname = stepname.replace(special+'_',"")
 118                     Step(stepname).print_doc()
 119
 120     def run (self):
 121         self.init_steps()
 122         usage = """usage: %%prog [options] steps
 123 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
 124    no default
 125 config defaults to the last value used, as stored in arg-config,
 126    or %r
 127 ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode},
 128    default is to use IP scanning
 129 steps refer to a method in TestPlc or to a step_* module
 130 ===
 131 """%(TestMain.default_config)
 132         usage += self.steps_message
 133         parser=OptionParser(usage=usage,version=self.subversion_id)
 134         parser.add_option("-u","--url",action="store", dest="arch_rpms_url",
 135                           help="URL of the arch-dependent RPMS area - for locating what to test")
 136         parser.add_option("-b","--build",action="store", dest="build_url",
 137                           help="ignored, for legacy only")
 138         parser.add_option("-c","--config",action="append", dest="config", default=[],
 139                           help="Config module - can be set multiple times, or use quotes")
 140         parser.add_option("-p","--personality",action="store", dest="personality",
 141                           help="personality - as in vbuild-nightly")
 142         parser.add_option("-d","--pldistro",action="store", dest="pldistro",
 143                           help="pldistro - as in vbuild-nightly")
 144         parser.add_option("-f","--fcdistro",action="store", dest="fcdistro",
 145                           help="fcdistro - as in vbuild-nightly")
 146         parser.add_option("-x","--exclude",action="append", dest="exclude", default=[],
 147                           help="steps to exclude - can be set multiple times, or use quotes")
 148         parser.add_option("-a","--all",action="store_true",dest="all_steps", default=False,
 149                           help="Run all default steps")
 150         parser.add_option("-l","--list",action="store_true",dest="list_steps", default=False,
 151                           help="List known steps")
 152         parser.add_option("-V","--vserver",action="append", dest="ips_bplc", default=[],
 153                           help="Specify the set of hostnames for the boxes that host the plcs")
 154         parser.add_option("-P","--plcs",action="append", dest="ips_vplc", default=[],
 155                           help="Specify the set of hostname/IP's to use for vplcs")
 156         parser.add_option("-Q","--qemus",action="append", dest="ips_bnode", default=[],
 157                           help="Specify the set of hostnames for the boxes that host the nodes")
 158         parser.add_option("-N","--nodes",action="append", dest="ips_vnode", default=[],
 159                           help="Specify the set of hostname/IP's to use for vnodes")
 160         parser.add_option("-s","--size",action="store",type="int",dest="size",default=1,
 161                           help="sets test size in # of plcs - default is 1")
 162         parser.add_option("-q","--qualifier",action="store",type="int",dest="qualifier",default=None,
 163                           help="run steps only on plc numbered <qualifier>, starting at 1")
 164         parser.add_option("-y","--rspec-style",action="append",dest="rspec_styles",default=[],
 165                           help="pl is for planetlab rspecs, pg is for protogeni")
 166         parser.add_option("-k","--keep-going",action="store",dest="keep_going",default=False,
 167                           help="proceeds even if some steps are failing")
 168         parser.add_option("-D","--dbname",action="store",dest="dbname",default=None,
 169                            help="Used by plc_db_dump and plc_db_restore")
 170         parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
 171                           help="Run in verbose mode")
 172         parser.add_option("-i","--interactive",action="store_true",dest="interactive",default=False,
 173                           help="prompts before each step")
 174         parser.add_option("-n","--dry-run", action="store_true", dest="dry_run", default=False,
 175                           help="Show environment and exits")
 176         parser.add_option("-r","--restart-nm", action="store_true", dest="forcenm", default=False,
 177                           help="Force the NM to restart in ssh_slices step")
 178         parser.add_option("-t","--trace", action="store", dest="trace_file", default=None,
 179                           help="Trace file location")
 180         (self.options, self.args) = parser.parse_args()
 181
 182         # allow things like "run -c 'c1 c2' -c c3"
 183         def flatten (x):
 184             result = []
 185             for el in x:
 186                 if hasattr(el, "__iter__") and not isinstance(el, basestring):
 187                     result.extend(flatten(el))
 188                 else:
 189                     result.append(el)
 190             return result
 191         # flatten relevant options
 192         for optname in ['config','exclude','ips_bplc','ips_vplc','ips_bnode','ips_vnode']:
 193             setattr(self.options,optname, flatten ( [ arg.split() for arg in getattr(self.options,optname) ] ))
 194
 195         if not self.options.rspec_styles:
 196             self.options.rspec_styles=TestMain.default_rspec_styles
 197
 198         # handle defaults and option persistence
 199         for (recname,filename,default,need_reverse) in (
 200             ('build_url','arg-build-url',TestMain.default_build_url,None) ,
 201             ('ips_bplc','arg-ips-bplc',[],True),
 202             ('ips_vplc','arg-ips-vplc',[],True) ,
 203             ('ips_bnode','arg-ips-bnode',[],True),
 204             ('ips_vnode','arg-ips-vnode',[],True) ,
 205             ('config','arg-config',TestMain.default_config,False) ,
 206             ('arch_rpms_url','arg-arch-rpms-url',"",None) ,
 207             ('personality','arg-personality',"linux64",None),
 208             ('pldistro','arg-pldistro',"onelab",None),
 209             ('fcdistro','arg-fcdistro','f14',None),
 210             ) :
 211 #            print 'handling',recname
 212             path=filename
 213             is_list = isinstance(default,list)
 214             if not getattr(self.options,recname):
 215                 try:
 216                     parsed=file(path).readlines()
 217                     if not is_list:    # strings
 218                         if len(parsed) != 1:
 219                             print "%s - error when parsing %s"%(sys.argv[1],path)
 220                             sys.exit(1)
 221                         parsed=parsed[0].strip()
 222                     else:              # lists
 223                         parsed=[x.strip() for x in parsed]
 224                     setattr(self.options,recname,parsed)
 225                 except:
 226                     if default != "":
 227                         setattr(self.options,recname,default)
 228                     else:
 229                         print "Cannot determine",recname
 230                         print "Run %s --help for help"%sys.argv[0]
 231                         sys.exit(1)
 232
 233             # save for next run
 234             fsave=open(path,"w")
 235             if not is_list:
 236                 fsave.write(getattr(self.options,recname) + "\n")
 237             else:
 238                 for value in getattr(self.options,recname):
 239                     fsave.write(value + "\n")
 240             fsave.close()
 241 #            utils.header('Saved %s into %s'%(recname,filename))
 242
 243             # lists need be reversed
 244             # I suspect this is useful for the various pools but for config, it's painful
 245             if isinstance(getattr(self.options,recname),list) and need_reverse:
 246                 getattr(self.options,recname).reverse()
 247
 248             if self.options.verbose:
 249                 utils.header('* Using %s = %s'%(recname,getattr(self.options,recname)))
 250
 251         # hack : if sfa is not among the published rpms, skip these tests
 252         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
 253
 254         # no step specified
 255         self.options.args = self.args
 256         if len(self.args) == 0:
 257             self.options.steps=TestPlc.default_steps
 258         else:
 259             self.options.steps = self.args
 260
 261         if self.options.list_steps:
 262             self.init_steps()
 263             self.list_steps()
 264             return True
 265
 266         # steps
 267         if not self.options.steps:
 268             #default (all) steps
 269             #self.options.steps=['dump','clean','install','populate']
 270             self.options.steps=TestPlc.default_steps
 271
 272         # rewrite '-' into '_' in step names
 273         self.options.steps = [ step.replace('-','_') for step in self.options.steps ]
 274
 275         # exclude
 276         selected=[]
 277         for step in self.options.steps:
 278             keep=True
 279             for exclude in self.options.exclude:
 280                 if utils.match(step,exclude):
 281                     keep=False
 282                     break
 283             if keep: selected.append(step)
 284         self.options.steps=selected
 285
 286         # this is useful when propagating on host boxes, to avoid conflicts
 287         self.options.buildname = os.path.basename (os.path.abspath (self.path))
 288
 289         if self.options.verbose:
 290             self.show_env(self.options,"Verbose")
 291
 292         # load configs
 293         all_plc_specs = []
 294         for config in self.options.config:
 295             modulename='config_'+config
 296             try:
 297                 m = __import__(modulename)
 298                 all_plc_specs = m.config(all_plc_specs,self.options)
 299             except :
 300                 traceback.print_exc()
 301                 print 'Cannot load config %s -- ignored'%modulename
 302                 raise
 303
 304         # provision on local substrate
 305         all_plc_specs = LocalSubstrate.local_substrate.provision(all_plc_specs,self.options)
 306
 307         # remember substrate IP address(es) for next run
 308         ips_bplc_file=open('arg-ips-bplc','w')
 309         for plc_spec in all_plc_specs:
 310             ips_bplc_file.write("%s\n"%plc_spec['host_box'])
 311         ips_bplc_file.close()
 312         ips_vplc_file=open('arg-ips-vplc','w')
 313         for plc_spec in all_plc_specs:
 314             ips_vplc_file.write("%s\n"%plc_spec['PLC_API_HOST'])
 315         ips_vplc_file.close()
 316         # ditto for nodes
 317         ips_bnode_file=open('arg-ips-bnode','w')
 318         for plc_spec in all_plc_specs:
 319             for site_spec in plc_spec['sites']:
 320                 for node_spec in site_spec['nodes']:
 321                     ips_bnode_file.write("%s\n"%node_spec['host_box'])
 322         ips_bnode_file.close()
 323         ips_vnode_file=open('arg-ips-vnode','w')
 324         for plc_spec in all_plc_specs:
 325             for site_spec in plc_spec['sites']:
 326                 for node_spec in site_spec['nodes']:
 327                     # back to normal (unqualified) form
 328                     stripped=node_spec['node_fields']['hostname'].split('.')[0]
 329                     ips_vnode_file.write("%s\n"%stripped)
 330         ips_vnode_file.close()
 331
 332         # build a TestPlc object from the result, passing options
 333         for spec in all_plc_specs:
 334             spec['failed_step'] = False
 335         all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs]
 336
 337         # pass options to utils as well
 338         utils.init_options(self.options)
 339
 340         overall_result = True
 341         testplc_method_dict = __import__("TestPlc").__dict__['TestPlc'].__dict__
 342         all_step_infos=[]
 343         for step in self.options.steps:
 344             if not TestPlc.valid_step(step):
 345                 continue
 346             # some steps need to be done regardless of the previous ones: we force them
 347             force=False
 348             if step.find("force_") == 0:
 349                 step=step.replace("force_","")
 350                 force=True
 351             # a cross step will run a method on TestPlc that has a signature like
 352             # def cross_foo (self, all_test_plcs)
 353             cross=False
 354             if step.find("cross_") == 0:
 355                 cross=True
 356             # allow for steps to specify an index like in
 357             # run checkslice@2
 358             try:        (step,qualifier)=step.split('@')
 359             except:     qualifier=self.options.qualifier
 360
 361             try:
 362                 stepobj = Step (step)
 363                 for (substep, method) in stepobj.tuples():
 364                     all_step_infos.append ( (substep, method, force, cross, qualifier, ) )
 365             except :
 366                 utils.header("********** FAILED step %s (NOT FOUND) -- won't be run"%step)
 367                 traceback.print_exc()
 368                 overall_result = False
 369
 370         if self.options.dry_run:
 371             self.show_env(self.options,"Dry run")
 372
 373         # init & open trace file if provided
 374         if self.options.trace_file and not self.options.dry_run:
 375             # create dir if needed
 376             trace_dir=os.path.dirname(self.options.trace_file)
 377             if trace_dir and not os.path.isdir(trace_dir):
 378                 os.makedirs(trace_dir)
 379             trace=open(self.options.trace_file,"w")
 380
 381         # do all steps on all plcs
 382         TIME_FORMAT="%H-%M-%S"
 383         TRACE_FORMAT="TRACE: %(plc_counter)d %(beg)s->%(end)s status=%(status)s step=%(stepname)s plc=%(plcname)s force=%(force)s\n"
 384         for (stepname,method,force,cross,qualifier) in all_step_infos:
 385             plc_counter=0
 386             for (spec,plc_obj) in all_plcs:
 387                 plc_counter+=1
 388                 # skip this step if we have specified a plc_explicit
 389                 if qualifier and plc_counter!=int(qualifier): continue
 390
 391                 plcname=spec['name']
 392                 across_plcs = [ o for (s,o) in all_plcs if o!=plc_obj ]
 393
 394                 # run the step
 395                 beg=strftime(TIME_FORMAT)
 396                 if not spec['failed_step'] or force or self.options.interactive or self.options.keep_going:
 397                     skip_step=False
 398                     if self.options.interactive:
 399                         prompting=True
 400                         while prompting:
 401                             msg="%d Run step %s on %s [r](un)/d(ry_run)/p(roceed)/s(kip)/q(uit) ? "%(plc_counter,stepname,plcname)
 402                             answer=raw_input(msg).strip().lower() or "r"
 403                             answer=answer[0]
 404                             if answer in ['s','n']:     # skip/no/next
 405                                 print '%s on %s skipped'%(stepname,plcname)
 406                                 prompting=False
 407                                 skip_step=True
 408                             elif answer in ['q','b']:   # quit/bye
 409                                 print 'Exiting'
 410                                 return
 411                             elif answer in ['d']:       # dry_run
 412                                 dry_run=self.options.dry_run
 413                                 self.options.dry_run=True
 414                                 plc_obj.options.dry_run=True
 415                                 plc_obj.apiserver.set_dry_run(True)
 416                                 if not cross:   step_result=method(plc_obj)
 417                                 else:           step_result=method(plc_obj,across_plcs)
 418                                 print 'dry_run step ->',step_result
 419                                 self.options.dry_run=dry_run
 420                                 plc_obj.options.dry_run=dry_run
 421                                 plc_obj.apiserver.set_dry_run(dry_run)
 422                             elif answer in ['p']:
 423                                 # take it as a yes and leave interactive mode
 424                                 prompting=False
 425                                 self.options.interactive=False
 426                             elif answer in ['r','y']:   # run/yes
 427                                 prompting=False
 428                     if skip_step:
 429                         continue
 430                     try:
 431                         force_msg=""
 432                         if force and spec['failed_step']: force_msg=" (forced after %s has failed)"%spec['failed_step']
 433                         utils.header("********** %d RUNNING step %s%s on plc %s"%(plc_counter,stepname,force_msg,plcname))
 434                         if not cross:   step_result = method(plc_obj)
 435                         else:           step_result = method(plc_obj,across_plcs)
 436                         if step_result:
 437                             utils.header('********** %d SUCCESSFUL step %s on %s'%(plc_counter,stepname,plcname))
 438                             status="OK"
 439                         else:
 440                             overall_result = False
 441                             spec['failed_step'] = stepname
 442                             utils.header('********** %d FAILED Step %s on %s (discarded from further steps)'\
 443                                              %(plc_counter,stepname,plcname))
 444                             status="KO"
 445                     except:
 446                         overall_result=False
 447                         spec['failed_step'] = stepname
 448                         traceback.print_exc()
 449                         utils.header ('********** %d FAILED (exception) Step %s on %s (discarded from further steps)'\
 450                                           %(plc_counter,stepname,plcname))
 451                         status="KO"
 452
 453                 # do not run, just display it's skipped
 454                 else:
 455                     why="has failed %s"%spec['failed_step']
 456                     utils.header("********** %d SKIPPED Step %s on %s (%s)"%(plc_counter,stepname,plcname,why))
 457                     status="UNDEF"
 458                 if not self.options.dry_run:
 459                     end=strftime(TIME_FORMAT)
 460                     # always do this on stdout
 461                     print TRACE_FORMAT%locals()
 462                     # duplicate on trace_file if provided
 463                     if self.options.trace_file:
 464                         trace.write(TRACE_FORMAT%locals())
 465                         trace.flush()
 466
 467         if self.options.trace_file and not self.options.dry_run:
 468             trace.close()
 469
 470         # free local substrate
 471         LocalSubstrate.local_substrate.release(self.options)
 472
 473         return overall_result
 474
 475     # wrapper to run, returns a shell-compatible result
 476     def main(self):
 477         try:
 478             success=self.run()
 479             if success:
 480                 return 0
 481             else:
 482                 return 1
 483         except SystemExit:
 484             print 'Caught SystemExit'
 485             raise
 486         except:
 487             traceback.print_exc()
 488             return 2
 489
 490 if __name__ == "__main__":
 491     sys.exit(TestMain().main())