system/TestMain.py

   1 #!/usr/bin/python -u
   2
   3 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
   4 # Copyright (C) 2010 INRIA
   5 #
   6 import sys, os, os.path
   7 from optparse import OptionParser
   8 import traceback
   9 from time import strftime
  10 import readline
  11 import glob
  12
  13 import utils
  14 from TestPlc import TestPlc
  15 from TestSite import TestSite
  16 from TestNode import TestNode
  17
  18 # add $HOME in PYTHONPATH so we can import LocalSubstrate.py
  19 sys.path.append(os.environ['HOME'])
  20 import LocalSubstrate
  21
  22 class Step:
  23
  24     natives=TestPlc.__dict__
  25
  26     def __init__ (self, name):
  27         self.name=name.replace('-','_')
  28         # a native step is implemented as a method on TestPlc
  29         self.native = name in Step.natives
  30         if self.native:
  31             self.method=Step.natives[self.name]
  32         else:
  33             # non-native steps (macros) are implemented as a 'Step'
  34             try:
  35                 modulename = 'macro_' + self.name
  36                 module = __import__ (modulename)
  37                 self.substeps = module.sequence
  38             except Exception,e:
  39                 print "Cannot load macro step %s (%s) - exiting"%(self.name,e)
  40                 raise
  41
  42     def norm_name (self): return self.name.replace('_','-')
  43
  44     def print_doc (self):
  45         if self.native:
  46             print '*',self.norm_name(),"\r",4*"\t",
  47             try:
  48                 print self.method.__doc__
  49             except:
  50                 print "*** no doc found"
  51         else:
  52             print '*',self.norm_name(),"\r",3*"\t","========== BEG MACRO step"
  53             for step in self.substeps:
  54                 Step(step).print_doc()
  55             print '*',self.norm_name(),"\r",3*"\t","========== END MACRO step"
  56
  57     # return a list of (name, method) for all native steps involved
  58     def tuples (self):
  59         if self.native: return [ (self.name, self.method,) ]
  60         else:
  61             result=[]
  62             for substep in [ Step(name) for name in self.substeps ] :
  63                 result += substep.tuples()
  64             return result
  65
  66     # convenience for listing macros
  67     # just do a listdir, hoping we're in the right directory...
  68     @staticmethod
  69     def list_macros ():
  70         names= [ filename.replace('macro_','').replace('.py','') for filename in glob.glob ('macro_*.py')]
  71         names.sort()
  72         return names
  73
  74 class TestMain:
  75
  76     subversion_id = "Now using git -- version tracker broken"
  77
  78     default_config = [ 'default' ]
  79     default_rspec_styles = [ 'pl', 'pg' ]
  80
  81     default_build_url = "git://git.onelab.eu/tests"
  82
  83     def __init__ (self):
  84         self.path=os.path.dirname(sys.argv[0]) or "."
  85         os.chdir(self.path)
  86
  87     def show_env (self,options, message):
  88         if self.options.verbose:
  89             utils.header (message)
  90             utils.show_options("main options",options)
  91
  92     def init_steps(self):
  93         self.steps_message  = 20*'x'+" Defaut steps are\n"+TestPlc.printable_steps(TestPlc.default_steps)
  94         self.steps_message += 20*'x'+" Other useful steps are\n"+TestPlc.printable_steps(TestPlc.other_steps)
  95         self.steps_message += 20*'x'+" Macro steps are\n"+" ".join(Step.list_macros())
  96
  97     def list_steps(self):
  98         if not self.options.verbose:
  99             print self.steps_message,
 100         else:
 101             # steps mentioned on the command line
 102             if self.options.args:
 103                 scopes = [("Argument steps",self.options.args)]
 104             else:
 105                 scopes = [("Default steps",TestPlc.default_steps)]
 106                 if self.options.all_steps:
 107                     scopes.append ( ("Other steps",TestPlc.other_steps) )
 108                     # try to list macro steps as well
 109                     scopes.append ( ("Macro steps", Step.list_macros()) )
 110             for (scope,steps) in scopes:
 111                 print '--------------------',scope
 112                 for step in [step for step in steps if TestPlc.valid_step(step)]:
 113                     try:        (step,qualifier)=step.split('@')
 114                     except:     pass
 115                     stepname=step
 116                     for special in ['force']:
 117                         stepname = stepname.replace(special+'_',"")
 118                     Step(stepname).print_doc()
 119
 120     def run (self):
 121         self.init_steps()
 122         usage = """usage: %%prog [options] steps
 123 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
 124    no default
 125 config defaults to the last value used, as stored in arg-config,
 126    or %r
 127 ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode},
 128    default is to use IP scanning
 129 steps refer to a method in TestPlc or to a step_* module
 130 ===
 131 """%(TestMain.default_config)
 132         usage += self.steps_message
 133         parser=OptionParser(usage=usage,version=self.subversion_id)
 134         parser.add_option("-u","--url",action="store", dest="arch_rpms_url",
 135                           help="URL of the arch-dependent RPMS area - for locating what to test")
 136         parser.add_option("-b","--build",action="store", dest="build_url",
 137                           help="ignored, for legacy only")
 138         parser.add_option("-c","--config",action="append", dest="config", default=[],
 139                           help="Config module - can be set multiple times, or use quotes")
 140         parser.add_option("-p","--personality",action="store", dest="personality",
 141                           help="personality - as in vbuild-nightly")
 142         parser.add_option("-d","--pldistro",action="store", dest="pldistro",
 143                           help="pldistro - as in vbuild-nightly")
 144         parser.add_option("-f","--fcdistro",action="store", dest="fcdistro",
 145                           help="fcdistro - as in vbuild-nightly")
 146         parser.add_option("-x","--exclude",action="append", dest="exclude", default=[],
 147                           help="steps to exclude - can be set multiple times, or use quotes")
 148         parser.add_option("-a","--all",action="store_true",dest="all_steps", default=False,
 149                           help="Run all default steps")
 150         parser.add_option("-l","--list",action="store_true",dest="list_steps", default=False,
 151                           help="List known steps")
 152         parser.add_option("-V","--vserver",action="append", dest="ips_bplc", default=[],
 153                           help="Specify the set of hostnames for the boxes that host the plcs")
 154         parser.add_option("-P","--plcs",action="append", dest="ips_vplc", default=[],
 155                           help="Specify the set of hostname/IP's to use for vplcs")
 156         parser.add_option("-Q","--qemus",action="append", dest="ips_bnode", default=[],
 157                           help="Specify the set of hostnames for the boxes that host the nodes")
 158         parser.add_option("-N","--nodes",action="append", dest="ips_vnode", default=[],
 159                           help="Specify the set of hostname/IP's to use for vnodes")
 160         parser.add_option("-s","--size",action="store",type="int",dest="size",default=1,
 161                           help="sets test size in # of plcs - default is 1")
 162         parser.add_option("-q","--qualifier",action="store",type="int",dest="qualifier",default=None,
 163                           help="run steps only on plc numbered <qualifier>, starting at 1")
 164         parser.add_option("-y","--rspec-style",action="append",dest="rspec_styles",default=[],
 165                           help="pl is for planetlab rspecs, pg is for protogeni")
 166         parser.add_option("-k","--keep-going",action="store",dest="keep_going",default=False,
 167                           help="proceeds even if some steps are failing")
 168         parser.add_option("-D","--dbname",action="store",dest="dbname",default=None,
 169                            help="Used by plc_db_dump and plc_db_restore")
 170         parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
 171                           help="Run in verbose mode")
 172         parser.add_option("-i","--interactive",action="store_true",dest="interactive",default=False,
 173                           help="prompts before each step")
 174         parser.add_option("-n","--dry-run", action="store_true", dest="dry_run", default=False,
 175                           help="Show environment and exits")
 176         parser.add_option("-r","--restart-nm", action="store_true", dest="forcenm", default=False,
 177                           help="Force the NM to restart in ssh_slices step")
 178         parser.add_option("-t","--trace", action="store", dest="trace_file", default=None,
 179                           help="Trace file location")
 180         (self.options, self.args) = parser.parse_args()
 181
 182         # allow things like "run -c 'c1 c2' -c c3"
 183         def flatten (x):
 184             result = []
 185             for el in x:
 186                 if hasattr(el, "__iter__") and not isinstance(el, basestring):
 187                     result.extend(flatten(el))
 188                 else:
 189                     result.append(el)
 190             return result
 191         # flatten relevant options
 192         for optname in ['config','exclude','ips_bplc','ips_vplc','ips_bnode','ips_vnode']:
 193             setattr(self.options,optname, flatten ( [ arg.split() for arg in getattr(self.options,optname) ] ))
 194
 195         if not self.options.rspec_styles:
 196             self.options.rspec_styles=TestMain.default_rspec_styles
 197
 198         # handle defaults and option persistence
 199         for (recname,filename,default,need_reverse) in (
 200             ('build_url','arg-build-url',TestMain.default_build_url,None) ,
 201             ('ips_bplc','arg-ips-bplc',[],True),
 202             ('ips_vplc','arg-ips-vplc',[],True) ,
 203             ('ips_bnode','arg-ips-bnode',[],True),
 204             ('ips_vnode','arg-ips-vnode',[],True) ,
 205             ('config','arg-config',TestMain.default_config,False) ,
 206             ('arch_rpms_url','arg-arch-rpms-url',"",None) ,
 207             ('personality','arg-personality',"linux64",None),
 208             ('pldistro','arg-pldistro',"onelab",None),
 209             ('fcdistro','arg-fcdistro','f14',None),
 210             ) :
 211 #            print 'handling',recname
 212             path=filename
 213             is_list = isinstance(default,list)
 214             if not getattr(self.options,recname):
 215                 try:
 216                     parsed=file(path).readlines()
 217                     if not is_list:    # strings
 218                         if len(parsed) != 1:
 219                             print "%s - error when parsing %s"%(sys.argv[1],path)
 220                             sys.exit(1)
 221                         parsed=parsed[0].strip()
 222                     else:              # lists
 223                         parsed=[x.strip() for x in parsed]
 224                     setattr(self.options,recname,parsed)
 225                 except:
 226                     if default != "":
 227                         setattr(self.options,recname,default)
 228                     else:
 229                         print "Cannot determine",recname
 230                         print "Run %s --help for help"%sys.argv[0]
 231                         sys.exit(1)
 232
 233             # save for next run
 234             fsave=open(path,"w")
 235             if not is_list:
 236                 fsave.write(getattr(self.options,recname) + "\n")
 237             else:
 238                 for value in getattr(self.options,recname):
 239                     fsave.write(value + "\n")
 240             fsave.close()
 241 #            utils.header('Saved %s into %s'%(recname,filename))
 242
 243             # lists need be reversed
 244             # I suspect this is useful for the various pools but for config, it's painful
 245             if isinstance(getattr(self.options,recname),list) and need_reverse:
 246                 getattr(self.options,recname).reverse()
 247
 248             if self.options.verbose:
 249                 utils.header('* Using %s = %s'%(recname,getattr(self.options,recname)))
 250
 251         # hack : if sfa is not among the published rpms, skip these tests
 252         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
 253
 254         # no step specified
 255         self.options.args = self.args
 256         if len(self.args) == 0:
 257             self.options.steps=TestPlc.default_steps
 258         else:
 259             self.options.steps = self.args
 260
 261         if self.options.list_steps:
 262             self.init_steps()
 263             self.list_steps()
 264             return True
 265
 266         # steps
 267         if not self.options.steps:
 268             #default (all) steps
 269             #self.options.steps=['dump','clean','install','populate']
 270             self.options.steps=TestPlc.default_steps
 271
 272         # rewrite '-' into '_' in step names
 273         self.options.steps = [ step.replace('-','_') for step in self.options.steps ]
 274
 275         # exclude
 276         selected=[]
 277         for step in self.options.steps:
 278             keep=True
 279             for exclude in self.options.exclude:
 280                 if utils.match(step,exclude):
 281                     keep=False
 282                     break
 283             if keep: selected.append(step)
 284         self.options.steps=selected
 285
 286         # this is useful when propagating on host boxes, to avoid conflicts
 287         self.options.buildname = os.path.basename (os.path.abspath (self.path))
 288
 289         if self.options.verbose:
 290             self.show_env(self.options,"Verbose")
 291
 292         # load configs
 293         all_plc_specs = []
 294         for config in self.options.config:
 295             modulename='config_'+config
 296             try:
 297                 m = __import__(modulename)
 298                 all_plc_specs = m.config(all_plc_specs,self.options)
 299             except :
 300                 traceback.print_exc()
 301                 print 'Cannot load config %s -- ignored'%modulename
 302                 raise
 303
 304         # provision on local substrate
 305         all_plc_specs = LocalSubstrate.local_substrate.provision(all_plc_specs,self.options)
 306
 307         # remember substrate IP address(es) for next run
 308         ips_bplc_file=open('arg-ips-bplc','w')
 309         for plc_spec in all_plc_specs:
 310             ips_bplc_file.write("%s\n"%plc_spec['host_box'])
 311         ips_bplc_file.close()
 312         ips_vplc_file=open('arg-ips-vplc','w')
 313         for plc_spec in all_plc_specs:
 314             ips_vplc_file.write("%s\n"%plc_spec['PLC_API_HOST'])
 315         ips_vplc_file.close()
 316         # ditto for nodes
 317         ips_bnode_file=open('arg-ips-bnode','w')
 318         for plc_spec in all_plc_specs:
 319             for site_spec in plc_spec['sites']:
 320                 for node_spec in site_spec['nodes']:
 321                     ips_bnode_file.write("%s\n"%node_spec['host_box'])
 322         ips_bnode_file.close()
 323         ips_vnode_file=open('arg-ips-vnode','w')
 324         for plc_spec in all_plc_specs:
 325             for site_spec in plc_spec['sites']:
 326                 for node_spec in site_spec['nodes']:
 327                     # back to normal (unqualified) form
 328                     stripped=node_spec['node_fields']['hostname'].split('.')[0]
 329                     ips_vnode_file.write("%s\n"%stripped)
 330         ips_vnode_file.close()
 331
 332         # build a TestPlc object from the result, passing options
 333         for spec in all_plc_specs:
 334             spec['failed_step'] = False
 335         all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs]
 336
 337         # pass options to utils as well
 338         utils.init_options(self.options)
 339
 340         overall_result = True
 341         all_step_infos=[]
 342         for step in self.options.steps:
 343             if not TestPlc.valid_step(step):
 344                 continue
 345             # some steps need to be done regardless of the previous ones: we force them
 346             force=False
 347             if step.find("force_") == 0:
 348                 step=step.replace("force_","")
 349                 force=True
 350             # allow for steps to specify an index like in
 351             # run checkslice@2
 352             try:        (step,qualifier)=step.split('@')
 353             except:     qualifier=self.options.qualifier
 354
 355             try:
 356                 stepobj = Step (step)
 357                 for (substep, method) in stepobj.tuples():
 358                     # a cross step will run a method on TestPlc that has a signature like
 359                     # def cross_foo (self, all_test_plcs)
 360                     cross=False
 361                     if substep.find("cross_") == 0:
 362                         cross=True
 363                     all_step_infos.append ( (substep, method, force, cross, qualifier, ) )
 364             except :
 365                 utils.header("********** FAILED step %s (NOT FOUND) -- won't be run"%step)
 366                 traceback.print_exc()
 367                 overall_result = False
 368
 369         if self.options.dry_run:
 370             self.show_env(self.options,"Dry run")
 371
 372         # init & open trace file if provided
 373         if self.options.trace_file and not self.options.dry_run:
 374             # create dir if needed
 375             trace_dir=os.path.dirname(self.options.trace_file)
 376             if trace_dir and not os.path.isdir(trace_dir):
 377                 os.makedirs(trace_dir)
 378             trace=open(self.options.trace_file,"w")
 379
 380         # do all steps on all plcs
 381         TIME_FORMAT="%H-%M-%S"
 382         TRACE_FORMAT="TRACE: %(plc_counter)d %(beg)s->%(end)s status=%(status)s step=%(stepname)s plc=%(plcname)s force=%(force)s\n"
 383         for (stepname,method,force,cross,qualifier) in all_step_infos:
 384             plc_counter=0
 385             for (spec,plc_obj) in all_plcs:
 386                 plc_counter+=1
 387                 # skip this step if we have specified a plc_explicit
 388                 if qualifier and plc_counter!=int(qualifier): continue
 389
 390                 plcname=spec['name']
 391                 across_plcs = [ o for (s,o) in all_plcs if o!=plc_obj ]
 392
 393                 # run the step
 394                 beg=strftime(TIME_FORMAT)
 395                 if not spec['failed_step'] or force or self.options.interactive or self.options.keep_going:
 396                     skip_step=False
 397                     if self.options.interactive:
 398                         prompting=True
 399                         while prompting:
 400                             msg="%d Run step %s on %s [r](un)/d(ry_run)/p(roceed)/s(kip)/q(uit) ? "%(plc_counter,stepname,plcname)
 401                             answer=raw_input(msg).strip().lower() or "r"
 402                             answer=answer[0]
 403                             if answer in ['s','n']:     # skip/no/next
 404                                 print '%s on %s skipped'%(stepname,plcname)
 405                                 prompting=False
 406                                 skip_step=True
 407                             elif answer in ['q','b']:   # quit/bye
 408                                 print 'Exiting'
 409                                 return
 410                             elif answer in ['d']:       # dry_run
 411                                 dry_run=self.options.dry_run
 412                                 self.options.dry_run=True
 413                                 plc_obj.options.dry_run=True
 414                                 plc_obj.apiserver.set_dry_run(True)
 415                                 if not cross:   step_result=method(plc_obj)
 416                                 else:           step_result=method(plc_obj,across_plcs)
 417                                 print 'dry_run step ->',step_result
 418                                 self.options.dry_run=dry_run
 419                                 plc_obj.options.dry_run=dry_run
 420                                 plc_obj.apiserver.set_dry_run(dry_run)
 421                             elif answer in ['p']:
 422                                 # take it as a yes and leave interactive mode
 423                                 prompting=False
 424                                 self.options.interactive=False
 425                             elif answer in ['r','y']:   # run/yes
 426                                 prompting=False
 427                     if skip_step:
 428                         continue
 429                     try:
 430                         force_msg=""
 431                         if force and spec['failed_step']: force_msg=" (forced after %s has failed)"%spec['failed_step']
 432                         utils.header("********** %d RUNNING step %s%s on plc %s"%(plc_counter,stepname,force_msg,plcname))
 433                         if not cross:   step_result = method(plc_obj)
 434                         else:           step_result = method(plc_obj,across_plcs)
 435                         if step_result:
 436                             utils.header('********** %d SUCCESSFUL step %s on %s'%(plc_counter,stepname,plcname))
 437                             status="OK"
 438                         else:
 439                             overall_result = False
 440                             spec['failed_step'] = stepname
 441                             utils.header('********** %d FAILED Step %s on %s (discarded from further steps)'\
 442                                              %(plc_counter,stepname,plcname))
 443                             status="KO"
 444                     except:
 445                         overall_result=False
 446                         spec['failed_step'] = stepname
 447                         traceback.print_exc()
 448                         utils.header ('********** %d FAILED (exception) Step %s on %s (discarded from further steps)'\
 449                                           %(plc_counter,stepname,plcname))
 450                         status="KO"
 451
 452                 # do not run, just display it's skipped
 453                 else:
 454                     why="has failed %s"%spec['failed_step']
 455                     utils.header("********** %d SKIPPED Step %s on %s (%s)"%(plc_counter,stepname,plcname,why))
 456                     status="UNDEF"
 457                 if not self.options.dry_run:
 458                     end=strftime(TIME_FORMAT)
 459                     # always do this on stdout
 460                     print TRACE_FORMAT%locals()
 461                     # duplicate on trace_file if provided
 462                     if self.options.trace_file:
 463                         trace.write(TRACE_FORMAT%locals())
 464                         trace.flush()
 465
 466         if self.options.trace_file and not self.options.dry_run:
 467             trace.close()
 468
 469         # free local substrate
 470         LocalSubstrate.local_substrate.release(self.options)
 471
 472         return overall_result
 473
 474     # wrapper to run, returns a shell-compatible result
 475     def main(self):
 476         try:
 477             success=self.run()
 478             if success:
 479                 return 0
 480             else:
 481                 return 1
 482         except SystemExit:
 483             print 'Caught SystemExit'
 484             raise
 485         except:
 486             traceback.print_exc()
 487             return 2
 488
 489 if __name__ == "__main__":
 490     sys.exit(TestMain().main())