system/TestMain.py

   1 #!/usr/bin/python -u
   2
   3 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
   4 # Copyright (C) 2010 INRIA
   5 #
   6 import sys, os, os.path
   7 from optparse import OptionParser
   8 import traceback
   9 from time import strftime
  10 import readline
  11 import glob
  12
  13 import utils
  14 from TestPlc import TestPlc
  15 from TestSite import TestSite
  16 from TestNode import TestNode
  17 from macros import sequences
  18
  19 # add $HOME in PYTHONPATH so we can import LocalSubstrate.py
  20 sys.path.append(os.environ['HOME'])
  21 import LocalSubstrate
  22
  23 class Step:
  24
  25     natives=TestPlc.__dict__
  26
  27     def __init__ (self, name):
  28         self.name=name.replace('-','_')
  29         # a native step is implemented as a method on TestPlc
  30         self.native = name in Step.natives
  31         if self.native:
  32             self.method=Step.natives[self.name]
  33         else:
  34             try:
  35                 self.substeps=sequences[self.name]
  36             except Exception,e:
  37                 print "macro step %s not found in macros.py (%s) - exiting"%(self.name,e)
  38                 raise
  39
  40     def norm_name (self): return self.name.replace('_','-')
  41
  42     def print_doc (self):
  43         if self.native:
  44             print '*',self.norm_name(),"\r",4*"\t",
  45             try:
  46                 print self.method.__doc__
  47             except:
  48                 print "*** no doc found"
  49         else:
  50             print '*',self.norm_name(),"\r",3*"\t","========== BEG MACRO step"
  51             for step in self.substeps:
  52                 Step(step).print_doc()
  53             print '*',self.norm_name(),"\r",3*"\t","========== END MACRO step"
  54
  55     # return a list of (name, method) for all native steps involved
  56     def tuples (self):
  57         if self.native: return [ (self.name, self.method,) ]
  58         else:
  59             result=[]
  60             for substep in [ Step(name) for name in self.substeps ] :
  61                 result += substep.tuples()
  62             return result
  63
  64     # convenience for listing macros
  65     # just do a listdir, hoping we're in the right directory...
  66     @staticmethod
  67     def list_macros ():
  68         names= sequences.keys()
  69         names.sort()
  70         return names
  71
  72 class TestMain:
  73
  74     subversion_id = "Now using git -- version tracker broken"
  75
  76     default_config = [ 'default' ]
  77     default_rspec_styles = [ 'pl', 'pg' ]
  78
  79     default_build_url = "git://git.onelab.eu/tests"
  80
  81     def __init__ (self):
  82         self.path=os.path.dirname(sys.argv[0]) or "."
  83         os.chdir(self.path)
  84
  85     def show_env (self,options, message):
  86         if self.options.verbose:
  87             utils.header (message)
  88             utils.show_options("main options",options)
  89
  90     def init_steps(self):
  91         self.steps_message  = 20*'x'+" Defaut steps are\n"+TestPlc.printable_steps(TestPlc.default_steps)
  92         self.steps_message += 20*'x'+" Other useful steps are\n"+TestPlc.printable_steps(TestPlc.other_steps)
  93         self.steps_message += 20*'x'+" Macro steps are\n"+" ".join(Step.list_macros())
  94
  95     def list_steps(self):
  96         if not self.options.verbose:
  97             print self.steps_message,
  98         else:
  99             # steps mentioned on the command line
 100             if self.options.args:
 101                 scopes = [("Argument steps",self.options.args)]
 102             else:
 103                 scopes = [("Default steps",TestPlc.default_steps)]
 104                 if self.options.all_steps:
 105                     scopes.append ( ("Other steps",TestPlc.other_steps) )
 106                     # try to list macro steps as well
 107                     scopes.append ( ("Macro steps", Step.list_macros()) )
 108             for (scope,steps) in scopes:
 109                 print '--------------------',scope
 110                 for step in [step for step in steps if TestPlc.valid_step(step)]:
 111                     try:        (step,qualifier)=step.split('@')
 112                     except:     pass
 113                     stepname=step
 114                     for special in ['force']:
 115                         stepname = stepname.replace(special+'_',"")
 116                     Step(stepname).print_doc()
 117
 118     def run (self):
 119         self.init_steps()
 120         usage = """usage: %%prog [options] steps
 121 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
 122    no default
 123 config defaults to the last value used, as stored in arg-config,
 124    or %r
 125 ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode},
 126    default is to use IP scanning
 127 steps refer to a method in TestPlc or to a step_* module
 128 ===
 129 """%(TestMain.default_config)
 130         usage += self.steps_message
 131         parser=OptionParser(usage=usage,version=self.subversion_id)
 132         parser.add_option("-u","--url",action="store", dest="arch_rpms_url",
 133                           help="URL of the arch-dependent RPMS area - for locating what to test")
 134         parser.add_option("-b","--build",action="store", dest="build_url",
 135                           help="ignored, for legacy only")
 136         parser.add_option("-c","--config",action="append", dest="config", default=[],
 137                           help="Config module - can be set multiple times, or use quotes")
 138         parser.add_option("-p","--personality",action="store", dest="personality",
 139                           help="personality - as in vbuild-nightly")
 140         parser.add_option("-d","--pldistro",action="store", dest="pldistro",
 141                           help="pldistro - as in vbuild-nightly")
 142         parser.add_option("-f","--fcdistro",action="store", dest="fcdistro",
 143                           help="fcdistro - as in vbuild-nightly")
 144         parser.add_option("-e","--exclude",action="append", dest="exclude", default=[],
 145                           help="steps to exclude - can be set multiple times, or use quotes")
 146         parser.add_option("-a","--all",action="store_true",dest="all_steps", default=False,
 147                           help="Run all default steps")
 148         parser.add_option("-l","--list",action="store_true",dest="list_steps", default=False,
 149                           help="List known steps")
 150         parser.add_option("-V","--vserver",action="append", dest="ips_bplc", default=[],
 151                           help="Specify the set of hostnames for the boxes that host the plcs")
 152         parser.add_option("-P","--plcs",action="append", dest="ips_vplc", default=[],
 153                           help="Specify the set of hostname/IP's to use for vplcs")
 154         parser.add_option("-Q","--qemus",action="append", dest="ips_bnode", default=[],
 155                           help="Specify the set of hostnames for the boxes that host the nodes")
 156         parser.add_option("-N","--nodes",action="append", dest="ips_vnode", default=[],
 157                           help="Specify the set of hostname/IP's to use for vnodes")
 158         parser.add_option ('-X', "--lxc",action='store_true',dest='plcs_use_lxc',
 159                            help='use lxc-enabled plc boxes instead of vs-enabled ones')
 160         parser.add_option("-s","--size",action="store",type="int",dest="size",default=1,
 161                           help="sets test size in # of plcs - default is 1")
 162         parser.add_option("-q","--qualifier",action="store",type="int",dest="qualifier",default=None,
 163                           help="run steps only on plc numbered <qualifier>, starting at 1")
 164         parser.add_option("-y","--rspec-style",action="append",dest="rspec_styles",default=[],
 165                           help="pl is for planetlab rspecs, pg is for protogeni")
 166         parser.add_option("-k","--keep-going",action="store",dest="keep_going",default=False,
 167                           help="proceeds even if some steps are failing")
 168         parser.add_option("-D","--dbname",action="store",dest="dbname",default=None,
 169                            help="Used by plc_db_dump and plc_db_restore")
 170         parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
 171                           help="Run in verbose mode")
 172         parser.add_option("-i","--interactive",action="store_true",dest="interactive",default=False,
 173                           help="prompts before each step")
 174         parser.add_option("-n","--dry-run", action="store_true", dest="dry_run", default=False,
 175                           help="Show environment and exits")
 176         parser.add_option("-r","--restart-nm", action="store_true", dest="forcenm", default=False,
 177                           help="Force the NM to restart in ssh_slices step")
 178         parser.add_option("-t","--trace", action="store", dest="trace_file", default=None,
 179                           help="Trace file location")
 180         (self.options, self.args) = parser.parse_args()
 181
 182         # allow things like "run -c 'c1 c2' -c c3"
 183         def flatten (x):
 184             result = []
 185             for el in x:
 186                 if hasattr(el, "__iter__") and not isinstance(el, basestring):
 187                     result.extend(flatten(el))
 188                 else:
 189                     result.append(el)
 190             return result
 191         # flatten relevant options
 192         for optname in ['config','exclude','ips_bplc','ips_vplc','ips_bnode','ips_vnode']:
 193             setattr(self.options,optname, flatten ( [ arg.split() for arg in getattr(self.options,optname) ] ))
 194
 195         if not self.options.rspec_styles:
 196             self.options.rspec_styles=TestMain.default_rspec_styles
 197
 198         # handle defaults and option persistence
 199         for (recname,filename,default,need_reverse) in (
 200             ('build_url','arg-build-url',TestMain.default_build_url,None) ,
 201             ('ips_bplc','arg-ips-bplc',[],True),
 202             ('ips_vplc','arg-ips-vplc',[],True) ,
 203             ('ips_bnode','arg-ips-bnode',[],True),
 204             ('ips_vnode','arg-ips-vnode',[],True) ,
 205             ('config','arg-config',TestMain.default_config,False) ,
 206             ('arch_rpms_url','arg-arch-rpms-url',"",None) ,
 207             ('personality','arg-personality',"linux64",None),
 208             ('pldistro','arg-pldistro',"onelab",None),
 209             ('fcdistro','arg-fcdistro','f14',None),
 210             ('plcs_use_lxc','arg-plcs-use-lxc',False,None),
 211             ) :
 212 #            print 'handling',recname
 213             path=filename
 214             is_list = isinstance(default,list)
 215             is_bool = isinstance(default,bool)
 216             if not getattr(self.options,recname):
 217                 try:
 218                     parsed=file(path).readlines()
 219                     if is_list:         # lists
 220                         parsed=[x.strip() for x in parsed]
 221                     else:               # strings and booleans
 222                         if len(parsed) != 1:
 223                             print "%s - error when parsing %s"%(sys.argv[1],path)
 224                             sys.exit(1)
 225                         parsed=parsed[0].strip()
 226                         if is_bool:
 227                             parsed = parsed.lower()=='true'
 228                     setattr(self.options,recname,parsed)
 229                 except:
 230                     if default != "":
 231                         setattr(self.options,recname,default)
 232                     else:
 233                         print "Cannot determine",recname
 234                         print "Run %s --help for help"%sys.argv[0]
 235                         sys.exit(1)
 236
 237             # save for next run
 238             fsave=open(path,"w")
 239             if is_list:                 # lists
 240                 for value in getattr(self.options,recname):
 241                     fsave.write(value + "\n")
 242             else:                       # strings and booleans - just call str()
 243                 fsave.write(str(getattr(self.options,recname)) + "\n")
 244             fsave.close()
 245 #            utils.header('Saved %s into %s'%(recname,filename))
 246
 247             # lists need be reversed
 248             # I suspect this is useful for the various pools but for config, it's painful
 249             if isinstance(getattr(self.options,recname),list) and need_reverse:
 250                 getattr(self.options,recname).reverse()
 251
 252             if self.options.verbose:
 253                 utils.header('* Using %s = %s'%(recname,getattr(self.options,recname)))
 254
 255         # hack : if sfa is not among the published rpms, skip these tests
 256         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
 257
 258         # no step specified
 259         self.options.args = self.args
 260         if len(self.args) == 0:
 261             self.options.steps=TestPlc.default_steps
 262         else:
 263             self.options.steps = self.args
 264
 265         if self.options.list_steps:
 266             self.init_steps()
 267             self.list_steps()
 268             return True
 269
 270         # steps
 271         if not self.options.steps:
 272             #default (all) steps
 273             #self.options.steps=['dump','clean','install','populate']
 274             self.options.steps=TestPlc.default_steps
 275
 276         # rewrite '-' into '_' in step names
 277         self.options.steps = [ step.replace('-','_') for step in self.options.steps ]
 278
 279         # exclude
 280         selected=[]
 281         for step in self.options.steps:
 282             keep=True
 283             for exclude in self.options.exclude:
 284                 if utils.match(step,exclude):
 285                     keep=False
 286                     break
 287             if keep: selected.append(step)
 288         self.options.steps=selected
 289
 290         # this is useful when propagating on host boxes, to avoid conflicts
 291         self.options.buildname = os.path.basename (os.path.abspath (self.path))
 292
 293         if self.options.verbose:
 294             self.show_env(self.options,"Verbose")
 295
 296         # load configs
 297         all_plc_specs = []
 298         for config in self.options.config:
 299             modulename='config_'+config
 300             try:
 301                 m = __import__(modulename)
 302                 all_plc_specs = m.config(all_plc_specs,self.options)
 303             except :
 304                 traceback.print_exc()
 305                 print 'Cannot load config %s -- ignored'%modulename
 306                 raise
 307
 308         # provision on local substrate
 309         if self.options.plcs_use_lxc: LocalSubstrate.local_substrate.rescope (plcs_on_vs=False, plcs_on_lxc=True)
 310         all_plc_specs = LocalSubstrate.local_substrate.provision(all_plc_specs,self.options)
 311
 312         # remember substrate IP address(es) for next run
 313         ips_bplc_file=open('arg-ips-bplc','w')
 314         for plc_spec in all_plc_specs:
 315             ips_bplc_file.write("%s\n"%plc_spec['host_box'])
 316         ips_bplc_file.close()
 317         ips_vplc_file=open('arg-ips-vplc','w')
 318         for plc_spec in all_plc_specs:
 319             ips_vplc_file.write("%s\n"%plc_spec['PLC_API_HOST'])
 320         ips_vplc_file.close()
 321         # ditto for nodes
 322         ips_bnode_file=open('arg-ips-bnode','w')
 323         for plc_spec in all_plc_specs:
 324             for site_spec in plc_spec['sites']:
 325                 for node_spec in site_spec['nodes']:
 326                     ips_bnode_file.write("%s\n"%node_spec['host_box'])
 327         ips_bnode_file.close()
 328         ips_vnode_file=open('arg-ips-vnode','w')
 329         for plc_spec in all_plc_specs:
 330             for site_spec in plc_spec['sites']:
 331                 for node_spec in site_spec['nodes']:
 332                     # back to normal (unqualified) form
 333                     stripped=node_spec['node_fields']['hostname'].split('.')[0]
 334                     ips_vnode_file.write("%s\n"%stripped)
 335         ips_vnode_file.close()
 336
 337         # build a TestPlc object from the result, passing options
 338         for spec in all_plc_specs:
 339             spec['failed_step'] = False
 340         all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs]
 341
 342         # pass options to utils as well
 343         utils.init_options(self.options)
 344
 345         overall_result = True
 346         all_step_infos=[]
 347         for step in self.options.steps:
 348             if not TestPlc.valid_step(step):
 349                 continue
 350             # some steps need to be done regardless of the previous ones: we force them
 351             force=False
 352             if step.find("force_") == 0:
 353                 step=step.replace("force_","")
 354                 force=True
 355             # allow for steps to specify an index like in
 356             # run checkslice@2
 357             try:        (step,qualifier)=step.split('@')
 358             except:     qualifier=self.options.qualifier
 359
 360             try:
 361                 stepobj = Step (step)
 362                 for (substep, method) in stepobj.tuples():
 363                     # a cross step will run a method on TestPlc that has a signature like
 364                     # def cross_foo (self, all_test_plcs)
 365                     cross=False
 366                     if substep.find("cross_") == 0:
 367                         cross=True
 368                     all_step_infos.append ( (substep, method, force, cross, qualifier, ) )
 369             except :
 370                 utils.header("********** FAILED step %s (NOT FOUND) -- won't be run"%step)
 371                 traceback.print_exc()
 372                 overall_result = False
 373
 374         if self.options.dry_run:
 375             self.show_env(self.options,"Dry run")
 376
 377         # init & open trace file if provided
 378         if self.options.trace_file and not self.options.dry_run:
 379             # create dir if needed
 380             trace_dir=os.path.dirname(self.options.trace_file)
 381             if trace_dir and not os.path.isdir(trace_dir):
 382                 os.makedirs(trace_dir)
 383             trace=open(self.options.trace_file,"w")
 384
 385         # do all steps on all plcs
 386         TIME_FORMAT="%H-%M-%S"
 387         TRACE_FORMAT="TRACE: %(plc_counter)d %(beg)s->%(end)s status=%(status)s step=%(stepname)s plc=%(plcname)s force=%(force)s\n"
 388         for (stepname,method,force,cross,qualifier) in all_step_infos:
 389             plc_counter=0
 390             for (spec,plc_obj) in all_plcs:
 391                 plc_counter+=1
 392                 # skip this step if we have specified a plc_explicit
 393                 if qualifier and plc_counter!=int(qualifier): continue
 394
 395                 plcname=spec['name']
 396                 across_plcs = [ o for (s,o) in all_plcs if o!=plc_obj ]
 397
 398                 # run the step
 399                 beg=strftime(TIME_FORMAT)
 400                 if not spec['failed_step'] or force or self.options.interactive or self.options.keep_going:
 401                     skip_step=False
 402                     if self.options.interactive:
 403                         prompting=True
 404                         while prompting:
 405                             msg="%d Run step %s on %s [r](un)/d(ry_run)/p(roceed)/s(kip)/q(uit) ? "%(plc_counter,stepname,plcname)
 406                             answer=raw_input(msg).strip().lower() or "r"
 407                             answer=answer[0]
 408                             if answer in ['s','n']:     # skip/no/next
 409                                 print '%s on %s skipped'%(stepname,plcname)
 410                                 prompting=False
 411                                 skip_step=True
 412                             elif answer in ['q','b']:   # quit/bye
 413                                 print 'Exiting'
 414                                 return
 415                             elif answer in ['d']:       # dry_run
 416                                 dry_run=self.options.dry_run
 417                                 self.options.dry_run=True
 418                                 plc_obj.options.dry_run=True
 419                                 plc_obj.apiserver.set_dry_run(True)
 420                                 if not cross:   step_result=method(plc_obj)
 421                                 else:           step_result=method(plc_obj,across_plcs)
 422                                 print 'dry_run step ->',step_result
 423                                 self.options.dry_run=dry_run
 424                                 plc_obj.options.dry_run=dry_run
 425                                 plc_obj.apiserver.set_dry_run(dry_run)
 426                             elif answer in ['p']:
 427                                 # take it as a yes and leave interactive mode
 428                                 prompting=False
 429                                 self.options.interactive=False
 430                             elif answer in ['r','y']:   # run/yes
 431                                 prompting=False
 432                     if skip_step:
 433                         continue
 434                     try:
 435                         force_msg=""
 436                         if force and spec['failed_step']: force_msg=" (forced after %s has failed)"%spec['failed_step']
 437                         utils.header("********** %d RUNNING step %s%s on plc %s"%(plc_counter,stepname,force_msg,plcname))
 438                         if not cross:   step_result = method(plc_obj)
 439                         else:           step_result = method(plc_obj,across_plcs)
 440                         if step_result:
 441                             utils.header('********** %d SUCCESSFUL step %s on %s'%(plc_counter,stepname,plcname))
 442                             status="OK"
 443                         else:
 444                             overall_result = False
 445                             spec['failed_step'] = stepname
 446                             utils.header('********** %d FAILED Step %s on %s (discarded from further steps)'\
 447                                              %(plc_counter,stepname,plcname))
 448                             status="KO"
 449                     except:
 450                         overall_result=False
 451                         spec['failed_step'] = stepname
 452                         traceback.print_exc()
 453                         utils.header ('********** %d FAILED (exception) Step %s on %s (discarded from further steps)'\
 454                                           %(plc_counter,stepname,plcname))
 455                         status="KO"
 456
 457                 # do not run, just display it's skipped
 458                 else:
 459                     why="has failed %s"%spec['failed_step']
 460                     utils.header("********** %d SKIPPED Step %s on %s (%s)"%(plc_counter,stepname,plcname,why))
 461                     status="UNDEF"
 462                 if not self.options.dry_run:
 463                     end=strftime(TIME_FORMAT)
 464                     # always do this on stdout
 465                     print TRACE_FORMAT%locals()
 466                     # duplicate on trace_file if provided
 467                     if self.options.trace_file:
 468                         trace.write(TRACE_FORMAT%locals())
 469                         trace.flush()
 470
 471         if self.options.trace_file and not self.options.dry_run:
 472             trace.close()
 473
 474         # free local substrate
 475         LocalSubstrate.local_substrate.release(self.options)
 476
 477         return overall_result
 478
 479     # wrapper to run, returns a shell-compatible result
 480     def main(self):
 481         try:
 482             success=self.run()
 483             if success:
 484                 return 0
 485             else:
 486                 return 1
 487         except SystemExit:
 488             print 'Caught SystemExit'
 489             raise
 490         except:
 491             traceback.print_exc()
 492             return 2
 493
 494 if __name__ == "__main__":
 495     sys.exit(TestMain().main())