system/TestMain.py

   1 #!/usr/bin/python -u
   2
   3 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
   4 # Copyright (C) 2010 INRIA
   5 #
   6 import sys, os, os.path
   7 from optparse import OptionParser
   8 import traceback
   9 from time import strftime
  10 import readline
  11 import glob
  12
  13 import utils
  14 from TestPlc import TestPlc
  15 from TestSite import TestSite
  16 from TestNode import TestNode
  17 from macros import sequences
  18
  19 # add $HOME in PYTHONPATH so we can import LocalSubstrate.py
  20 sys.path.append(os.environ['HOME'])
  21 import LocalSubstrate
  22
  23 class Step:
  24
  25     natives=TestPlc.__dict__
  26
  27     def __init__ (self, name):
  28         self.name=name.replace('-','_')
  29         # a native step is implemented as a method on TestPlc
  30         self.native = name in Step.natives
  31         if self.native:
  32             self.method=Step.natives[self.name]
  33         else:
  34             try:
  35                 self.substeps=sequences[self.name]
  36             except Exception,e:
  37                 print "macro step %s not found in macros.py (%s) - exiting"%(self.name,e)
  38                 raise
  39
  40     def norm_name (self): return self.name.replace('_','-')
  41
  42     def print_doc (self):
  43         if self.native:
  44             print '*',self.norm_name(),"\r",4*"\t",
  45             try:
  46                 print self.method.__doc__
  47             except:
  48                 print "*** no doc found"
  49         else:
  50             print '*',self.norm_name(),"\r",3*"\t","========== BEG MACRO step"
  51             for step in self.substeps:
  52                 Step(step).print_doc()
  53             print '*',self.norm_name(),"\r",3*"\t","========== END MACRO step"
  54
  55     # return a list of (name, method) for all native steps involved
  56     def tuples (self):
  57         if self.native: return [ (self.name, self.method,) ]
  58         else:
  59             result=[]
  60             for substep in [ Step(name) for name in self.substeps ] :
  61                 result += substep.tuples()
  62             return result
  63
  64     # convenience for listing macros
  65     # just do a listdir, hoping we're in the right directory...
  66     @staticmethod
  67     def list_macros ():
  68         names= sequences.keys()
  69         names.sort()
  70         return names
  71
  72 class TestMain:
  73
  74     subversion_id = "Now using git -- version tracker broken"
  75
  76     default_config = [ 'default' ]
  77     default_rspec_styles = [ 'pl', 'pg' ]
  78
  79     default_build_url = "git://git.onelab.eu/tests"
  80
  81     def __init__ (self):
  82         self.path=os.path.dirname(sys.argv[0]) or "."
  83         os.chdir(self.path)
  84
  85     def show_env (self,options, message):
  86         if self.options.verbose:
  87             utils.header (message)
  88             utils.show_options("main options",options)
  89
  90     def init_steps(self):
  91         self.steps_message  = 20*'x'+" Defaut steps are\n"+TestPlc.printable_steps(TestPlc.default_steps)
  92         self.steps_message += 20*'x'+" Other useful steps are\n"+TestPlc.printable_steps(TestPlc.other_steps)
  93         self.steps_message += 20*'x'+" Macro steps are\n"+" ".join(Step.list_macros())
  94
  95     def list_steps(self):
  96         if not self.options.verbose:
  97             print self.steps_message,
  98         else:
  99             # steps mentioned on the command line
 100             if self.options.args:
 101                 scopes = [("Argument steps",self.options.args)]
 102             else:
 103                 scopes = [("Default steps",TestPlc.default_steps)]
 104                 if self.options.all_steps:
 105                     scopes.append ( ("Other steps",TestPlc.other_steps) )
 106                     # try to list macro steps as well
 107                     scopes.append ( ("Macro steps", Step.list_macros()) )
 108             for (scope,steps) in scopes:
 109                 print '--------------------',scope
 110                 for step in [step for step in steps if TestPlc.valid_step(step)]:
 111                     try:        (step,qualifier)=step.split('@')
 112                     except:     pass
 113                     stepname=step
 114                     for special in ['force']:
 115                         stepname = stepname.replace(special+'_',"")
 116                     Step(stepname).print_doc()
 117
 118     def run (self):
 119         self.init_steps()
 120         usage = """usage: %%prog [options] steps
 121 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
 122    no default
 123 config defaults to the last value used, as stored in arg-config,
 124    or %r
 125 ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode},
 126    default is to use IP scanning
 127 steps refer to a method in TestPlc or to a step_* module
 128 ===
 129 """%(TestMain.default_config)
 130         usage += self.steps_message
 131         parser=OptionParser(usage=usage,version=self.subversion_id)
 132         parser.add_option("-u","--url",action="store", dest="arch_rpms_url",
 133                           help="URL of the arch-dependent RPMS area - for locating what to test")
 134         parser.add_option("-b","--build",action="store", dest="build_url",
 135                           help="ignored, for legacy only")
 136         parser.add_option("-c","--config",action="append", dest="config", default=[],
 137                           help="Config module - can be set multiple times, or use quotes")
 138         parser.add_option("-p","--personality",action="store", dest="personality",
 139                           help="personality - as in vbuild-nightly")
 140         parser.add_option("-d","--pldistro",action="store", dest="pldistro",
 141                           help="pldistro - as in vbuild-nightly")
 142         parser.add_option("-f","--fcdistro",action="store", dest="fcdistro",
 143                           help="fcdistro - as in vbuild-nightly")
 144         parser.add_option("-e","--exclude",action="append", dest="exclude", default=[],
 145                           help="steps to exclude - can be set multiple times, or use quotes")
 146         parser.add_option("-a","--all",action="store_true",dest="all_steps", default=False,
 147                           help="Run all default steps")
 148         parser.add_option("-l","--list",action="store_true",dest="list_steps", default=False,
 149                           help="List known steps")
 150         parser.add_option("-V","--vserver",action="append", dest="ips_bplc", default=[],
 151                           help="Specify the set of hostnames for the boxes that host the plcs")
 152         parser.add_option("-P","--plcs",action="append", dest="ips_vplc", default=[],
 153                           help="Specify the set of hostname/IP's to use for vplcs")
 154         parser.add_option("-Q","--qemus",action="append", dest="ips_bnode", default=[],
 155                           help="Specify the set of hostnames for the boxes that host the nodes")
 156         parser.add_option("-N","--nodes",action="append", dest="ips_vnode", default=[],
 157                           help="Specify the set of hostname/IP's to use for vnodes")
 158         parser.add_option ('-X', "--lxc",action='store_true',dest='plcs_use_lxc',
 159                            help='use lxc-enabled plc boxes instead of vs-enabled ones')
 160         parser.add_option("-s","--size",action="store",type="int",dest="size",default=1,
 161                           help="sets test size in # of plcs - default is 1")
 162         parser.add_option("-q","--qualifier",action="store",type="int",dest="qualifier",default=None,
 163                           help="run steps only on plc numbered <qualifier>, starting at 1")
 164         parser.add_option("-y","--rspec-style",action="append",dest="rspec_styles",default=[],
 165                           help="pl is for planetlab rspecs, pg is for protogeni")
 166         parser.add_option("-k","--keep-going",action="store",dest="keep_going",default=False,
 167                           help="proceeds even if some steps are failing")
 168         parser.add_option("-D","--dbname",action="store",dest="dbname",default=None,
 169                            help="Used by plc_db_dump and plc_db_restore")
 170         parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
 171                           help="Run in verbose mode")
 172         parser.add_option("-i","--interactive",action="store_true",dest="interactive",default=False,
 173                           help="prompts before each step")
 174         parser.add_option("-n","--dry-run", action="store_true", dest="dry_run", default=False,
 175                           help="Show environment and exits")
 176         parser.add_option("-r","--restart-nm", action="store_true", dest="forcenm", default=False,
 177                           help="Force the NM to restart in ssh_slices step")
 178         parser.add_option("-t","--trace", action="store", dest="trace_file", default=None,
 179                           help="Trace file location")
 180         (self.options, self.args) = parser.parse_args()
 181
 182         # allow things like "run -c 'c1 c2' -c c3"
 183         def flatten (x):
 184             result = []
 185             for el in x:
 186                 if hasattr(el, "__iter__") and not isinstance(el, basestring):
 187                     result.extend(flatten(el))
 188                 else:
 189                     result.append(el)
 190             return result
 191         # flatten relevant options
 192         for optname in ['config','exclude','ips_bplc','ips_vplc','ips_bnode','ips_vnode']:
 193             setattr(self.options,optname, flatten ( [ arg.split() for arg in getattr(self.options,optname) ] ))
 194
 195         if not self.options.rspec_styles:
 196             self.options.rspec_styles=TestMain.default_rspec_styles
 197
 198         # handle defaults and option persistence
 199         for (recname,filename,default,need_reverse) in (
 200             ('build_url','arg-build-url',TestMain.default_build_url,None) ,
 201             ('ips_bplc','arg-ips-bplc',[],True),
 202             ('ips_vplc','arg-ips-vplc',[],True) ,
 203             ('ips_bnode','arg-ips-bnode',[],True),
 204             ('ips_vnode','arg-ips-vnode',[],True) ,
 205             ('config','arg-config',TestMain.default_config,False) ,
 206             ('arch_rpms_url','arg-arch-rpms-url',"",None) ,
 207             ('personality','arg-personality',"linux64",None),
 208             ('pldistro','arg-pldistro',"onelab",None),
 209             ('fcdistro','arg-fcdistro','f14',None),
 210             ) :
 211 #            print 'handling',recname
 212             path=filename
 213             is_list = isinstance(default,list)
 214             if not getattr(self.options,recname):
 215                 try:
 216                     parsed=file(path).readlines()
 217                     if not is_list:    # strings
 218                         if len(parsed) != 1:
 219                             print "%s - error when parsing %s"%(sys.argv[1],path)
 220                             sys.exit(1)
 221                         parsed=parsed[0].strip()
 222                     else:              # lists
 223                         parsed=[x.strip() for x in parsed]
 224                     setattr(self.options,recname,parsed)
 225                 except:
 226                     if default != "":
 227                         setattr(self.options,recname,default)
 228                     else:
 229                         print "Cannot determine",recname
 230                         print "Run %s --help for help"%sys.argv[0]
 231                         sys.exit(1)
 232
 233             # save for next run
 234             fsave=open(path,"w")
 235             if not is_list:
 236                 fsave.write(getattr(self.options,recname) + "\n")
 237             else:
 238                 for value in getattr(self.options,recname):
 239                     fsave.write(value + "\n")
 240             fsave.close()
 241 #            utils.header('Saved %s into %s'%(recname,filename))
 242
 243             # lists need be reversed
 244             # I suspect this is useful for the various pools but for config, it's painful
 245             if isinstance(getattr(self.options,recname),list) and need_reverse:
 246                 getattr(self.options,recname).reverse()
 247
 248             if self.options.verbose:
 249                 utils.header('* Using %s = %s'%(recname,getattr(self.options,recname)))
 250
 251         # hack : if sfa is not among the published rpms, skip these tests
 252         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
 253
 254         # no step specified
 255         self.options.args = self.args
 256         if len(self.args) == 0:
 257             self.options.steps=TestPlc.default_steps
 258         else:
 259             self.options.steps = self.args
 260
 261         if self.options.list_steps:
 262             self.init_steps()
 263             self.list_steps()
 264             return True
 265
 266         # steps
 267         if not self.options.steps:
 268             #default (all) steps
 269             #self.options.steps=['dump','clean','install','populate']
 270             self.options.steps=TestPlc.default_steps
 271
 272         # rewrite '-' into '_' in step names
 273         self.options.steps = [ step.replace('-','_') for step in self.options.steps ]
 274
 275         # exclude
 276         selected=[]
 277         for step in self.options.steps:
 278             keep=True
 279             for exclude in self.options.exclude:
 280                 if utils.match(step,exclude):
 281                     keep=False
 282                     break
 283             if keep: selected.append(step)
 284         self.options.steps=selected
 285
 286         # this is useful when propagating on host boxes, to avoid conflicts
 287         self.options.buildname = os.path.basename (os.path.abspath (self.path))
 288
 289         if self.options.verbose:
 290             self.show_env(self.options,"Verbose")
 291
 292         # load configs
 293         all_plc_specs = []
 294         for config in self.options.config:
 295             modulename='config_'+config
 296             try:
 297                 m = __import__(modulename)
 298                 all_plc_specs = m.config(all_plc_specs,self.options)
 299             except :
 300                 traceback.print_exc()
 301                 print 'Cannot load config %s -- ignored'%modulename
 302                 raise
 303
 304         # provision on local substrate
 305         if self.options.plcs_use_lxc: LocalSubstrate.local_substrate.rescope (plcs_on_vs=False, plcs_on_lxc=True)
 306         all_plc_specs = LocalSubstrate.local_substrate.provision(all_plc_specs,self.options)
 307
 308         # remember substrate IP address(es) for next run
 309         ips_bplc_file=open('arg-ips-bplc','w')
 310         for plc_spec in all_plc_specs:
 311             ips_bplc_file.write("%s\n"%plc_spec['host_box'])
 312         ips_bplc_file.close()
 313         ips_vplc_file=open('arg-ips-vplc','w')
 314         for plc_spec in all_plc_specs:
 315             ips_vplc_file.write("%s\n"%plc_spec['PLC_API_HOST'])
 316         ips_vplc_file.close()
 317         # ditto for nodes
 318         ips_bnode_file=open('arg-ips-bnode','w')
 319         for plc_spec in all_plc_specs:
 320             for site_spec in plc_spec['sites']:
 321                 for node_spec in site_spec['nodes']:
 322                     ips_bnode_file.write("%s\n"%node_spec['host_box'])
 323         ips_bnode_file.close()
 324         ips_vnode_file=open('arg-ips-vnode','w')
 325         for plc_spec in all_plc_specs:
 326             for site_spec in plc_spec['sites']:
 327                 for node_spec in site_spec['nodes']:
 328                     # back to normal (unqualified) form
 329                     stripped=node_spec['node_fields']['hostname'].split('.')[0]
 330                     ips_vnode_file.write("%s\n"%stripped)
 331         ips_vnode_file.close()
 332
 333         # build a TestPlc object from the result, passing options
 334         for spec in all_plc_specs:
 335             spec['failed_step'] = False
 336         all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs]
 337
 338         # pass options to utils as well
 339         utils.init_options(self.options)
 340
 341         overall_result = True
 342         all_step_infos=[]
 343         for step in self.options.steps:
 344             if not TestPlc.valid_step(step):
 345                 continue
 346             # some steps need to be done regardless of the previous ones: we force them
 347             force=False
 348             if step.find("force_") == 0:
 349                 step=step.replace("force_","")
 350                 force=True
 351             # allow for steps to specify an index like in
 352             # run checkslice@2
 353             try:        (step,qualifier)=step.split('@')
 354             except:     qualifier=self.options.qualifier
 355
 356             try:
 357                 stepobj = Step (step)
 358                 for (substep, method) in stepobj.tuples():
 359                     # a cross step will run a method on TestPlc that has a signature like
 360                     # def cross_foo (self, all_test_plcs)
 361                     cross=False
 362                     if substep.find("cross_") == 0:
 363                         cross=True
 364                     all_step_infos.append ( (substep, method, force, cross, qualifier, ) )
 365             except :
 366                 utils.header("********** FAILED step %s (NOT FOUND) -- won't be run"%step)
 367                 traceback.print_exc()
 368                 overall_result = False
 369
 370         if self.options.dry_run:
 371             self.show_env(self.options,"Dry run")
 372
 373         # init & open trace file if provided
 374         if self.options.trace_file and not self.options.dry_run:
 375             # create dir if needed
 376             trace_dir=os.path.dirname(self.options.trace_file)
 377             if trace_dir and not os.path.isdir(trace_dir):
 378                 os.makedirs(trace_dir)
 379             trace=open(self.options.trace_file,"w")
 380
 381         # do all steps on all plcs
 382         TIME_FORMAT="%H-%M-%S"
 383         TRACE_FORMAT="TRACE: %(plc_counter)d %(beg)s->%(end)s status=%(status)s step=%(stepname)s plc=%(plcname)s force=%(force)s\n"
 384         for (stepname,method,force,cross,qualifier) in all_step_infos:
 385             plc_counter=0
 386             for (spec,plc_obj) in all_plcs:
 387                 plc_counter+=1
 388                 # skip this step if we have specified a plc_explicit
 389                 if qualifier and plc_counter!=int(qualifier): continue
 390
 391                 plcname=spec['name']
 392                 across_plcs = [ o for (s,o) in all_plcs if o!=plc_obj ]
 393
 394                 # run the step
 395                 beg=strftime(TIME_FORMAT)
 396                 if not spec['failed_step'] or force or self.options.interactive or self.options.keep_going:
 397                     skip_step=False
 398                     if self.options.interactive:
 399                         prompting=True
 400                         while prompting:
 401                             msg="%d Run step %s on %s [r](un)/d(ry_run)/p(roceed)/s(kip)/q(uit) ? "%(plc_counter,stepname,plcname)
 402                             answer=raw_input(msg).strip().lower() or "r"
 403                             answer=answer[0]
 404                             if answer in ['s','n']:     # skip/no/next
 405                                 print '%s on %s skipped'%(stepname,plcname)
 406                                 prompting=False
 407                                 skip_step=True
 408                             elif answer in ['q','b']:   # quit/bye
 409                                 print 'Exiting'
 410                                 return
 411                             elif answer in ['d']:       # dry_run
 412                                 dry_run=self.options.dry_run
 413                                 self.options.dry_run=True
 414                                 plc_obj.options.dry_run=True
 415                                 plc_obj.apiserver.set_dry_run(True)
 416                                 if not cross:   step_result=method(plc_obj)
 417                                 else:           step_result=method(plc_obj,across_plcs)
 418                                 print 'dry_run step ->',step_result
 419                                 self.options.dry_run=dry_run
 420                                 plc_obj.options.dry_run=dry_run
 421                                 plc_obj.apiserver.set_dry_run(dry_run)
 422                             elif answer in ['p']:
 423                                 # take it as a yes and leave interactive mode
 424                                 prompting=False
 425                                 self.options.interactive=False
 426                             elif answer in ['r','y']:   # run/yes
 427                                 prompting=False
 428                     if skip_step:
 429                         continue
 430                     try:
 431                         force_msg=""
 432                         if force and spec['failed_step']: force_msg=" (forced after %s has failed)"%spec['failed_step']
 433                         utils.header("********** %d RUNNING step %s%s on plc %s"%(plc_counter,stepname,force_msg,plcname))
 434                         if not cross:   step_result = method(plc_obj)
 435                         else:           step_result = method(plc_obj,across_plcs)
 436                         if step_result:
 437                             utils.header('********** %d SUCCESSFUL step %s on %s'%(plc_counter,stepname,plcname))
 438                             status="OK"
 439                         else:
 440                             overall_result = False
 441                             spec['failed_step'] = stepname
 442                             utils.header('********** %d FAILED Step %s on %s (discarded from further steps)'\
 443                                              %(plc_counter,stepname,plcname))
 444                             status="KO"
 445                     except:
 446                         overall_result=False
 447                         spec['failed_step'] = stepname
 448                         traceback.print_exc()
 449                         utils.header ('********** %d FAILED (exception) Step %s on %s (discarded from further steps)'\
 450                                           %(plc_counter,stepname,plcname))
 451                         status="KO"
 452
 453                 # do not run, just display it's skipped
 454                 else:
 455                     why="has failed %s"%spec['failed_step']
 456                     utils.header("********** %d SKIPPED Step %s on %s (%s)"%(plc_counter,stepname,plcname,why))
 457                     status="UNDEF"
 458                 if not self.options.dry_run:
 459                     end=strftime(TIME_FORMAT)
 460                     # always do this on stdout
 461                     print TRACE_FORMAT%locals()
 462                     # duplicate on trace_file if provided
 463                     if self.options.trace_file:
 464                         trace.write(TRACE_FORMAT%locals())
 465                         trace.flush()
 466
 467         if self.options.trace_file and not self.options.dry_run:
 468             trace.close()
 469
 470         # free local substrate
 471         LocalSubstrate.local_substrate.release(self.options)
 472
 473         return overall_result
 474
 475     # wrapper to run, returns a shell-compatible result
 476     def main(self):
 477         try:
 478             success=self.run()
 479             if success:
 480                 return 0
 481             else:
 482                 return 1
 483         except SystemExit:
 484             print 'Caught SystemExit'
 485             raise
 486         except:
 487             traceback.print_exc()
 488             return 2
 489
 490 if __name__ == "__main__":
 491     sys.exit(TestMain().main())