system/TestMain.py

   1 #!/usr/bin/python -u
   2
   3 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
   4 # Copyright (C) 2010 INRIA
   5 #
   6 import sys, os, os.path
   7 from optparse import OptionParser
   8 import traceback
   9 from time import strftime
  10 import readline
  11 import glob
  12
  13 import utils
  14 from TestPlc import TestPlc
  15 from TestSite import TestSite
  16 from TestNode import TestNode
  17 from macros import sequences
  18
  19 # add $HOME in PYTHONPATH so we can import LocalSubstrate.py
  20 sys.path.append(os.environ['HOME'])
  21 import LocalSubstrate
  22
  23 class Step:
  24
  25     natives=TestPlc.__dict__
  26
  27     def display (self): return self.name.replace('_','-')
  28     def internal (self): return self.name.replace('-','_')
  29
  30     def __init__ (self, name):
  31         self.name=name
  32         # a native step is implemented as a method on TestPlc
  33         self.native = self.internal() in Step.natives
  34         if self.native:
  35             self.method=Step.natives[self.internal()]
  36         else:
  37             try:
  38                 self.substeps=sequences[self.internal()]
  39             except Exception,e:
  40                 print "macro step %s not found in macros.py (%s) - exiting"%(self.display(),e)
  41                 raise
  42
  43     def print_doc (self,level=0):
  44         tab=32
  45         trail=8
  46         if self.native:
  47             start=level*' '+'* '
  48             # 2 is the len of '* '
  49             width=tab-level-2
  50             format="%%-%ds"%width
  51             line=start+format%self.display()
  52             print line,
  53             try:
  54                 print self.method.__doc__
  55             except:
  56                 print "*** no doc found"
  57         else:
  58             beg_start=level*' '+'>>> '
  59             end_start=level*' '+'<<< '
  60             trailer=trail*'-'
  61             # 4 is the len of '>>> '
  62             width=tab-level-4-trail
  63             format=("%%-%ds"%width)
  64             beg_line=beg_start+format%self.display()+trail*'>'
  65             end_line=end_start+format%self.display()+trail*'<'
  66             print beg_line
  67             for step in self.substeps:
  68                 Step(step).print_doc(level+1)
  69             print end_line
  70
  71     # return a list of (name, method) for all native steps involved
  72     def tuples (self):
  73         if self.native: return [ (self.internal(), self.method,) ]
  74         else:
  75             result=[]
  76             for substep in [ Step(name) for name in self.substeps ] :
  77                 result += substep.tuples()
  78             return result
  79
  80     # convenience for listing macros
  81     # just do a listdir, hoping we're in the right directory...
  82     @staticmethod
  83     def list_macros ():
  84         names= sequences.keys()
  85         names.sort()
  86         return names
  87
  88 class TestMain:
  89
  90     subversion_id = "Now using git -- version tracker broken"
  91
  92     default_config = [ 'default' ]
  93     default_rspec_styles = [ 'pl', 'pg' ]
  94
  95     default_build_url = "git://git.onelab.eu/tests"
  96
  97     def __init__ (self):
  98         self.path=os.path.dirname(sys.argv[0]) or "."
  99         os.chdir(self.path)
 100
 101     def show_env (self,options, message):
 102         if self.options.verbose:
 103             utils.header (message)
 104             utils.show_options("main options",options)
 105
 106     def init_steps(self):
 107         self.steps_message  = 20*'x'+" Defaut steps are\n"+TestPlc.printable_steps(TestPlc.default_steps)
 108         self.steps_message += 20*'x'+" Other useful steps are\n"+TestPlc.printable_steps(TestPlc.other_steps)
 109         self.steps_message += 20*'x'+" Macro steps are\n"+" ".join(Step.list_macros())
 110
 111     def list_steps(self):
 112         if not self.options.verbose:
 113             print self.steps_message,
 114         else:
 115             # steps mentioned on the command line
 116             if self.options.args:
 117                 scopes = [("Argument steps",self.options.args)]
 118             else:
 119                 scopes = [("Default steps",TestPlc.default_steps)]
 120                 if self.options.all_steps:
 121                     scopes.append ( ("Other steps",TestPlc.other_steps) )
 122                     # try to list macro steps as well
 123                     scopes.append ( ("Macro steps", Step.list_macros()) )
 124             for (scope,steps) in scopes:
 125                 print '--------------------',scope
 126                 for step in [step for step in steps if TestPlc.valid_step(step)]:
 127                     try:        (step,qualifier)=step.split('@')
 128                     except:     pass
 129                     stepname=step
 130                     for special in ['force']:
 131                         stepname = stepname.replace(special+'_',"")
 132                     Step(stepname).print_doc()
 133
 134     def run (self):
 135         self.init_steps()
 136         usage = """usage: %%prog [options] steps
 137 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
 138    no default
 139 config defaults to the last value used, as stored in arg-config,
 140    or %r
 141 ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode},
 142    default is to use IP scanning
 143 steps refer to a method in TestPlc or to a step_* module
 144 ===
 145 """%(TestMain.default_config)
 146         usage += self.steps_message
 147         parser=OptionParser(usage=usage,version=self.subversion_id)
 148         parser.add_option("-u","--url",action="store", dest="arch_rpms_url",
 149                           help="URL of the arch-dependent RPMS area - for locating what to test")
 150         parser.add_option("-b","--build",action="store", dest="build_url",
 151                           help="ignored, for legacy only")
 152         parser.add_option("-c","--config",action="append", dest="config", default=[],
 153                           help="Config module - can be set multiple times, or use quotes")
 154         parser.add_option("-p","--personality",action="store", dest="personality",
 155                           help="personality - as in vbuild-nightly")
 156         parser.add_option("-d","--pldistro",action="store", dest="pldistro",
 157                           help="pldistro - as in vbuild-nightly")
 158         parser.add_option("-f","--fcdistro",action="store", dest="fcdistro",
 159                           help="fcdistro - as in vbuild-nightly")
 160         parser.add_option("-e","--exclude",action="append", dest="exclude", default=[],
 161                           help="steps to exclude - can be set multiple times, or use quotes")
 162         parser.add_option("-a","--all",action="store_true",dest="all_steps", default=False,
 163                           help="Run all default steps")
 164         parser.add_option("-l","--list",action="store_true",dest="list_steps", default=False,
 165                           help="List known steps")
 166         parser.add_option("-V","--vserver",action="append", dest="ips_bplc", default=[],
 167                           help="Specify the set of hostnames for the boxes that host the plcs")
 168         parser.add_option("-P","--plcs",action="append", dest="ips_vplc", default=[],
 169                           help="Specify the set of hostname/IP's to use for vplcs")
 170         parser.add_option("-Q","--qemus",action="append", dest="ips_bnode", default=[],
 171                           help="Specify the set of hostnames for the boxes that host the nodes")
 172         parser.add_option("-N","--nodes",action="append", dest="ips_vnode", default=[],
 173                           help="Specify the set of hostname/IP's to use for vnodes")
 174         parser.add_option ('-X', "--lxc",action='store_true',dest='plcs_use_lxc',
 175                            help='use lxc-enabled plc boxes instead of vs-enabled ones')
 176         parser.add_option("-s","--size",action="store",type="int",dest="size",default=1,
 177                           help="sets test size in # of plcs - default is 1")
 178         parser.add_option("-q","--qualifier",action="store",type="int",dest="qualifier",default=None,
 179                           help="run steps only on plc numbered <qualifier>, starting at 1")
 180         parser.add_option("-y","--rspec-style",action="append",dest="rspec_styles",default=[],
 181                           help="pl is for planetlab rspecs, pg is for protogeni")
 182         parser.add_option("-k","--keep-going",action="store",dest="keep_going",default=False,
 183                           help="proceeds even if some steps are failing")
 184         parser.add_option("-D","--dbname",action="store",dest="dbname",default=None,
 185                            help="Used by plc_db_dump and plc_db_restore")
 186         parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
 187                           help="Run in verbose mode")
 188         parser.add_option("-i","--interactive",action="store_true",dest="interactive",default=False,
 189                           help="prompts before each step")
 190         parser.add_option("-n","--dry-run", action="store_true", dest="dry_run", default=False,
 191                           help="Show environment and exits")
 192 # dropped when added Completer.py
 193 #        parser.add_option("-r","--restart-nm", action="store_true", dest="forcenm", default=False,
 194 #                          help="Force the NM to restart in ssh_slices step")
 195         parser.add_option("-t","--trace", action="store", dest="trace_file", default=None,
 196                           help="Trace file location")
 197         (self.options, self.args) = parser.parse_args()
 198
 199         # allow things like "run -c 'c1 c2' -c c3"
 200         def flatten (x):
 201             result = []
 202             for el in x:
 203                 if hasattr(el, "__iter__") and not isinstance(el, basestring):
 204                     result.extend(flatten(el))
 205                 else:
 206                     result.append(el)
 207             return result
 208         # flatten relevant options
 209         for optname in ['config','exclude','ips_bplc','ips_vplc','ips_bnode','ips_vnode']:
 210             setattr(self.options,optname, flatten ( [ arg.split() for arg in getattr(self.options,optname) ] ))
 211
 212         if not self.options.rspec_styles:
 213             self.options.rspec_styles=TestMain.default_rspec_styles
 214
 215         # handle defaults and option persistence
 216         for (recname,filename,default,need_reverse) in (
 217             ('build_url','arg-build-url',TestMain.default_build_url,None) ,
 218             ('ips_bplc','arg-ips-bplc',[],True),
 219             ('ips_vplc','arg-ips-vplc',[],True) ,
 220             ('ips_bnode','arg-ips-bnode',[],True),
 221             ('ips_vnode','arg-ips-vnode',[],True) ,
 222             ('config','arg-config',TestMain.default_config,False) ,
 223             ('arch_rpms_url','arg-arch-rpms-url',"",None) ,
 224             ('personality','arg-personality',"linux64",None),
 225             ('pldistro','arg-pldistro',"onelab",None),
 226             ('fcdistro','arg-fcdistro','f14',None),
 227             ('plcs_use_lxc','arg-plcs-use-lxc',False,None),
 228             ) :
 229 #            print 'handling',recname
 230             path=filename
 231             is_list = isinstance(default,list)
 232             is_bool = isinstance(default,bool)
 233             if not getattr(self.options,recname):
 234                 try:
 235                     parsed=file(path).readlines()
 236                     if is_list:         # lists
 237                         parsed=[x.strip() for x in parsed]
 238                     else:               # strings and booleans
 239                         if len(parsed) != 1:
 240                             print "%s - error when parsing %s"%(sys.argv[1],path)
 241                             sys.exit(1)
 242                         parsed=parsed[0].strip()
 243                         if is_bool:
 244                             parsed = parsed.lower()=='true'
 245                     setattr(self.options,recname,parsed)
 246                 except:
 247                     if default != "":
 248                         setattr(self.options,recname,default)
 249                     else:
 250                         print "Cannot determine",recname
 251                         print "Run %s --help for help"%sys.argv[0]
 252                         sys.exit(1)
 253
 254             # save for next run
 255             fsave=open(path,"w")
 256             if is_list:                 # lists
 257                 for value in getattr(self.options,recname):
 258                     fsave.write(value + "\n")
 259             else:                       # strings and booleans - just call str()
 260                 fsave.write(str(getattr(self.options,recname)) + "\n")
 261             fsave.close()
 262 #            utils.header('Saved %s into %s'%(recname,filename))
 263
 264             # lists need be reversed
 265             # I suspect this is useful for the various pools but for config, it's painful
 266             if isinstance(getattr(self.options,recname),list) and need_reverse:
 267                 getattr(self.options,recname).reverse()
 268
 269             if self.options.verbose:
 270                 utils.header('* Using %s = %s'%(recname,getattr(self.options,recname)))
 271
 272         # hack : if sfa is not among the published rpms, skip these tests
 273         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
 274
 275         # no step specified
 276         self.options.args = self.args
 277         if len(self.args) == 0:
 278             self.options.steps=TestPlc.default_steps
 279         else:
 280             self.options.steps = self.args
 281
 282         if self.options.list_steps:
 283             self.init_steps()
 284             self.list_steps()
 285             return True
 286
 287         # steps
 288         if not self.options.steps:
 289             #default (all) steps
 290             #self.options.steps=['dump','clean','install','populate']
 291             self.options.steps=TestPlc.default_steps
 292
 293         # rewrite '-' into '_' in step names
 294         self.options.steps = [ step.replace('-','_') for step in self.options.steps ]
 295         self.options.exclude = [ step.replace('-','_') for step in self.options.exclude ]
 296
 297         # exclude
 298         selected=[]
 299         for step in self.options.steps:
 300             keep=True
 301             for exclude in self.options.exclude:
 302                 if utils.match(step,exclude):
 303                     keep=False
 304                     break
 305             if keep: selected.append(step)
 306         self.options.steps=selected
 307
 308         # this is useful when propagating on host boxes, to avoid conflicts
 309         self.options.buildname = os.path.basename (os.path.abspath (self.path))
 310
 311         if self.options.verbose:
 312             self.show_env(self.options,"Verbose")
 313
 314         # load configs
 315         all_plc_specs = []
 316         for config in self.options.config:
 317             modulename='config_'+config
 318             try:
 319                 m = __import__(modulename)
 320                 all_plc_specs = m.config(all_plc_specs,self.options)
 321             except :
 322                 traceback.print_exc()
 323                 print 'Cannot load config %s -- ignored'%modulename
 324                 raise
 325
 326         # provision on local substrate
 327         if self.options.plcs_use_lxc: LocalSubstrate.local_substrate.rescope (plcs_on_vs=False, plcs_on_lxc=True)
 328         all_plc_specs = LocalSubstrate.local_substrate.provision(all_plc_specs,self.options)
 329
 330         # remember substrate IP address(es) for next run
 331         ips_bplc_file=open('arg-ips-bplc','w')
 332         for plc_spec in all_plc_specs:
 333             ips_bplc_file.write("%s\n"%plc_spec['host_box'])
 334         ips_bplc_file.close()
 335         ips_vplc_file=open('arg-ips-vplc','w')
 336         for plc_spec in all_plc_specs:
 337             ips_vplc_file.write("%s\n"%plc_spec['PLC_API_HOST'])
 338         ips_vplc_file.close()
 339         # ditto for nodes
 340         ips_bnode_file=open('arg-ips-bnode','w')
 341         for plc_spec in all_plc_specs:
 342             for site_spec in plc_spec['sites']:
 343                 for node_spec in site_spec['nodes']:
 344                     ips_bnode_file.write("%s\n"%node_spec['host_box'])
 345         ips_bnode_file.close()
 346         ips_vnode_file=open('arg-ips-vnode','w')
 347         for plc_spec in all_plc_specs:
 348             for site_spec in plc_spec['sites']:
 349                 for node_spec in site_spec['nodes']:
 350                     # back to normal (unqualified) form
 351                     stripped=node_spec['node_fields']['hostname'].split('.')[0]
 352                     ips_vnode_file.write("%s\n"%stripped)
 353         ips_vnode_file.close()
 354
 355         # build a TestPlc object from the result, passing options
 356         for spec in all_plc_specs:
 357             spec['failed_step'] = False
 358         all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs]
 359
 360         # pass options to utils as well
 361         utils.init_options(self.options)
 362
 363         overall_result = True
 364         all_step_infos=[]
 365         for step in self.options.steps:
 366             if not TestPlc.valid_step(step):
 367                 continue
 368             # some steps need to be done regardless of the previous ones: we force them
 369             force=False
 370             if step.find("force_") == 0:
 371                 step=step.replace("force_","")
 372                 force=True
 373             # allow for steps to specify an index like in
 374             # run checkslice@2
 375             try:        (step,qualifier)=step.split('@')
 376             except:     qualifier=self.options.qualifier
 377
 378             try:
 379                 stepobj = Step (step)
 380                 for (substep, method) in stepobj.tuples():
 381                     # a cross step will run a method on TestPlc that has a signature like
 382                     # def cross_foo (self, all_test_plcs)
 383                     cross=False
 384                     if substep.find("cross_") == 0:
 385                         cross=True
 386                     all_step_infos.append ( (substep, method, force, cross, qualifier, ) )
 387             except :
 388                 utils.header("********** FAILED step %s (NOT FOUND) -- won't be run"%step)
 389                 traceback.print_exc()
 390                 overall_result = False
 391
 392         if self.options.dry_run:
 393             self.show_env(self.options,"Dry run")
 394
 395         # init & open trace file if provided
 396         if self.options.trace_file and not self.options.dry_run:
 397             # create dir if needed
 398             trace_dir=os.path.dirname(self.options.trace_file)
 399             if trace_dir and not os.path.isdir(trace_dir):
 400                 os.makedirs(trace_dir)
 401             trace=open(self.options.trace_file,"w")
 402
 403         # do all steps on all plcs
 404         TIME_FORMAT="%H-%M-%S"
 405         TRACE_FORMAT="TRACE: %(plc_counter)d %(beg)s->%(end)s status=%(status)s step=%(stepname)s plc=%(plcname)s force=%(force)s\n"
 406         for (stepname,method,force,cross,qualifier) in all_step_infos:
 407             plc_counter=0
 408             for (spec,plc_obj) in all_plcs:
 409                 plc_counter+=1
 410                 # skip this step if we have specified a plc_explicit
 411                 if qualifier and plc_counter!=int(qualifier): continue
 412
 413                 plcname=spec['name']
 414                 across_plcs = [ o for (s,o) in all_plcs if o!=plc_obj ]
 415
 416                 # run the step
 417                 beg=strftime(TIME_FORMAT)
 418                 if not spec['failed_step'] or force or self.options.interactive or self.options.keep_going:
 419                     skip_step=False
 420                     if self.options.interactive:
 421                         prompting=True
 422                         while prompting:
 423                             msg="%d Run step %s on %s [r](un)/d(ry_run)/p(roceed)/s(kip)/q(uit) ? "%(plc_counter,stepname,plcname)
 424                             answer=raw_input(msg).strip().lower() or "r"
 425                             answer=answer[0]
 426                             if answer in ['s','n']:     # skip/no/next
 427                                 print '%s on %s skipped'%(stepname,plcname)
 428                                 prompting=False
 429                                 skip_step=True
 430                             elif answer in ['q','b']:   # quit/bye
 431                                 print 'Exiting'
 432                                 return
 433                             elif answer in ['d']:       # dry_run
 434                                 dry_run=self.options.dry_run
 435                                 self.options.dry_run=True
 436                                 plc_obj.options.dry_run=True
 437                                 plc_obj.apiserver.set_dry_run(True)
 438                                 if not cross:   step_result=method(plc_obj)
 439                                 else:           step_result=method(plc_obj,across_plcs)
 440                                 print 'dry_run step ->',step_result
 441                                 self.options.dry_run=dry_run
 442                                 plc_obj.options.dry_run=dry_run
 443                                 plc_obj.apiserver.set_dry_run(dry_run)
 444                             elif answer in ['p']:
 445                                 # take it as a yes and leave interactive mode
 446                                 prompting=False
 447                                 self.options.interactive=False
 448                             elif answer in ['r','y']:   # run/yes
 449                                 prompting=False
 450                     if skip_step:
 451                         continue
 452                     try:
 453                         force_msg=""
 454                         if force and spec['failed_step']: force_msg=" (forced after %s has failed)"%spec['failed_step']
 455                         utils.header("********** %d RUNNING step %s%s on plc %s"%(plc_counter,stepname,force_msg,plcname))
 456                         if not cross:   step_result = method(plc_obj)
 457                         else:           step_result = method(plc_obj,across_plcs)
 458                         if step_result:
 459                             utils.header('********** %d SUCCESSFUL step %s on %s'%(plc_counter,stepname,plcname))
 460                             status="OK"
 461                         else:
 462                             overall_result = False
 463                             spec['failed_step'] = stepname
 464                             utils.header('********** %d FAILED Step %s on %s (discarded from further steps)'\
 465                                              %(plc_counter,stepname,plcname))
 466                             status="KO"
 467                     except:
 468                         overall_result=False
 469                         spec['failed_step'] = stepname
 470                         traceback.print_exc()
 471                         utils.header ('********** %d FAILED (exception) Step %s on %s (discarded from further steps)'\
 472                                           %(plc_counter,stepname,plcname))
 473                         status="KO"
 474
 475                 # do not run, just display it's skipped
 476                 else:
 477                     why="has failed %s"%spec['failed_step']
 478                     utils.header("********** %d SKIPPED Step %s on %s (%s)"%(plc_counter,stepname,plcname,why))
 479                     status="UNDEF"
 480                 if not self.options.dry_run:
 481                     end=strftime(TIME_FORMAT)
 482                     # always do this on stdout
 483                     print TRACE_FORMAT%locals()
 484                     # duplicate on trace_file if provided
 485                     if self.options.trace_file:
 486                         trace.write(TRACE_FORMAT%locals())
 487                         trace.flush()
 488
 489         if self.options.trace_file and not self.options.dry_run:
 490             trace.close()
 491
 492         # free local substrate
 493         LocalSubstrate.local_substrate.release(self.options)
 494
 495         return overall_result
 496
 497     # wrapper to run, returns a shell-compatible result
 498     def main(self):
 499         try:
 500             success=self.run()
 501             if success:
 502                 return 0
 503             else:
 504                 return 1
 505         except SystemExit:
 506             print 'Caught SystemExit'
 507             raise
 508         except:
 509             traceback.print_exc()
 510             return 2
 511
 512 if __name__ == "__main__":
 513     sys.exit(TestMain().main())