system/TestMain.py

   1 #!/usr/bin/python -u
   2
   3 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
   4 # Copyright (C) 2010 INRIA
   5 #
   6 import sys, os, os.path
   7 from optparse import OptionParser
   8 import traceback
   9 from time import strftime
  10 import readline
  11 import glob
  12
  13 import utils
  14 from TestPlc import TestPlc
  15 from TestSite import TestSite
  16 from TestNode import TestNode
  17 from macros import sequences
  18
  19 # add $HOME in PYTHONPATH so we can import LocalSubstrate.py
  20 sys.path.append(os.environ['HOME'])
  21 import LocalSubstrate
  22
  23 class Step:
  24
  25     natives=TestPlc.__dict__
  26
  27     def display (self): return self.name.replace('_','-')
  28     def internal (self): return self.name.replace('-','_')
  29
  30     def __init__ (self, name):
  31         self.name=name
  32         # a native step is implemented as a method on TestPlc
  33         self.native = self.internal() in Step.natives
  34         if self.native:
  35             self.method=Step.natives[self.internal()]
  36         else:
  37             try:
  38                 self.substeps=sequences[self.internal()]
  39             except Exception,e:
  40                 print "macro step %s not found in macros.py (%s) - exiting"%(self.display(),e)
  41                 raise
  42
  43     def print_doc (self,level=0):
  44         tab=32
  45         trail=8
  46         if self.native:
  47             start=level*' '+'* '
  48             # 2 is the len of '* '
  49             width=tab-level-2
  50             format="%%-%ds"%width
  51             line=start+format%self.display()
  52             print line,
  53             try:
  54                 print self.method.__doc__
  55             except:
  56                 print "*** no doc found"
  57         else:
  58             beg_start=level*' '+'>>> '
  59             end_start=level*' '+'<<< '
  60             trailer=trail*'-'
  61             # 4 is the len of '>>> '
  62             width=tab-level-4-trail
  63             format=("%%-%ds"%width)
  64             beg_line=beg_start+format%self.display()+trail*'>'
  65             end_line=end_start+format%self.display()+trail*'<'
  66             print beg_line
  67             for step in self.substeps:
  68                 Step(step).print_doc(level+1)
  69             print end_line
  70
  71     # return a list of (name, method) for all native steps involved
  72     def tuples (self):
  73         if self.native: return [ (self.internal(), self.method,) ]
  74         else:
  75             result=[]
  76             for substep in [ Step(name) for name in self.substeps ] :
  77                 result += substep.tuples()
  78             return result
  79
  80     # convenience for listing macros
  81     # just do a listdir, hoping we're in the right directory...
  82     @staticmethod
  83     def list_macros ():
  84         names= sequences.keys()
  85         names.sort()
  86         return names
  87
  88 class TestMain:
  89
  90     subversion_id = "Now using git -- version tracker broken"
  91
  92     default_config = [ 'default' ]
  93     default_rspec_styles = [ 'pl', 'pg' ]
  94
  95     default_build_url = "git://git.onelab.eu/tests"
  96
  97     def __init__ (self):
  98         self.path=os.path.dirname(sys.argv[0]) or "."
  99         os.chdir(self.path)
 100
 101     def show_env (self,options, message):
 102         if self.options.verbose:
 103             utils.header (message)
 104             utils.show_options("main options",options)
 105
 106     def init_steps(self):
 107         self.steps_message  = 20*'x'+" Defaut steps are\n"+TestPlc.printable_steps(TestPlc.default_steps)
 108         self.steps_message += 20*'x'+" Other useful steps are\n"+TestPlc.printable_steps(TestPlc.other_steps)
 109         self.steps_message += 20*'x'+" Macro steps are\n"+" ".join(Step.list_macros())
 110
 111     def list_steps(self):
 112         if not self.options.verbose:
 113             print self.steps_message,
 114         else:
 115             # steps mentioned on the command line
 116             if self.options.args:
 117                 scopes = [("Argument steps",self.options.args)]
 118             else:
 119                 scopes = [("Default steps",TestPlc.default_steps)]
 120                 if self.options.all_steps:
 121                     scopes.append ( ("Other steps",TestPlc.other_steps) )
 122                     # try to list macro steps as well
 123                     scopes.append ( ("Macro steps", Step.list_macros()) )
 124             for (scope,steps) in scopes:
 125                 print '--------------------',scope
 126                 for step in [step for step in steps if TestPlc.valid_step(step)]:
 127                     try:        (step,qualifier)=step.split('@')
 128                     except:     pass
 129                     stepname=step
 130                     for special in ['force']:
 131                         stepname = stepname.replace(special+'_',"")
 132                     Step(stepname).print_doc()
 133
 134     def run (self):
 135         self.init_steps()
 136         usage = """usage: %%prog [options] steps
 137 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
 138    no default
 139 config defaults to the last value used, as stored in arg-config,
 140    or %r
 141 ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode},
 142    default is to use IP scanning
 143 steps refer to a method in TestPlc or to a step_* module
 144 ===
 145 """%(TestMain.default_config)
 146         usage += self.steps_message
 147         parser=OptionParser(usage=usage,version=self.subversion_id)
 148         parser.add_option("-u","--url",action="store", dest="arch_rpms_url",
 149                           help="URL of the arch-dependent RPMS area - for locating what to test")
 150         parser.add_option("-b","--build",action="store", dest="build_url",
 151                           help="ignored, for legacy only")
 152         parser.add_option("-c","--config",action="append", dest="config", default=[],
 153                           help="Config module - can be set multiple times, or use quotes")
 154         parser.add_option("-p","--personality",action="store", dest="personality",
 155                           help="personality - as in vbuild-nightly")
 156         parser.add_option("-d","--pldistro",action="store", dest="pldistro",
 157                           help="pldistro - as in vbuild-nightly")
 158         parser.add_option("-f","--fcdistro",action="store", dest="fcdistro",
 159                           help="fcdistro - as in vbuild-nightly")
 160         parser.add_option("-e","--exclude",action="append", dest="exclude", default=[],
 161                           help="steps to exclude - can be set multiple times, or use quotes")
 162         parser.add_option("-a","--all",action="store_true",dest="all_steps", default=False,
 163                           help="Run all default steps")
 164         parser.add_option("-l","--list",action="store_true",dest="list_steps", default=False,
 165                           help="List known steps")
 166         parser.add_option("-V","--vserver",action="append", dest="ips_bplc", default=[],
 167                           help="Specify the set of hostnames for the boxes that host the plcs")
 168         parser.add_option("-P","--plcs",action="append", dest="ips_vplc", default=[],
 169                           help="Specify the set of hostname/IP's to use for vplcs")
 170         parser.add_option("-Q","--qemus",action="append", dest="ips_bnode", default=[],
 171                           help="Specify the set of hostnames for the boxes that host the nodes")
 172         parser.add_option("-N","--nodes",action="append", dest="ips_vnode", default=[],
 173                           help="Specify the set of hostname/IP's to use for vnodes")
 174         parser.add_option ('-X', "--lxc",action='store_true',dest='plcs_use_lxc',default=True,
 175                            help='use lxc-enabled plc boxes instead of vs-enabled ones')
 176         parser.add_option ('-S', "--vs",action='store_false',dest='plcs_use_lxc',
 177                            help='use lxc-enabled plc boxes instead of vs-enabled ones')
 178         parser.add_option("-s","--size",action="store",type="int",dest="size",default=1,
 179                           help="sets test size in # of plcs - default is 1")
 180         parser.add_option("-q","--qualifier",action="store",type="int",dest="qualifier",default=None,
 181                           help="run steps only on plc numbered <qualifier>, starting at 1")
 182         parser.add_option("-y","--rspec-style",action="append",dest="rspec_styles",default=[],
 183                           help="pl is for planetlab rspecs, pg is for protogeni")
 184         parser.add_option("-k","--keep-going",action="store",dest="keep_going",default=False,
 185                           help="proceeds even if some steps are failing")
 186         parser.add_option("-D","--dbname",action="store",dest="dbname",default=None,
 187                            help="Used by plc_db_dump and plc_db_restore")
 188         parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
 189                           help="Run in verbose mode")
 190         parser.add_option("-i","--interactive",action="store_true",dest="interactive",default=False,
 191                           help="prompts before each step")
 192         parser.add_option("-n","--dry-run", action="store_true", dest="dry_run", default=False,
 193                           help="Show environment and exits")
 194 # dropped when added Completer.py
 195 #        parser.add_option("-r","--restart-nm", action="store_true", dest="forcenm", default=False,
 196 #                          help="Force the NM to restart in ssh_slices step")
 197         parser.add_option("-t","--trace", action="store", dest="trace_file", default=None,
 198                           help="Trace file location")
 199         (self.options, self.args) = parser.parse_args()
 200
 201         # allow things like "run -c 'c1 c2' -c c3"
 202         def flatten (x):
 203             result = []
 204             for el in x:
 205                 if hasattr(el, "__iter__") and not isinstance(el, basestring):
 206                     result.extend(flatten(el))
 207                 else:
 208                     result.append(el)
 209             return result
 210         # flatten relevant options
 211         for optname in ['config','exclude','ips_bplc','ips_vplc','ips_bnode','ips_vnode']:
 212             setattr(self.options,optname, flatten ( [ arg.split() for arg in getattr(self.options,optname) ] ))
 213
 214         if not self.options.rspec_styles:
 215             self.options.rspec_styles=TestMain.default_rspec_styles
 216
 217         # handle defaults and option persistence
 218         for (recname,filename,default,need_reverse) in (
 219             ('build_url','arg-build-url',TestMain.default_build_url,None) ,
 220             ('ips_bplc','arg-ips-bplc',[],True),
 221             ('ips_vplc','arg-ips-vplc',[],True) ,
 222             ('ips_bnode','arg-ips-bnode',[],True),
 223             ('ips_vnode','arg-ips-vnode',[],True) ,
 224             ('config','arg-config',TestMain.default_config,False) ,
 225             ('arch_rpms_url','arg-arch-rpms-url',"",None) ,
 226             ('personality','arg-personality',"linux64",None),
 227             ('pldistro','arg-pldistro',"onelab",None),
 228             ('fcdistro','arg-fcdistro','f14',None),
 229             ('plcs_use_lxc','arg-plcs-use-lxc',False,None),
 230             ) :
 231 #            print 'handling',recname
 232             path=filename
 233             is_list = isinstance(default,list)
 234             is_bool = isinstance(default,bool)
 235             if not getattr(self.options,recname):
 236                 try:
 237                     parsed=file(path).readlines()
 238                     if is_list:         # lists
 239                         parsed=[x.strip() for x in parsed]
 240                     else:               # strings and booleans
 241                         if len(parsed) != 1:
 242                             print "%s - error when parsing %s"%(sys.argv[1],path)
 243                             sys.exit(1)
 244                         parsed=parsed[0].strip()
 245                         if is_bool:
 246                             parsed = parsed.lower()=='true'
 247                     setattr(self.options,recname,parsed)
 248                 except:
 249                     if default != "":
 250                         setattr(self.options,recname,default)
 251                     else:
 252                         print "Cannot determine",recname
 253                         print "Run %s --help for help"%sys.argv[0]
 254                         sys.exit(1)
 255
 256             # save for next run
 257             fsave=open(path,"w")
 258             if is_list:                 # lists
 259                 for value in getattr(self.options,recname):
 260                     fsave.write(value + "\n")
 261             else:                       # strings and booleans - just call str()
 262                 fsave.write(str(getattr(self.options,recname)) + "\n")
 263             fsave.close()
 264 #            utils.header('Saved %s into %s'%(recname,filename))
 265
 266             # lists need be reversed
 267             # I suspect this is useful for the various pools but for config, it's painful
 268             if isinstance(getattr(self.options,recname),list) and need_reverse:
 269                 getattr(self.options,recname).reverse()
 270
 271             if self.options.verbose:
 272                 utils.header('* Using %s = %s'%(recname,getattr(self.options,recname)))
 273
 274         # hack : if sfa is not among the published rpms, skip these tests
 275         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
 276
 277         # no step specified
 278         self.options.args = self.args
 279         if len(self.args) == 0:
 280             self.options.steps=TestPlc.default_steps
 281         else:
 282             self.options.steps = self.args
 283
 284         if self.options.list_steps:
 285             self.init_steps()
 286             self.list_steps()
 287             return True
 288
 289         # steps
 290         if not self.options.steps:
 291             #default (all) steps
 292             #self.options.steps=['dump','clean','install','populate']
 293             self.options.steps=TestPlc.default_steps
 294
 295         # rewrite '-' into '_' in step names
 296         self.options.steps = [ step.replace('-','_') for step in self.options.steps ]
 297         self.options.exclude = [ step.replace('-','_') for step in self.options.exclude ]
 298
 299         # exclude
 300         selected=[]
 301         for step in self.options.steps:
 302             keep=True
 303             for exclude in self.options.exclude:
 304                 if utils.match(step,exclude):
 305                     keep=False
 306                     break
 307             if keep: selected.append(step)
 308         self.options.steps=selected
 309
 310         # this is useful when propagating on host boxes, to avoid conflicts
 311         self.options.buildname = os.path.basename (os.path.abspath (self.path))
 312
 313         if self.options.verbose:
 314             self.show_env(self.options,"Verbose")
 315
 316         # load configs
 317         all_plc_specs = []
 318         for config in self.options.config:
 319             modulename='config_'+config
 320             try:
 321                 m = __import__(modulename)
 322                 all_plc_specs = m.config(all_plc_specs,self.options)
 323             except :
 324                 traceback.print_exc()
 325                 print 'Cannot load config %s -- ignored'%modulename
 326                 raise
 327
 328         # provision on local substrate
 329         if self.options.plcs_use_lxc: LocalSubstrate.local_substrate.rescope (plcs_on_vs=False, plcs_on_lxc=True)
 330         all_plc_specs = LocalSubstrate.local_substrate.provision(all_plc_specs,self.options)
 331
 332         # remember substrate IP address(es) for next run
 333         ips_bplc_file=open('arg-ips-bplc','w')
 334         for plc_spec in all_plc_specs:
 335             ips_bplc_file.write("%s\n"%plc_spec['host_box'])
 336         ips_bplc_file.close()
 337         ips_vplc_file=open('arg-ips-vplc','w')
 338         for plc_spec in all_plc_specs:
 339             ips_vplc_file.write("%s\n"%plc_spec['PLC_API_HOST'])
 340         ips_vplc_file.close()
 341         # ditto for nodes
 342         ips_bnode_file=open('arg-ips-bnode','w')
 343         for plc_spec in all_plc_specs:
 344             for site_spec in plc_spec['sites']:
 345                 for node_spec in site_spec['nodes']:
 346                     ips_bnode_file.write("%s\n"%node_spec['host_box'])
 347         ips_bnode_file.close()
 348         ips_vnode_file=open('arg-ips-vnode','w')
 349         for plc_spec in all_plc_specs:
 350             for site_spec in plc_spec['sites']:
 351                 for node_spec in site_spec['nodes']:
 352                     # back to normal (unqualified) form
 353                     stripped=node_spec['node_fields']['hostname'].split('.')[0]
 354                     ips_vnode_file.write("%s\n"%stripped)
 355         ips_vnode_file.close()
 356
 357         # build a TestPlc object from the result, passing options
 358         for spec in all_plc_specs:
 359             spec['failed_step'] = False
 360         all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs]
 361
 362         # pass options to utils as well
 363         utils.init_options(self.options)
 364
 365         overall_result = True
 366         all_step_infos=[]
 367         for step in self.options.steps:
 368             if not TestPlc.valid_step(step):
 369                 continue
 370             # some steps need to be done regardless of the previous ones: we force them
 371             force=False
 372             if step.find("force_") == 0:
 373                 step=step.replace("force_","")
 374                 force=True
 375             # allow for steps to specify an index like in
 376             # run checkslice@2
 377             try:        (step,qualifier)=step.split('@')
 378             except:     qualifier=self.options.qualifier
 379
 380             try:
 381                 stepobj = Step (step)
 382                 for (substep, method) in stepobj.tuples():
 383                     # a cross step will run a method on TestPlc that has a signature like
 384                     # def cross_foo (self, all_test_plcs)
 385                     cross=False
 386                     if substep.find("cross_") == 0:
 387                         cross=True
 388                     all_step_infos.append ( (substep, method, force, cross, qualifier, ) )
 389             except :
 390                 utils.header("********** FAILED step %s (NOT FOUND) -- won't be run"%step)
 391                 traceback.print_exc()
 392                 overall_result = False
 393
 394         if self.options.dry_run:
 395             self.show_env(self.options,"Dry run")
 396
 397         # init & open trace file if provided
 398         if self.options.trace_file and not self.options.dry_run:
 399             # create dir if needed
 400             trace_dir=os.path.dirname(self.options.trace_file)
 401             if trace_dir and not os.path.isdir(trace_dir):
 402                 os.makedirs(trace_dir)
 403             trace=open(self.options.trace_file,"w")
 404
 405         # do all steps on all plcs
 406         TIME_FORMAT="%H-%M-%S"
 407         TRACE_FORMAT="TRACE: %(plc_counter)d %(beg)s->%(end)s status=%(status)s step=%(stepname)s plc=%(plcname)s force=%(force)s\n"
 408         for (stepname,method,force,cross,qualifier) in all_step_infos:
 409             plc_counter=0
 410             for (spec,plc_obj) in all_plcs:
 411                 plc_counter+=1
 412                 # skip this step if we have specified a plc_explicit
 413                 if qualifier and plc_counter!=int(qualifier): continue
 414
 415                 plcname=spec['name']
 416                 across_plcs = [ o for (s,o) in all_plcs if o!=plc_obj ]
 417
 418                 # run the step
 419                 beg=strftime(TIME_FORMAT)
 420                 if not spec['failed_step'] or force or self.options.interactive or self.options.keep_going:
 421                     skip_step=False
 422                     if self.options.interactive:
 423                         prompting=True
 424                         while prompting:
 425                             msg="%d Run step %s on %s [r](un)/d(ry_run)/p(roceed)/s(kip)/q(uit) ? "%(plc_counter,stepname,plcname)
 426                             answer=raw_input(msg).strip().lower() or "r"
 427                             answer=answer[0]
 428                             if answer in ['s','n']:     # skip/no/next
 429                                 print '%s on %s skipped'%(stepname,plcname)
 430                                 prompting=False
 431                                 skip_step=True
 432                             elif answer in ['q','b']:   # quit/bye
 433                                 print 'Exiting'
 434                                 return
 435                             elif answer in ['d']:       # dry_run
 436                                 dry_run=self.options.dry_run
 437                                 self.options.dry_run=True
 438                                 plc_obj.options.dry_run=True
 439                                 plc_obj.apiserver.set_dry_run(True)
 440                                 if not cross:   step_result=method(plc_obj)
 441                                 else:           step_result=method(plc_obj,across_plcs)
 442                                 print 'dry_run step ->',step_result
 443                                 self.options.dry_run=dry_run
 444                                 plc_obj.options.dry_run=dry_run
 445                                 plc_obj.apiserver.set_dry_run(dry_run)
 446                             elif answer in ['p']:
 447                                 # take it as a yes and leave interactive mode
 448                                 prompting=False
 449                                 self.options.interactive=False
 450                             elif answer in ['r','y']:   # run/yes
 451                                 prompting=False
 452                     if skip_step:
 453                         continue
 454                     try:
 455                         force_msg=""
 456                         if force and spec['failed_step']: force_msg=" (forced after %s has failed)"%spec['failed_step']
 457                         utils.header("********** %d RUNNING step %s%s on plc %s"%(plc_counter,stepname,force_msg,plcname))
 458                         if not cross:   step_result = method(plc_obj)
 459                         else:           step_result = method(plc_obj,across_plcs)
 460                         if step_result:
 461                             utils.header('********** %d SUCCESSFUL step %s on %s'%(plc_counter,stepname,plcname))
 462                             status="OK"
 463                         else:
 464                             overall_result = False
 465                             spec['failed_step'] = stepname
 466                             utils.header('********** %d FAILED Step %s on %s (discarded from further steps)'\
 467                                              %(plc_counter,stepname,plcname))
 468                             status="KO"
 469                     except:
 470                         overall_result=False
 471                         spec['failed_step'] = stepname
 472                         traceback.print_exc()
 473                         utils.header ('********** %d FAILED (exception) Step %s on %s (discarded from further steps)'\
 474                                           %(plc_counter,stepname,plcname))
 475                         status="KO"
 476
 477                 # do not run, just display it's skipped
 478                 else:
 479                     why="has failed %s"%spec['failed_step']
 480                     utils.header("********** %d SKIPPED Step %s on %s (%s)"%(plc_counter,stepname,plcname,why))
 481                     status="UNDEF"
 482                 if not self.options.dry_run:
 483                     end=strftime(TIME_FORMAT)
 484                     # always do this on stdout
 485                     print TRACE_FORMAT%locals()
 486                     # duplicate on trace_file if provided
 487                     if self.options.trace_file:
 488                         trace.write(TRACE_FORMAT%locals())
 489                         trace.flush()
 490
 491         if self.options.trace_file and not self.options.dry_run:
 492             trace.close()
 493
 494         # free local substrate
 495         LocalSubstrate.local_substrate.release(self.options)
 496
 497         return overall_result
 498
 499     # wrapper to run, returns a shell-compatible result
 500     def main(self):
 501         try:
 502             success=self.run()
 503             if success:
 504                 return 0
 505             else:
 506                 return 1
 507         except SystemExit:
 508             print 'Caught SystemExit'
 509             raise
 510         except:
 511             traceback.print_exc()
 512             return 2
 513
 514 if __name__ == "__main__":
 515     sys.exit(TestMain().main())