system/TestMain.py

   1 #!/usr/bin/python -u
   2
   3 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
   4 # Copyright (C) 2010 INRIA
   5 #
   6 import sys, os, os.path
   7 from optparse import OptionParser
   8 import traceback
   9 from time import strftime
  10 import readline
  11 import glob
  12
  13 import utils
  14 from TestPlc import TestPlc
  15 from TestSite import TestSite
  16 from TestNode import TestNode
  17 from macros import sequences
  18
  19 # add $HOME in PYTHONPATH so we can import LocalSubstrate.py
  20 sys.path.append(os.environ['HOME'])
  21 import LocalSubstrate
  22
  23 class Step:
  24
  25     natives=TestPlc.__dict__
  26
  27     def display (self): return self.name.replace('_','-')
  28     def internal (self): return self.name.replace('-','_')
  29
  30     def __init__ (self, name):
  31         self.name=name
  32         # a native step is implemented as a method on TestPlc
  33         self.native = self.internal() in Step.natives
  34         if self.native:
  35             self.method=Step.natives[self.internal()]
  36         else:
  37             try:
  38                 self.substeps=sequences[self.internal()]
  39             except Exception,e:
  40                 print "macro step %s not found in macros.py (%s) - exiting"%(self.display(),e)
  41                 raise
  42
  43     def print_doc (self):
  44         if self.native:
  45             print '*',self.display(),"\r",4*"\t",
  46             try:
  47                 print self.method.__doc__
  48             except:
  49                 print "*** no doc found"
  50         else:
  51             print '*',self.display(),"\r",3*"\t","========== BEG MACRO step"
  52             for step in self.substeps:
  53                 Step(step).print_doc()
  54             print '*',self.display(),"\r",3*"\t","========== END MACRO step"
  55
  56     # return a list of (name, method) for all native steps involved
  57     def tuples (self):
  58         if self.native: return [ (self.internal(), self.method,) ]
  59         else:
  60             result=[]
  61             for substep in [ Step(name) for name in self.substeps ] :
  62                 result += substep.tuples()
  63             return result
  64
  65     # convenience for listing macros
  66     # just do a listdir, hoping we're in the right directory...
  67     @staticmethod
  68     def list_macros ():
  69         names= sequences.keys()
  70         names.sort()
  71         return names
  72
  73 class TestMain:
  74
  75     subversion_id = "Now using git -- version tracker broken"
  76
  77     default_config = [ 'default' ]
  78     default_rspec_styles = [ 'pl', 'pg' ]
  79
  80     default_build_url = "git://git.onelab.eu/tests"
  81
  82     def __init__ (self):
  83         self.path=os.path.dirname(sys.argv[0]) or "."
  84         os.chdir(self.path)
  85
  86     def show_env (self,options, message):
  87         if self.options.verbose:
  88             utils.header (message)
  89             utils.show_options("main options",options)
  90
  91     def init_steps(self):
  92         self.steps_message  = 20*'x'+" Defaut steps are\n"+TestPlc.printable_steps(TestPlc.default_steps)
  93         self.steps_message += 20*'x'+" Other useful steps are\n"+TestPlc.printable_steps(TestPlc.other_steps)
  94         self.steps_message += 20*'x'+" Macro steps are\n"+" ".join(Step.list_macros())
  95
  96     def list_steps(self):
  97         if not self.options.verbose:
  98             print self.steps_message,
  99         else:
 100             # steps mentioned on the command line
 101             if self.options.args:
 102                 scopes = [("Argument steps",self.options.args)]
 103             else:
 104                 scopes = [("Default steps",TestPlc.default_steps)]
 105                 if self.options.all_steps:
 106                     scopes.append ( ("Other steps",TestPlc.other_steps) )
 107                     # try to list macro steps as well
 108                     scopes.append ( ("Macro steps", Step.list_macros()) )
 109             for (scope,steps) in scopes:
 110                 print '--------------------',scope
 111                 for step in [step for step in steps if TestPlc.valid_step(step)]:
 112                     try:        (step,qualifier)=step.split('@')
 113                     except:     pass
 114                     stepname=step
 115                     for special in ['force']:
 116                         stepname = stepname.replace(special+'_',"")
 117                     Step(stepname).print_doc()
 118
 119     def run (self):
 120         self.init_steps()
 121         usage = """usage: %%prog [options] steps
 122 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
 123    no default
 124 config defaults to the last value used, as stored in arg-config,
 125    or %r
 126 ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode},
 127    default is to use IP scanning
 128 steps refer to a method in TestPlc or to a step_* module
 129 ===
 130 """%(TestMain.default_config)
 131         usage += self.steps_message
 132         parser=OptionParser(usage=usage,version=self.subversion_id)
 133         parser.add_option("-u","--url",action="store", dest="arch_rpms_url",
 134                           help="URL of the arch-dependent RPMS area - for locating what to test")
 135         parser.add_option("-b","--build",action="store", dest="build_url",
 136                           help="ignored, for legacy only")
 137         parser.add_option("-c","--config",action="append", dest="config", default=[],
 138                           help="Config module - can be set multiple times, or use quotes")
 139         parser.add_option("-p","--personality",action="store", dest="personality",
 140                           help="personality - as in vbuild-nightly")
 141         parser.add_option("-d","--pldistro",action="store", dest="pldistro",
 142                           help="pldistro - as in vbuild-nightly")
 143         parser.add_option("-f","--fcdistro",action="store", dest="fcdistro",
 144                           help="fcdistro - as in vbuild-nightly")
 145         parser.add_option("-e","--exclude",action="append", dest="exclude", default=[],
 146                           help="steps to exclude - can be set multiple times, or use quotes")
 147         parser.add_option("-a","--all",action="store_true",dest="all_steps", default=False,
 148                           help="Run all default steps")
 149         parser.add_option("-l","--list",action="store_true",dest="list_steps", default=False,
 150                           help="List known steps")
 151         parser.add_option("-V","--vserver",action="append", dest="ips_bplc", default=[],
 152                           help="Specify the set of hostnames for the boxes that host the plcs")
 153         parser.add_option("-P","--plcs",action="append", dest="ips_vplc", default=[],
 154                           help="Specify the set of hostname/IP's to use for vplcs")
 155         parser.add_option("-Q","--qemus",action="append", dest="ips_bnode", default=[],
 156                           help="Specify the set of hostnames for the boxes that host the nodes")
 157         parser.add_option("-N","--nodes",action="append", dest="ips_vnode", default=[],
 158                           help="Specify the set of hostname/IP's to use for vnodes")
 159         parser.add_option ('-X', "--lxc",action='store_true',dest='plcs_use_lxc',
 160                            help='use lxc-enabled plc boxes instead of vs-enabled ones')
 161         parser.add_option("-s","--size",action="store",type="int",dest="size",default=1,
 162                           help="sets test size in # of plcs - default is 1")
 163         parser.add_option("-q","--qualifier",action="store",type="int",dest="qualifier",default=None,
 164                           help="run steps only on plc numbered <qualifier>, starting at 1")
 165         parser.add_option("-y","--rspec-style",action="append",dest="rspec_styles",default=[],
 166                           help="pl is for planetlab rspecs, pg is for protogeni")
 167         parser.add_option("-k","--keep-going",action="store",dest="keep_going",default=False,
 168                           help="proceeds even if some steps are failing")
 169         parser.add_option("-D","--dbname",action="store",dest="dbname",default=None,
 170                            help="Used by plc_db_dump and plc_db_restore")
 171         parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False,
 172                           help="Run in verbose mode")
 173         parser.add_option("-i","--interactive",action="store_true",dest="interactive",default=False,
 174                           help="prompts before each step")
 175         parser.add_option("-n","--dry-run", action="store_true", dest="dry_run", default=False,
 176                           help="Show environment and exits")
 177         parser.add_option("-r","--restart-nm", action="store_true", dest="forcenm", default=False,
 178                           help="Force the NM to restart in ssh_slices step")
 179         parser.add_option("-t","--trace", action="store", dest="trace_file", default=None,
 180                           help="Trace file location")
 181         (self.options, self.args) = parser.parse_args()
 182
 183         # allow things like "run -c 'c1 c2' -c c3"
 184         def flatten (x):
 185             result = []
 186             for el in x:
 187                 if hasattr(el, "__iter__") and not isinstance(el, basestring):
 188                     result.extend(flatten(el))
 189                 else:
 190                     result.append(el)
 191             return result
 192         # flatten relevant options
 193         for optname in ['config','exclude','ips_bplc','ips_vplc','ips_bnode','ips_vnode']:
 194             setattr(self.options,optname, flatten ( [ arg.split() for arg in getattr(self.options,optname) ] ))
 195
 196         if not self.options.rspec_styles:
 197             self.options.rspec_styles=TestMain.default_rspec_styles
 198
 199         # handle defaults and option persistence
 200         for (recname,filename,default,need_reverse) in (
 201             ('build_url','arg-build-url',TestMain.default_build_url,None) ,
 202             ('ips_bplc','arg-ips-bplc',[],True),
 203             ('ips_vplc','arg-ips-vplc',[],True) ,
 204             ('ips_bnode','arg-ips-bnode',[],True),
 205             ('ips_vnode','arg-ips-vnode',[],True) ,
 206             ('config','arg-config',TestMain.default_config,False) ,
 207             ('arch_rpms_url','arg-arch-rpms-url',"",None) ,
 208             ('personality','arg-personality',"linux64",None),
 209             ('pldistro','arg-pldistro',"onelab",None),
 210             ('fcdistro','arg-fcdistro','f14',None),
 211             ('plcs_use_lxc','arg-plcs-use-lxc',False,None),
 212             ) :
 213 #            print 'handling',recname
 214             path=filename
 215             is_list = isinstance(default,list)
 216             is_bool = isinstance(default,bool)
 217             if not getattr(self.options,recname):
 218                 try:
 219                     parsed=file(path).readlines()
 220                     if is_list:         # lists
 221                         parsed=[x.strip() for x in parsed]
 222                     else:               # strings and booleans
 223                         if len(parsed) != 1:
 224                             print "%s - error when parsing %s"%(sys.argv[1],path)
 225                             sys.exit(1)
 226                         parsed=parsed[0].strip()
 227                         if is_bool:
 228                             parsed = parsed.lower()=='true'
 229                     setattr(self.options,recname,parsed)
 230                 except:
 231                     if default != "":
 232                         setattr(self.options,recname,default)
 233                     else:
 234                         print "Cannot determine",recname
 235                         print "Run %s --help for help"%sys.argv[0]
 236                         sys.exit(1)
 237
 238             # save for next run
 239             fsave=open(path,"w")
 240             if is_list:                 # lists
 241                 for value in getattr(self.options,recname):
 242                     fsave.write(value + "\n")
 243             else:                       # strings and booleans - just call str()
 244                 fsave.write(str(getattr(self.options,recname)) + "\n")
 245             fsave.close()
 246 #            utils.header('Saved %s into %s'%(recname,filename))
 247
 248             # lists need be reversed
 249             # I suspect this is useful for the various pools but for config, it's painful
 250             if isinstance(getattr(self.options,recname),list) and need_reverse:
 251                 getattr(self.options,recname).reverse()
 252
 253             if self.options.verbose:
 254                 utils.header('* Using %s = %s'%(recname,getattr(self.options,recname)))
 255
 256         # hack : if sfa is not among the published rpms, skip these tests
 257         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
 258
 259         # no step specified
 260         self.options.args = self.args
 261         if len(self.args) == 0:
 262             self.options.steps=TestPlc.default_steps
 263         else:
 264             self.options.steps = self.args
 265
 266         if self.options.list_steps:
 267             self.init_steps()
 268             self.list_steps()
 269             return True
 270
 271         # steps
 272         if not self.options.steps:
 273             #default (all) steps
 274             #self.options.steps=['dump','clean','install','populate']
 275             self.options.steps=TestPlc.default_steps
 276
 277         # rewrite '-' into '_' in step names
 278         self.options.steps = [ step.replace('-','_') for step in self.options.steps ]
 279
 280         # exclude
 281         selected=[]
 282         for step in self.options.steps:
 283             keep=True
 284             for exclude in self.options.exclude:
 285                 if utils.match(step,exclude):
 286                     keep=False
 287                     break
 288             if keep: selected.append(step)
 289         self.options.steps=selected
 290
 291         # this is useful when propagating on host boxes, to avoid conflicts
 292         self.options.buildname = os.path.basename (os.path.abspath (self.path))
 293
 294         if self.options.verbose:
 295             self.show_env(self.options,"Verbose")
 296
 297         # load configs
 298         all_plc_specs = []
 299         for config in self.options.config:
 300             modulename='config_'+config
 301             try:
 302                 m = __import__(modulename)
 303                 all_plc_specs = m.config(all_plc_specs,self.options)
 304             except :
 305                 traceback.print_exc()
 306                 print 'Cannot load config %s -- ignored'%modulename
 307                 raise
 308
 309         # provision on local substrate
 310         if self.options.plcs_use_lxc: LocalSubstrate.local_substrate.rescope (plcs_on_vs=False, plcs_on_lxc=True)
 311         all_plc_specs = LocalSubstrate.local_substrate.provision(all_plc_specs,self.options)
 312
 313         # remember substrate IP address(es) for next run
 314         ips_bplc_file=open('arg-ips-bplc','w')
 315         for plc_spec in all_plc_specs:
 316             ips_bplc_file.write("%s\n"%plc_spec['host_box'])
 317         ips_bplc_file.close()
 318         ips_vplc_file=open('arg-ips-vplc','w')
 319         for plc_spec in all_plc_specs:
 320             ips_vplc_file.write("%s\n"%plc_spec['PLC_API_HOST'])
 321         ips_vplc_file.close()
 322         # ditto for nodes
 323         ips_bnode_file=open('arg-ips-bnode','w')
 324         for plc_spec in all_plc_specs:
 325             for site_spec in plc_spec['sites']:
 326                 for node_spec in site_spec['nodes']:
 327                     ips_bnode_file.write("%s\n"%node_spec['host_box'])
 328         ips_bnode_file.close()
 329         ips_vnode_file=open('arg-ips-vnode','w')
 330         for plc_spec in all_plc_specs:
 331             for site_spec in plc_spec['sites']:
 332                 for node_spec in site_spec['nodes']:
 333                     # back to normal (unqualified) form
 334                     stripped=node_spec['node_fields']['hostname'].split('.')[0]
 335                     ips_vnode_file.write("%s\n"%stripped)
 336         ips_vnode_file.close()
 337
 338         # build a TestPlc object from the result, passing options
 339         for spec in all_plc_specs:
 340             spec['failed_step'] = False
 341         all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs]
 342
 343         # pass options to utils as well
 344         utils.init_options(self.options)
 345
 346         overall_result = True
 347         all_step_infos=[]
 348         for step in self.options.steps:
 349             if not TestPlc.valid_step(step):
 350                 continue
 351             # some steps need to be done regardless of the previous ones: we force them
 352             force=False
 353             if step.find("force_") == 0:
 354                 step=step.replace("force_","")
 355                 force=True
 356             # allow for steps to specify an index like in
 357             # run checkslice@2
 358             try:        (step,qualifier)=step.split('@')
 359             except:     qualifier=self.options.qualifier
 360
 361             try:
 362                 stepobj = Step (step)
 363                 for (substep, method) in stepobj.tuples():
 364                     # a cross step will run a method on TestPlc that has a signature like
 365                     # def cross_foo (self, all_test_plcs)
 366                     cross=False
 367                     if substep.find("cross_") == 0:
 368                         cross=True
 369                     all_step_infos.append ( (substep, method, force, cross, qualifier, ) )
 370             except :
 371                 utils.header("********** FAILED step %s (NOT FOUND) -- won't be run"%step)
 372                 traceback.print_exc()
 373                 overall_result = False
 374
 375         if self.options.dry_run:
 376             self.show_env(self.options,"Dry run")
 377
 378         # init & open trace file if provided
 379         if self.options.trace_file and not self.options.dry_run:
 380             # create dir if needed
 381             trace_dir=os.path.dirname(self.options.trace_file)
 382             if trace_dir and not os.path.isdir(trace_dir):
 383                 os.makedirs(trace_dir)
 384             trace=open(self.options.trace_file,"w")
 385
 386         # do all steps on all plcs
 387         TIME_FORMAT="%H-%M-%S"
 388         TRACE_FORMAT="TRACE: %(plc_counter)d %(beg)s->%(end)s status=%(status)s step=%(stepname)s plc=%(plcname)s force=%(force)s\n"
 389         for (stepname,method,force,cross,qualifier) in all_step_infos:
 390             plc_counter=0
 391             for (spec,plc_obj) in all_plcs:
 392                 plc_counter+=1
 393                 # skip this step if we have specified a plc_explicit
 394                 if qualifier and plc_counter!=int(qualifier): continue
 395
 396                 plcname=spec['name']
 397                 across_plcs = [ o for (s,o) in all_plcs if o!=plc_obj ]
 398
 399                 # run the step
 400                 beg=strftime(TIME_FORMAT)
 401                 if not spec['failed_step'] or force or self.options.interactive or self.options.keep_going:
 402                     skip_step=False
 403                     if self.options.interactive:
 404                         prompting=True
 405                         while prompting:
 406                             msg="%d Run step %s on %s [r](un)/d(ry_run)/p(roceed)/s(kip)/q(uit) ? "%(plc_counter,stepname,plcname)
 407                             answer=raw_input(msg).strip().lower() or "r"
 408                             answer=answer[0]
 409                             if answer in ['s','n']:     # skip/no/next
 410                                 print '%s on %s skipped'%(stepname,plcname)
 411                                 prompting=False
 412                                 skip_step=True
 413                             elif answer in ['q','b']:   # quit/bye
 414                                 print 'Exiting'
 415                                 return
 416                             elif answer in ['d']:       # dry_run
 417                                 dry_run=self.options.dry_run
 418                                 self.options.dry_run=True
 419                                 plc_obj.options.dry_run=True
 420                                 plc_obj.apiserver.set_dry_run(True)
 421                                 if not cross:   step_result=method(plc_obj)
 422                                 else:           step_result=method(plc_obj,across_plcs)
 423                                 print 'dry_run step ->',step_result
 424                                 self.options.dry_run=dry_run
 425                                 plc_obj.options.dry_run=dry_run
 426                                 plc_obj.apiserver.set_dry_run(dry_run)
 427                             elif answer in ['p']:
 428                                 # take it as a yes and leave interactive mode
 429                                 prompting=False
 430                                 self.options.interactive=False
 431                             elif answer in ['r','y']:   # run/yes
 432                                 prompting=False
 433                     if skip_step:
 434                         continue
 435                     try:
 436                         force_msg=""
 437                         if force and spec['failed_step']: force_msg=" (forced after %s has failed)"%spec['failed_step']
 438                         utils.header("********** %d RUNNING step %s%s on plc %s"%(plc_counter,stepname,force_msg,plcname))
 439                         if not cross:   step_result = method(plc_obj)
 440                         else:           step_result = method(plc_obj,across_plcs)
 441                         if step_result:
 442                             utils.header('********** %d SUCCESSFUL step %s on %s'%(plc_counter,stepname,plcname))
 443                             status="OK"
 444                         else:
 445                             overall_result = False
 446                             spec['failed_step'] = stepname
 447                             utils.header('********** %d FAILED Step %s on %s (discarded from further steps)'\
 448                                              %(plc_counter,stepname,plcname))
 449                             status="KO"
 450                     except:
 451                         overall_result=False
 452                         spec['failed_step'] = stepname
 453                         traceback.print_exc()
 454                         utils.header ('********** %d FAILED (exception) Step %s on %s (discarded from further steps)'\
 455                                           %(plc_counter,stepname,plcname))
 456                         status="KO"
 457
 458                 # do not run, just display it's skipped
 459                 else:
 460                     why="has failed %s"%spec['failed_step']
 461                     utils.header("********** %d SKIPPED Step %s on %s (%s)"%(plc_counter,stepname,plcname,why))
 462                     status="UNDEF"
 463                 if not self.options.dry_run:
 464                     end=strftime(TIME_FORMAT)
 465                     # always do this on stdout
 466                     print TRACE_FORMAT%locals()
 467                     # duplicate on trace_file if provided
 468                     if self.options.trace_file:
 469                         trace.write(TRACE_FORMAT%locals())
 470                         trace.flush()
 471
 472         if self.options.trace_file and not self.options.dry_run:
 473             trace.close()
 474
 475         # free local substrate
 476         LocalSubstrate.local_substrate.release(self.options)
 477
 478         return overall_result
 479
 480     # wrapper to run, returns a shell-compatible result
 481     def main(self):
 482         try:
 483             success=self.run()
 484             if success:
 485                 return 0
 486             else:
 487                 return 1
 488         except SystemExit:
 489             print 'Caught SystemExit'
 490             raise
 491         except:
 492             traceback.print_exc()
 493             return 2
 494
 495 if __name__ == "__main__":
 496     sys.exit(TestMain().main())