..
[tests.git] / system / TestMain.py
1 #!/usr/bin/python -u
2
3 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
4 # Copyright (C) 2010 INRIA 
5 #
6 import sys, os, os.path
7 from optparse import OptionParser
8 import traceback
9 from time import strftime
10 import readline
11 import glob
12
13 import utils
14 from TestPlc import TestPlc
15 from TestSite import TestSite
16 from TestNode import TestNode
17 from macros import sequences
18
19 # add $HOME in PYTHONPATH so we can import LocalSubstrate.py
20 sys.path.append(os.environ['HOME'])
21 import LocalSubstrate
22
23 class Step:
24
25     natives=TestPlc.__dict__
26
27     def display (self): return self.name.replace('_','-')
28     def internal (self): return self.name.replace('-','_')
29
30     def __init__ (self, name):
31         self.name=name
32         # a native step is implemented as a method on TestPlc
33         self.native = self.internal() in Step.natives
34         if self.native:
35             self.method=Step.natives[self.internal()]
36         else:
37             try:
38                 self.substeps=sequences[self.internal()]
39             except Exception,e:
40                 print "macro step %s not found in macros.py (%s) - exiting"%(self.display(),e)
41                 raise
42
43     def print_doc (self,level=0):
44         tab=32
45         trail=8
46         if self.native:
47             start=level*' '+'* '
48             # 2 is the len of '* '
49             width=tab-level-2
50             format="%%-%ds"%width
51             line=start+format%self.display()
52             print line,
53             try:
54                 print self.method.__doc__
55             except:
56                 print "*** no doc found"
57         else:
58             beg_start=level*' '+'>>> '
59             end_start=level*' '+'<<< '
60             trailer=trail*'-'
61             # 4 is the len of '>>> '
62             width=tab-level-4-trail
63             format=("%%-%ds"%width)
64             beg_line=beg_start+format%self.display()+trail*'>'
65             end_line=end_start+format%self.display()+trail*'<'
66             print beg_line
67             for step in self.substeps:
68                 Step(step).print_doc(level+1)
69             print end_line
70
71     # return a list of (name, method) for all native steps involved
72     def tuples (self):
73         if self.native: return [ (self.internal(), self.method,) ]
74         else:
75             result=[]
76             for substep in [ Step(name) for name in self.substeps ] : 
77                 result += substep.tuples()
78             return result
79
80     # convenience for listing macros
81     # just do a listdir, hoping we're in the right directory...
82     @staticmethod
83     def list_macros ():
84         names= sequences.keys()
85         names.sort()
86         return names
87
88 class TestMain:
89
90     subversion_id = "Now using git -- version tracker broken"
91
92     default_config = [ 'default' ] 
93     default_rspec_styles = [ 'pl', 'pg' ]
94
95     default_build_url = "git://git.onelab.eu/tests"
96
97     def __init__ (self):
98         self.path=os.path.dirname(sys.argv[0]) or "."
99         os.chdir(self.path)
100
101     def show_env (self,options, message):
102         if self.options.verbose:
103             utils.header (message)
104             utils.show_options("main options",options)
105
106     def init_steps(self):
107         self.steps_message  = 20*'x'+" Defaut steps are\n"+TestPlc.printable_steps(TestPlc.default_steps)
108         self.steps_message += 20*'x'+" Other useful steps are\n"+TestPlc.printable_steps(TestPlc.other_steps)
109         self.steps_message += 20*'x'+" Macro steps are\n"+" ".join(Step.list_macros())
110
111     def list_steps(self):
112         if not self.options.verbose:
113             print self.steps_message,
114         else:
115             # steps mentioned on the command line
116             if self.options.args:
117                 scopes = [("Argument steps",self.options.args)]
118             else:
119                 scopes = [("Default steps",TestPlc.default_steps)]
120                 if self.options.all_steps:
121                     scopes.append ( ("Other steps",TestPlc.other_steps) )
122                     # try to list macro steps as well
123                     scopes.append ( ("Macro steps", Step.list_macros()) )
124             for (scope,steps) in scopes:
125                 print '--------------------',scope
126                 for step in [step for step in steps if TestPlc.valid_step(step)]:
127                     try:        (step,qualifier)=step.split('@')
128                     except:     pass
129                     stepname=step
130                     for special in ['force']:
131                         stepname = stepname.replace(special+'_',"")
132                     Step(stepname).print_doc()
133
134     def run (self):
135         self.init_steps()
136         usage = """usage: %%prog [options] steps
137 arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url,
138    no default
139 config defaults to the last value used, as stored in arg-config,
140    or %r
141 ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode},
142    default is to use IP scanning
143 steps refer to a method in TestPlc or to a step_* module
144 ===
145 """%(TestMain.default_config)
146         usage += self.steps_message
147         parser=OptionParser(usage=usage,version=self.subversion_id)
148         parser.add_option("-u","--url",action="store", dest="arch_rpms_url", 
149                           help="URL of the arch-dependent RPMS area - for locating what to test")
150         parser.add_option("-b","--build",action="store", dest="build_url", 
151                           help="ignored, for legacy only")
152         parser.add_option("-c","--config",action="append", dest="config", default=[],
153                           help="Config module - can be set multiple times, or use quotes")
154         parser.add_option("-p","--personality",action="store", dest="personality", 
155                           help="personality - as in vbuild-nightly")
156         parser.add_option("-d","--pldistro",action="store", dest="pldistro", 
157                           help="pldistro - as in vbuild-nightly")
158         parser.add_option("-f","--fcdistro",action="store", dest="fcdistro", 
159                           help="fcdistro - as in vbuild-nightly")
160         parser.add_option("-e","--exclude",action="append", dest="exclude", default=[],
161                           help="steps to exclude - can be set multiple times, or use quotes")
162         parser.add_option("-a","--all",action="store_true",dest="all_steps", default=False,
163                           help="Run all default steps")
164         parser.add_option("-l","--list",action="store_true",dest="list_steps", default=False,
165                           help="List known steps")
166         parser.add_option("-V","--vserver",action="append", dest="ips_bplc", default=[],
167                           help="Specify the set of hostnames for the boxes that host the plcs")
168         parser.add_option("-P","--plcs",action="append", dest="ips_vplc", default=[],
169                           help="Specify the set of hostname/IP's to use for vplcs")
170         parser.add_option("-Q","--qemus",action="append", dest="ips_bnode", default=[],
171                           help="Specify the set of hostnames for the boxes that host the nodes")
172         parser.add_option("-N","--nodes",action="append", dest="ips_vnode", default=[],
173                           help="Specify the set of hostname/IP's to use for vnodes")
174         parser.add_option ('-X', "--lxc",action='store_true',dest='plcs_use_lxc',
175                            help='use lxc-enabled plc boxes instead of vs-enabled ones')
176         parser.add_option("-s","--size",action="store",type="int",dest="size",default=1,
177                           help="sets test size in # of plcs - default is 1")
178         parser.add_option("-q","--qualifier",action="store",type="int",dest="qualifier",default=None,
179                           help="run steps only on plc numbered <qualifier>, starting at 1")
180         parser.add_option("-y","--rspec-style",action="append",dest="rspec_styles",default=[],
181                           help="pl is for planetlab rspecs, pg is for protogeni")
182         parser.add_option("-k","--keep-going",action="store",dest="keep_going",default=False,
183                           help="proceeds even if some steps are failing")
184         parser.add_option("-D","--dbname",action="store",dest="dbname",default=None,
185                            help="Used by plc_db_dump and plc_db_restore")
186         parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False, 
187                           help="Run in verbose mode")
188         parser.add_option("-i","--interactive",action="store_true",dest="interactive",default=False,
189                           help="prompts before each step")
190         parser.add_option("-n","--dry-run", action="store_true", dest="dry_run", default=False,
191                           help="Show environment and exits")
192         parser.add_option("-r","--restart-nm", action="store_true", dest="forcenm", default=False, 
193                           help="Force the NM to restart in ssh_slices step")
194         parser.add_option("-t","--trace", action="store", dest="trace_file", default=None,
195                           help="Trace file location")
196         (self.options, self.args) = parser.parse_args()
197
198         # allow things like "run -c 'c1 c2' -c c3"
199         def flatten (x):
200             result = []
201             for el in x:
202                 if hasattr(el, "__iter__") and not isinstance(el, basestring):
203                     result.extend(flatten(el))
204                 else:
205                     result.append(el)
206             return result
207         # flatten relevant options
208         for optname in ['config','exclude','ips_bplc','ips_vplc','ips_bnode','ips_vnode']:
209             setattr(self.options,optname, flatten ( [ arg.split() for arg in getattr(self.options,optname) ] ))
210
211         if not self.options.rspec_styles:
212             self.options.rspec_styles=TestMain.default_rspec_styles
213
214         # handle defaults and option persistence
215         for (recname,filename,default,need_reverse) in (
216             ('build_url','arg-build-url',TestMain.default_build_url,None) ,
217             ('ips_bplc','arg-ips-bplc',[],True),
218             ('ips_vplc','arg-ips-vplc',[],True) , 
219             ('ips_bnode','arg-ips-bnode',[],True),
220             ('ips_vnode','arg-ips-vnode',[],True) , 
221             ('config','arg-config',TestMain.default_config,False) , 
222             ('arch_rpms_url','arg-arch-rpms-url',"",None) , 
223             ('personality','arg-personality',"linux64",None),
224             ('pldistro','arg-pldistro',"onelab",None),
225             ('fcdistro','arg-fcdistro','f14',None),
226             ('plcs_use_lxc','arg-plcs-use-lxc',False,None),
227             ) :
228 #            print 'handling',recname
229             path=filename
230             is_list = isinstance(default,list)
231             is_bool = isinstance(default,bool)
232             if not getattr(self.options,recname):
233                 try:
234                     parsed=file(path).readlines()
235                     if is_list:         # lists
236                         parsed=[x.strip() for x in parsed]
237                     else:               # strings and booleans
238                         if len(parsed) != 1:
239                             print "%s - error when parsing %s"%(sys.argv[1],path)
240                             sys.exit(1)
241                         parsed=parsed[0].strip()
242                         if is_bool:
243                             parsed = parsed.lower()=='true'
244                     setattr(self.options,recname,parsed)
245                 except:
246                     if default != "":
247                         setattr(self.options,recname,default)
248                     else:
249                         print "Cannot determine",recname
250                         print "Run %s --help for help"%sys.argv[0]                        
251                         sys.exit(1)
252
253             # save for next run
254             fsave=open(path,"w")
255             if is_list:                 # lists
256                 for value in getattr(self.options,recname):
257                     fsave.write(value + "\n")
258             else:                       # strings and booleans - just call str()
259                 fsave.write(str(getattr(self.options,recname)) + "\n")
260             fsave.close()
261 #            utils.header('Saved %s into %s'%(recname,filename))
262
263             # lists need be reversed
264             # I suspect this is useful for the various pools but for config, it's painful
265             if isinstance(getattr(self.options,recname),list) and need_reverse:
266                 getattr(self.options,recname).reverse()
267
268             if self.options.verbose:
269                 utils.header('* Using %s = %s'%(recname,getattr(self.options,recname)))
270
271         # hack : if sfa is not among the published rpms, skip these tests
272         TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url)
273
274         # no step specified
275         self.options.args = self.args
276         if len(self.args) == 0:
277             self.options.steps=TestPlc.default_steps
278         else:
279             self.options.steps = self.args
280
281         if self.options.list_steps:
282             self.init_steps()
283             self.list_steps()
284             return True
285
286         # steps
287         if not self.options.steps:
288             #default (all) steps
289             #self.options.steps=['dump','clean','install','populate']
290             self.options.steps=TestPlc.default_steps
291
292         # rewrite '-' into '_' in step names
293         self.options.steps = [ step.replace('-','_') for step in self.options.steps ]
294         self.options.exclude = [ step.replace('-','_') for step in self.options.exclude ]
295
296         # exclude
297         selected=[]
298         for step in self.options.steps:
299             keep=True
300             for exclude in self.options.exclude:
301                 if utils.match(step,exclude):
302                     keep=False
303                     break
304             if keep: selected.append(step)
305         self.options.steps=selected
306
307         # this is useful when propagating on host boxes, to avoid conflicts
308         self.options.buildname = os.path.basename (os.path.abspath (self.path))
309
310         if self.options.verbose:
311             self.show_env(self.options,"Verbose")
312
313         # load configs
314         all_plc_specs = []
315         for config in self.options.config:
316             modulename='config_'+config
317             try:
318                 m = __import__(modulename)
319                 all_plc_specs = m.config(all_plc_specs,self.options)
320             except :
321                 traceback.print_exc()
322                 print 'Cannot load config %s -- ignored'%modulename
323                 raise
324
325         # provision on local substrate
326         if self.options.plcs_use_lxc: LocalSubstrate.local_substrate.rescope (plcs_on_vs=False, plcs_on_lxc=True)
327         all_plc_specs = LocalSubstrate.local_substrate.provision(all_plc_specs,self.options)
328
329         # remember substrate IP address(es) for next run
330         ips_bplc_file=open('arg-ips-bplc','w')
331         for plc_spec in all_plc_specs:
332             ips_bplc_file.write("%s\n"%plc_spec['host_box'])
333         ips_bplc_file.close()
334         ips_vplc_file=open('arg-ips-vplc','w')
335         for plc_spec in all_plc_specs:
336             ips_vplc_file.write("%s\n"%plc_spec['PLC_API_HOST'])
337         ips_vplc_file.close()
338         # ditto for nodes
339         ips_bnode_file=open('arg-ips-bnode','w')
340         for plc_spec in all_plc_specs:
341             for site_spec in plc_spec['sites']:
342                 for node_spec in site_spec['nodes']:
343                     ips_bnode_file.write("%s\n"%node_spec['host_box'])
344         ips_bnode_file.close()
345         ips_vnode_file=open('arg-ips-vnode','w')
346         for plc_spec in all_plc_specs:
347             for site_spec in plc_spec['sites']:
348                 for node_spec in site_spec['nodes']:
349                     # back to normal (unqualified) form
350                     stripped=node_spec['node_fields']['hostname'].split('.')[0]
351                     ips_vnode_file.write("%s\n"%stripped)
352         ips_vnode_file.close()
353
354         # build a TestPlc object from the result, passing options
355         for spec in all_plc_specs:
356             spec['failed_step'] = False
357         all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs]
358
359         # pass options to utils as well
360         utils.init_options(self.options)
361
362         overall_result = True
363         all_step_infos=[]
364         for step in self.options.steps:
365             if not TestPlc.valid_step(step):
366                 continue
367             # some steps need to be done regardless of the previous ones: we force them
368             force=False
369             if step.find("force_") == 0:
370                 step=step.replace("force_","")
371                 force=True
372             # allow for steps to specify an index like in 
373             # run checkslice@2
374             try:        (step,qualifier)=step.split('@')
375             except:     qualifier=self.options.qualifier
376
377             try:
378                 stepobj = Step (step)
379                 for (substep, method) in stepobj.tuples():
380                     # a cross step will run a method on TestPlc that has a signature like
381                     # def cross_foo (self, all_test_plcs)
382                     cross=False
383                     if substep.find("cross_") == 0:
384                         cross=True
385                     all_step_infos.append ( (substep, method, force, cross, qualifier, ) )
386             except :
387                 utils.header("********** FAILED step %s (NOT FOUND) -- won't be run"%step)
388                 traceback.print_exc()
389                 overall_result = False
390             
391         if self.options.dry_run:
392             self.show_env(self.options,"Dry run")
393         
394         # init & open trace file if provided
395         if self.options.trace_file and not self.options.dry_run:
396             # create dir if needed
397             trace_dir=os.path.dirname(self.options.trace_file)
398             if trace_dir and not os.path.isdir(trace_dir):
399                 os.makedirs(trace_dir)
400             trace=open(self.options.trace_file,"w")
401
402         # do all steps on all plcs
403         TIME_FORMAT="%H-%M-%S"
404         TRACE_FORMAT="TRACE: %(plc_counter)d %(beg)s->%(end)s status=%(status)s step=%(stepname)s plc=%(plcname)s force=%(force)s\n"
405         for (stepname,method,force,cross,qualifier) in all_step_infos:
406             plc_counter=0
407             for (spec,plc_obj) in all_plcs:
408                 plc_counter+=1
409                 # skip this step if we have specified a plc_explicit
410                 if qualifier and plc_counter!=int(qualifier): continue
411
412                 plcname=spec['name']
413                 across_plcs = [ o for (s,o) in all_plcs if o!=plc_obj ]
414
415                 # run the step
416                 beg=strftime(TIME_FORMAT)
417                 if not spec['failed_step'] or force or self.options.interactive or self.options.keep_going:
418                     skip_step=False
419                     if self.options.interactive:
420                         prompting=True
421                         while prompting:
422                             msg="%d Run step %s on %s [r](un)/d(ry_run)/p(roceed)/s(kip)/q(uit) ? "%(plc_counter,stepname,plcname)
423                             answer=raw_input(msg).strip().lower() or "r"
424                             answer=answer[0]
425                             if answer in ['s','n']:     # skip/no/next
426                                 print '%s on %s skipped'%(stepname,plcname)
427                                 prompting=False
428                                 skip_step=True
429                             elif answer in ['q','b']:   # quit/bye
430                                 print 'Exiting'
431                                 return
432                             elif answer in ['d']:       # dry_run
433                                 dry_run=self.options.dry_run
434                                 self.options.dry_run=True
435                                 plc_obj.options.dry_run=True
436                                 plc_obj.apiserver.set_dry_run(True)
437                                 if not cross:   step_result=method(plc_obj)
438                                 else:           step_result=method(plc_obj,across_plcs)
439                                 print 'dry_run step ->',step_result
440                                 self.options.dry_run=dry_run
441                                 plc_obj.options.dry_run=dry_run
442                                 plc_obj.apiserver.set_dry_run(dry_run)
443                             elif answer in ['p']:
444                                 # take it as a yes and leave interactive mode
445                                 prompting=False
446                                 self.options.interactive=False
447                             elif answer in ['r','y']:   # run/yes
448                                 prompting=False
449                     if skip_step:
450                         continue
451                     try:
452                         force_msg=""
453                         if force and spec['failed_step']: force_msg=" (forced after %s has failed)"%spec['failed_step']
454                         utils.header("********** %d RUNNING step %s%s on plc %s"%(plc_counter,stepname,force_msg,plcname))
455                         if not cross:   step_result = method(plc_obj)
456                         else:           step_result = method(plc_obj,across_plcs)
457                         if step_result:
458                             utils.header('********** %d SUCCESSFUL step %s on %s'%(plc_counter,stepname,plcname))
459                             status="OK"
460                         else:
461                             overall_result = False
462                             spec['failed_step'] = stepname
463                             utils.header('********** %d FAILED Step %s on %s (discarded from further steps)'\
464                                              %(plc_counter,stepname,plcname))
465                             status="KO"
466                     except:
467                         overall_result=False
468                         spec['failed_step'] = stepname
469                         traceback.print_exc()
470                         utils.header ('********** %d FAILED (exception) Step %s on %s (discarded from further steps)'\
471                                           %(plc_counter,stepname,plcname))
472                         status="KO"
473
474                 # do not run, just display it's skipped
475                 else:
476                     why="has failed %s"%spec['failed_step']
477                     utils.header("********** %d SKIPPED Step %s on %s (%s)"%(plc_counter,stepname,plcname,why))
478                     status="UNDEF"
479                 if not self.options.dry_run:
480                     end=strftime(TIME_FORMAT)
481                     # always do this on stdout
482                     print TRACE_FORMAT%locals()
483                     # duplicate on trace_file if provided
484                     if self.options.trace_file:
485                         trace.write(TRACE_FORMAT%locals())
486                         trace.flush()
487
488         if self.options.trace_file and not self.options.dry_run:
489             trace.close()
490
491         # free local substrate
492         LocalSubstrate.local_substrate.release(self.options)
493         
494         return overall_result
495
496     # wrapper to run, returns a shell-compatible result
497     def main(self):
498         try:
499             success=self.run()
500             if success:
501                 return 0
502             else:
503                 return 1 
504         except SystemExit:
505             print 'Caught SystemExit'
506             raise
507         except:
508             traceback.print_exc()
509             return 2
510
511 if __name__ == "__main__":
512     sys.exit(TestMain().main())