1 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
2 # Copyright (C) 2010 INRIA
9 from datetime import datetime, timedelta
10 from types import StringTypes
13 from TestSite import TestSite
14 from TestNode import TestNode
15 from TestUser import TestUser
16 from TestKey import TestKey
17 from TestSlice import TestSlice
18 from TestSliver import TestSliver
19 from TestBoxQemu import TestBoxQemu
20 from TestSsh import TestSsh
21 from TestApiserver import TestApiserver
22 from TestAuthSfa import TestAuthSfa
23 from PlcapiUrlScanner import PlcapiUrlScanner
24 from Completer import Completer, CompleterTask
26 # step methods must take (self) and return a boolean (options is a member of the class)
28 def standby(minutes,dry_run):
29 utils.header('Entering StandBy for %d mn'%minutes)
33 time.sleep(60*minutes)
36 def standby_generic (func):
38 minutes=int(func.__name__.split("_")[1])
39 return standby(minutes,self.options.dry_run)
42 def node_mapper (method):
43 def actual(self,*args, **kwds):
45 node_method = TestNode.__dict__[method.__name__]
46 for test_node in self.all_nodes():
47 if not node_method(test_node, *args, **kwds): overall=False
49 # restore the doc text
50 actual.__doc__=TestNode.__dict__[method.__name__].__doc__
53 def slice_mapper (method):
56 slice_method = TestSlice.__dict__[method.__name__]
57 for slice_spec in self.plc_spec['slices']:
58 site_spec = self.locate_site (slice_spec['sitename'])
59 test_site = TestSite(self,site_spec)
60 test_slice=TestSlice(self,test_site,slice_spec)
61 if not slice_method(test_slice,self.options): overall=False
63 # restore the doc text
64 actual.__doc__=TestSlice.__dict__[method.__name__].__doc__
67 # run a step but return True so that we can go on
68 def ignore_result (method):
70 # ssh_slice_ignore->ssh_slice
71 ref_name=method.__name__.replace('_ignore','').replace('force_','')
72 ref_method=TestPlc.__dict__[ref_name]
73 result=ref_method(self)
74 print "Actual (but ignored) result for %(ref_name)s is %(result)s"%locals()
75 return Ignored (result)
76 wrappee.__doc__="ignored version of " + method.__name__.replace('_ignore','').replace('ignore_','')
79 # a variant that expects the TestSlice method to return a list of CompleterTasks that
80 # are then merged into a single Completer run to avoid wating for all the slices
81 # esp. useful when a test fails of course
82 # because we need to pass arguments we use a class instead..
83 class slice_mapper__tasks (object):
84 # could not get this to work with named arguments
85 def __init__ (self,timeout_minutes,silent_minutes,period_seconds):
86 self.timeout=timedelta(minutes=timeout_minutes)
87 self.silent=timedelta(minutes=silent_minutes)
88 self.period=timedelta(seconds=period_seconds)
89 def __call__ (self, method):
91 # compute augmented method name
92 method_name = method.__name__ + "__tasks"
94 slice_method = TestSlice.__dict__[ method_name ]
97 for slice_spec in self.plc_spec['slices']:
98 site_spec = self.locate_site (slice_spec['sitename'])
99 test_site = TestSite(self,site_spec)
100 test_slice=TestSlice(self,test_site,slice_spec)
101 tasks += slice_method (test_slice, self.options)
102 return Completer (tasks).run (decorator_self.timeout, decorator_self.silent, decorator_self.period)
103 # restore the doc text from the TestSlice method even if a bit odd
104 wrappee.__doc__ = slice_method.__doc__
107 def auth_sfa_mapper (method):
110 auth_method = TestAuthSfa.__dict__[method.__name__]
111 for auth_spec in self.plc_spec['sfa']['auth_sfa_specs']:
112 test_auth=TestAuthSfa(self,auth_spec)
113 if not auth_method(test_auth,self.options): overall=False
115 # restore the doc text
116 actual.__doc__=TestAuthSfa.__dict__[method.__name__].__doc__
120 def __init__ (self,result):
130 'vs_delete','timestamp_vs','vs_create', SEP,
131 'plc_install', 'plc_configure', 'plc_start', SEP,
132 'keys_fetch', 'keys_store', 'keys_clear_known_hosts', SEP,
133 'plcapi_urls','speed_up_slices', SEP,
134 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', 'leases', SEP,
135 # slices created under plcsh interactively seem to be fine but these ones don't have the tags
136 # keep this our of the way for now
137 # 'check_vsys_defaults', SEP,
138 'nodestate_reinstall', 'qemu_local_init','bootcd', 'qemu_local_config', SEP,
139 'qemu_kill_mine','qemu_clean_mine', 'qemu_export', 'qemu_start', 'timestamp_qemu', SEP,
140 'sfa_install_all', 'sfa_configure', 'cross_sfa_configure', 'sfa_start', 'sfa_import', SEPSFA,
141 'sfi_configure@1', 'sfa_add_site@1','sfa_add_pi@1', SEPSFA,
142 'sfa_add_user@1', 'sfa_update_user@1', 'sfa_add_slice@1', 'sfa_renew_slice@1', SEPSFA,
143 'sfa_discover@1', 'sfa_create_slice@1', 'sfa_check_slice_plc@1', 'sfa_update_slice@1', SEPSFA,
144 'sfi_list@1', 'sfi_show@1', 'sfa_utest@1', SEPSFA,
145 # we used to run plcsh_stress_test, and then ssh_node_debug and ssh_node_boot
146 # but as the stress test might take a while, we sometimes missed the debug mode..
147 'ssh_node_debug@1', 'plcsh_stress_test@1', SEP,
148 'ssh_node_boot@1', 'node_bmlogs@1', 'ssh_slice', 'ssh_slice_basics', 'check_initscripts_ignore', SEP,
149 'ssh_slice_sfa@1', 'sfa_delete_slice@1', 'sfa_delete_user@1', SEPSFA,
150 'cross_check_tcp@1', 'check_system_slice', SEP,
151 # check slices are turned off properly
152 'empty_slices', 'ssh_slice_off', SEP,
153 # check they are properly re-created with the same name
154 'fill_slices', 'ssh_slice_again_ignore', SEP,
155 'gather_logs_force', SEP,
158 'export', 'show_boxes', SEP,
159 'check_hooks', 'plc_stop', 'vs_start', 'vs_stop', SEP,
160 'delete_initscripts', 'delete_nodegroups','delete_all_sites', SEP,
161 'delete_sites', 'delete_nodes', 'delete_slices', 'keys_clean', SEP,
162 'delete_leases', 'list_leases', SEP,
164 'nodestate_show','nodestate_safeboot','nodestate_boot', SEP,
165 'qemu_list_all', 'qemu_list_mine', 'qemu_kill_all', SEP,
166 'sfa_install_core', 'sfa_install_sfatables', 'sfa_install_plc', 'sfa_install_client', SEPSFA,
167 'sfa_plcclean', 'sfa_dbclean', 'sfa_stop','sfa_uninstall', 'sfi_clean', SEPSFA,
168 'plc_db_dump' , 'plc_db_restore', SEP,
169 'check_netflow','check_drl', SEP,
170 'debug_nodemanager', SEP,
171 'standby_1_through_20','yes','no',SEP,
175 def printable_steps (list):
176 single_line=" ".join(list)+" "
177 return single_line.replace(" "+SEP+" "," \\\n").replace(" "+SEPSFA+" "," \\\n")
179 def valid_step (step):
180 return step != SEP and step != SEPSFA
182 # turn off the sfa-related steps when build has skipped SFA
183 # this was originally for centos5 but is still valid
184 # for up to f12 as recent SFAs with sqlalchemy won't build before f14
186 def check_whether_build_has_sfa (rpms_url):
187 utils.header ("Checking if build provides SFA package...")
188 # warning, we're now building 'sface' so let's be a bit more picky
189 retcod=os.system ("curl --silent %s/ | grep -q sfa-"%rpms_url)
190 # full builds are expected to return with 0 here
192 utils.header("build does provide SFA")
194 # move all steps containing 'sfa' from default_steps to other_steps
195 utils.header("SFA package not found - removing steps with sfa or sfi")
196 sfa_steps= [ step for step in TestPlc.default_steps if step.find('sfa')>=0 or step.find("sfi")>=0 ]
197 TestPlc.other_steps += sfa_steps
198 for step in sfa_steps: TestPlc.default_steps.remove(step)
200 def __init__ (self,plc_spec,options):
201 self.plc_spec=plc_spec
203 self.test_ssh=TestSsh(self.plc_spec['host_box'],self.options.buildname)
204 self.vserverip=plc_spec['vserverip']
205 self.vservername=plc_spec['vservername']
206 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
207 self.apiserver=TestApiserver(self.url,options.dry_run)
208 (self.ssh_node_boot_timeout,self.ssh_node_boot_silent)=plc_spec['ssh_node_boot_timers']
209 (self.ssh_node_debug_timeout,self.ssh_node_debug_silent)=plc_spec['ssh_node_debug_timers']
211 def has_addresses_api (self):
212 return self.apiserver.has_method('AddIpAddress')
215 name=self.plc_spec['name']
216 return "%s.%s"%(name,self.vservername)
219 return self.plc_spec['host_box']
222 return self.test_ssh.is_local()
224 # define the API methods on this object through xmlrpc
225 # would help, but not strictly necessary
229 def actual_command_in_guest (self,command, backslash=False):
230 raw1=self.host_to_guest(command)
231 raw2=self.test_ssh.actual_command(raw1,dry_run=self.options.dry_run, backslash=backslash)
234 def start_guest (self):
235 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host(),dry_run=self.options.dry_run))
237 def stop_guest (self):
238 return utils.system(self.test_ssh.actual_command(self.stop_guest_in_host(),dry_run=self.options.dry_run))
240 def run_in_guest (self,command,backslash=False):
241 raw=self.actual_command_in_guest(command,backslash)
242 return utils.system(raw)
244 def run_in_host (self,command):
245 return self.test_ssh.run_in_buildname(command, dry_run=self.options.dry_run)
247 # backslashing turned out so awful at some point that I've turned off auto-backslashing
248 # see e.g. plc_start esp. the version for f14
249 #command gets run in the plc's vm
250 def host_to_guest(self,command):
251 # f14 still needs some extra help
252 if self.options.fcdistro == 'f14':
253 raw="virsh -c lxc:/// lxc-enter-namespace %s -- /usr/bin/env PATH=/bin:/sbin:/usr/bin:/usr/sbin %s" %(self.vservername,command)
255 raw="virsh -c lxc:/// lxc-enter-namespace %s -- /usr/bin/env %s" %(self.vservername,command)
258 # this /vservers thing is legacy...
259 def vm_root_in_host(self):
260 return "/vservers/%s/"%(self.vservername)
262 def vm_timestamp_path (self):
263 return "/vservers/%s/%s.timestamp"%(self.vservername,self.vservername)
265 #start/stop the vserver
266 def start_guest_in_host(self):
267 return "virsh -c lxc:/// start %s"%(self.vservername)
269 def stop_guest_in_host(self):
270 return "virsh -c lxc:/// destroy %s"%(self.vservername)
273 def run_in_guest_piped (self,local,remote):
274 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
276 def yum_check_installed (self, rpms):
277 if isinstance (rpms, list):
279 return self.run_in_guest("rpm -q %s"%rpms)==0
281 # does a yum install in the vs, ignore yum retcod, check with rpm
282 def yum_install (self, rpms):
283 if isinstance (rpms, list):
285 self.run_in_guest("yum -y install %s"%rpms)
286 # yum-complete-transaction comes with yum-utils, that is in vtest.pkgs
287 self.run_in_guest("yum-complete-transaction -y")
288 return self.yum_check_installed (rpms)
290 def auth_root (self):
291 return {'Username':self.plc_spec['PLC_ROOT_USER'],
292 'AuthMethod':'password',
293 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
294 'Role' : self.plc_spec['role']
296 def locate_site (self,sitename):
297 for site in self.plc_spec['sites']:
298 if site['site_fields']['name'] == sitename:
300 if site['site_fields']['login_base'] == sitename:
302 raise Exception,"Cannot locate site %s"%sitename
304 def locate_node (self,nodename):
305 for site in self.plc_spec['sites']:
306 for node in site['nodes']:
307 if node['name'] == nodename:
309 raise Exception,"Cannot locate node %s"%nodename
311 def locate_hostname (self,hostname):
312 for site in self.plc_spec['sites']:
313 for node in site['nodes']:
314 if node['node_fields']['hostname'] == hostname:
316 raise Exception,"Cannot locate hostname %s"%hostname
318 def locate_key (self,key_name):
319 for key in self.plc_spec['keys']:
320 if key['key_name'] == key_name:
322 raise Exception,"Cannot locate key %s"%key_name
324 def locate_private_key_from_key_names (self, key_names):
325 # locate the first avail. key
327 for key_name in key_names:
328 key_spec=self.locate_key(key_name)
329 test_key=TestKey(self,key_spec)
330 publickey=test_key.publicpath()
331 privatekey=test_key.privatepath()
332 if os.path.isfile(publickey) and os.path.isfile(privatekey):
334 if found: return privatekey
337 def locate_slice (self, slicename):
338 for slice in self.plc_spec['slices']:
339 if slice['slice_fields']['name'] == slicename:
341 raise Exception,"Cannot locate slice %s"%slicename
343 def all_sliver_objs (self):
345 for slice_spec in self.plc_spec['slices']:
346 slicename = slice_spec['slice_fields']['name']
347 for nodename in slice_spec['nodenames']:
348 result.append(self.locate_sliver_obj (nodename,slicename))
351 def locate_sliver_obj (self,nodename,slicename):
352 (site,node) = self.locate_node(nodename)
353 slice = self.locate_slice (slicename)
355 test_site = TestSite (self, site)
356 test_node = TestNode (self, test_site,node)
357 # xxx the slice site is assumed to be the node site - mhh - probably harmless
358 test_slice = TestSlice (self, test_site, slice)
359 return TestSliver (self, test_node, test_slice)
361 def locate_first_node(self):
362 nodename=self.plc_spec['slices'][0]['nodenames'][0]
363 (site,node) = self.locate_node(nodename)
364 test_site = TestSite (self, site)
365 test_node = TestNode (self, test_site,node)
368 def locate_first_sliver (self):
369 slice_spec=self.plc_spec['slices'][0]
370 slicename=slice_spec['slice_fields']['name']
371 nodename=slice_spec['nodenames'][0]
372 return self.locate_sliver_obj(nodename,slicename)
374 # all different hostboxes used in this plc
375 def get_BoxNodes(self):
376 # maps on sites and nodes, return [ (host_box,test_node) ]
378 for site_spec in self.plc_spec['sites']:
379 test_site = TestSite (self,site_spec)
380 for node_spec in site_spec['nodes']:
381 test_node = TestNode (self, test_site, node_spec)
382 if not test_node.is_real():
383 tuples.append( (test_node.host_box(),test_node) )
384 # transform into a dict { 'host_box' -> [ test_node .. ] }
386 for (box,node) in tuples:
387 if not result.has_key(box):
390 result[box].append(node)
393 # a step for checking this stuff
394 def show_boxes (self):
395 'print summary of nodes location'
396 for (box,nodes) in self.get_BoxNodes().iteritems():
397 print box,":"," + ".join( [ node.name() for node in nodes ] )
400 # make this a valid step
401 def qemu_kill_all(self):
402 'kill all qemu instances on the qemu boxes involved by this setup'
403 # this is the brute force version, kill all qemus on that host box
404 for (box,nodes) in self.get_BoxNodes().iteritems():
405 # pass the first nodename, as we don't push template-qemu on testboxes
406 nodedir=nodes[0].nodedir()
407 TestBoxQemu(box,self.options.buildname).qemu_kill_all(nodedir)
410 # make this a valid step
411 def qemu_list_all(self):
412 'list all qemu instances on the qemu boxes involved by this setup'
413 for (box,nodes) in self.get_BoxNodes().iteritems():
414 # this is the brute force version, kill all qemus on that host box
415 TestBoxQemu(box,self.options.buildname).qemu_list_all()
418 # kill only the qemus related to this test
419 def qemu_list_mine(self):
420 'list qemu instances for our nodes'
421 for (box,nodes) in self.get_BoxNodes().iteritems():
422 # the fine-grain version
427 # kill only the qemus related to this test
428 def qemu_clean_mine(self):
429 'cleanup (rm -rf) qemu instances for our nodes'
430 for (box,nodes) in self.get_BoxNodes().iteritems():
431 # the fine-grain version
436 # kill only the right qemus
437 def qemu_kill_mine(self):
438 'kill the qemu instances for our nodes'
439 for (box,nodes) in self.get_BoxNodes().iteritems():
440 # the fine-grain version
445 #################### display config
447 "show test configuration after localization"
452 # uggly hack to make sure 'run export' only reports about the 1st plc
453 # to avoid confusion - also we use 'inri_slice1' in various aliases..
456 "print cut'n paste-able stuff to export env variables to your shell"
457 # guess local domain from hostname
458 if TestPlc.exported_id>1:
459 print "export GUESTHOSTNAME%d=%s"%(TestPlc.exported_id,self.plc_spec['vservername'])
461 TestPlc.exported_id+=1
462 domain=socket.gethostname().split('.',1)[1]
463 fqdn="%s.%s"%(self.plc_spec['host_box'],domain)
464 print "export BUILD=%s"%self.options.buildname
465 print "export PLCHOSTLXC=%s"%fqdn
466 print "export GUESTNAME=%s"%self.plc_spec['vservername']
467 vplcname=self.plc_spec['vservername'].split('-')[-1]
468 print "export GUESTHOSTNAME=%s.%s"%(vplcname,domain)
469 # find hostname of first node
470 (hostname,qemubox) = self.all_node_infos()[0]
471 print "export KVMHOST=%s.%s"%(qemubox,domain)
472 print "export NODE=%s"%(hostname)
476 always_display_keys=['PLC_WWW_HOST','nodes','sites',]
477 def show_pass (self,passno):
478 for (key,val) in self.plc_spec.iteritems():
479 if not self.options.verbose and key not in TestPlc.always_display_keys: continue
483 self.display_site_spec(site)
484 for node in site['nodes']:
485 self.display_node_spec(node)
486 elif key=='initscripts':
487 for initscript in val:
488 self.display_initscript_spec (initscript)
491 self.display_slice_spec (slice)
494 self.display_key_spec (key)
496 if key not in ['sites','initscripts','slices','keys', 'sfa']:
497 print '+ ',key,':',val
499 def display_site_spec (self,site):
500 print '+ ======== site',site['site_fields']['name']
501 for (k,v) in site.iteritems():
502 if not self.options.verbose and k not in TestPlc.always_display_keys: continue
505 print '+ ','nodes : ',
507 print node['node_fields']['hostname'],'',
513 print user['name'],'',
515 elif k == 'site_fields':
516 print '+ login_base',':',v['login_base']
517 elif k == 'address_fields':
523 def display_initscript_spec (self,initscript):
524 print '+ ======== initscript',initscript['initscript_fields']['name']
526 def display_key_spec (self,key):
527 print '+ ======== key',key['key_name']
529 def display_slice_spec (self,slice):
530 print '+ ======== slice',slice['slice_fields']['name']
531 for (k,v) in slice.iteritems():
544 elif k=='slice_fields':
545 print '+ fields',':',
546 print 'max_nodes=',v['max_nodes'],
551 def display_node_spec (self,node):
552 print "+ node=%s host_box=%s"%(node['name'],node['host_box']),
553 print "hostname=",node['node_fields']['hostname'],
554 print "ip=",node['interface_fields']['ip']
555 if self.options.verbose:
556 utils.pprint("node details",node,depth=3)
558 # another entry point for just showing the boxes involved
559 def display_mapping (self):
560 TestPlc.display_mapping_plc(self.plc_spec)
564 def display_mapping_plc (plc_spec):
565 print '+ MyPLC',plc_spec['name']
566 # WARNING this would not be right for lxc-based PLC's - should be harmless though
567 print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['host_box'],plc_spec['vservername'])
568 print '+\tIP = %s/%s'%(plc_spec['PLC_API_HOST'],plc_spec['vserverip'])
569 for site_spec in plc_spec['sites']:
570 for node_spec in site_spec['nodes']:
571 TestPlc.display_mapping_node(node_spec)
574 def display_mapping_node (node_spec):
575 print '+ NODE %s'%(node_spec['name'])
576 print '+\tqemu box %s'%node_spec['host_box']
577 print '+\thostname=%s'%node_spec['node_fields']['hostname']
579 # write a timestamp in /vservers/<>.timestamp
580 # cannot be inside the vserver, that causes vserver .. build to cough
581 def timestamp_vs (self):
582 "Create a timestamp to remember creation date for this plc"
584 # TODO-lxc check this one
585 # a first approx. is to store the timestamp close to the VM root like vs does
586 stamp_path=self.vm_timestamp_path ()
587 stamp_dir = os.path.dirname (stamp_path)
588 utils.system(self.test_ssh.actual_command("mkdir -p %s"%stamp_dir))
589 return utils.system(self.test_ssh.actual_command("echo %d > %s"%(now,stamp_path)))==0
591 # this is called inconditionnally at the beginning of the test sequence
592 # just in case this is a rerun, so if the vm is not running it's fine
594 "vserver delete the test myplc"
595 stamp_path=self.vm_timestamp_path()
596 self.run_in_host("rm -f %s"%stamp_path)
597 self.run_in_host("virsh -c lxc:// destroy %s"%self.vservername)
598 self.run_in_host("virsh -c lxc:// undefine %s"%self.vservername)
599 self.run_in_host("rm -fr /vservers/%s"%self.vservername)
603 # historically the build was being fetched by the tests
604 # now the build pushes itself as a subdir of the tests workdir
605 # so that the tests do not have to worry about extracting the build (svn, git, or whatever)
606 def vs_create (self):
607 "vserver creation (no install done)"
608 # push the local build/ dir to the testplc box
610 # a full path for the local calls
611 build_dir=os.path.dirname(sys.argv[0])
612 # sometimes this is empty - set to "." in such a case
613 if not build_dir: build_dir="."
614 build_dir += "/build"
616 # use a standard name - will be relative to remote buildname
618 # remove for safety; do *not* mkdir first, otherwise we end up with build/build/
619 self.test_ssh.rmdir(build_dir)
620 self.test_ssh.copy(build_dir,recursive=True)
621 # the repo url is taken from arch-rpms-url
622 # with the last step (i386) removed
623 repo_url = self.options.arch_rpms_url
624 for level in [ 'arch' ]:
625 repo_url = os.path.dirname(repo_url)
627 # invoke initvm (drop support for vs)
628 script="lbuild-initvm.sh"
630 # pass the vbuild-nightly options to [lv]test-initvm
631 script_options += " -p %s"%self.options.personality
632 script_options += " -d %s"%self.options.pldistro
633 script_options += " -f %s"%self.options.fcdistro
634 script_options += " -r %s"%repo_url
635 vserver_name = self.vservername
637 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
638 script_options += " -n %s"%vserver_hostname
640 print "Cannot reverse lookup %s"%self.vserverip
641 print "This is considered fatal, as this might pollute the test results"
643 create_vserver="%(build_dir)s/%(script)s %(script_options)s %(vserver_name)s"%locals()
644 return self.run_in_host(create_vserver) == 0
647 def plc_install(self):
648 "yum install myplc, noderepo, and the plain bootstrapfs"
650 # workaround for getting pgsql8.2 on centos5
651 if self.options.fcdistro == "centos5":
652 self.run_in_guest("rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-3.noarch.rpm")
655 if self.options.personality == "linux32":
657 elif self.options.personality == "linux64":
660 raise Exception, "Unsupported personality %r"%self.options.personality
661 nodefamily="%s-%s-%s"%(self.options.pldistro,self.options.fcdistro,arch)
664 pkgs_list.append ("slicerepo-%s"%nodefamily)
665 pkgs_list.append ("myplc")
666 pkgs_list.append ("noderepo-%s"%nodefamily)
667 pkgs_list.append ("nodeimage-%s-plain"%nodefamily)
668 pkgs_string=" ".join(pkgs_list)
669 return self.yum_install (pkgs_list)
672 def mod_python(self):
673 """yum install mod_python, useful on f18 and above so as to avoid broken wsgi"""
674 return self.yum_install ( [ 'mod_python' ] )
677 def plc_configure(self):
679 tmpname='%s.plc-config-tty'%(self.name())
680 fileconf=open(tmpname,'w')
681 for var in [ 'PLC_NAME',
686 'PLC_MAIL_SUPPORT_ADDRESS',
689 # Above line was added for integrating SFA Testing
695 'PLC_RESERVATION_GRANULARITY',
697 'PLC_OMF_XMPP_SERVER',
700 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
701 fileconf.write('w\n')
702 fileconf.write('q\n')
704 utils.system('cat %s'%tmpname)
705 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
706 utils.system('rm %s'%tmpname)
709 # f14 is a bit odd in this respect, although this worked fine in guests up to f18
710 # however using a vplc guest under f20 requires this trick
711 # the symptom is this: service plc start
712 # Starting plc (via systemctl): Failed to get D-Bus connection: \
713 # Failed to connect to socket /org/freedesktop/systemd1/private: Connection refused
714 # weird thing is the doc says f14 uses upstart by default and not systemd
715 # so this sounds kind of harmless
716 def start_service (self,service): return self.start_stop_service (service,'start')
717 def stop_service (self,service): return self.start_stop_service (service,'stop')
719 def start_stop_service (self, service,start_or_stop):
720 "utility to start/stop a service with the special trick for f14"
721 if self.options.fcdistro != 'f14':
722 return self.run_in_guest ("service %s %s"%(service,start_or_stop))==0
724 # patch /sbin/service so it does not reset environment
725 self.run_in_guest ('sed -i -e \\"s,env -i,env,\\" /sbin/service')
726 # this is because our own scripts in turn call service
727 return self.run_in_guest("SYSTEMCTL_SKIP_REDIRECT=true service %s %s"%(service,start_or_stop))==0
731 return self.start_service ('plc')
735 return self.stop_service ('plc')
738 "start the PLC vserver"
743 "stop the PLC vserver"
747 # stores the keys from the config for further use
748 def keys_store(self):
749 "stores test users ssh keys in keys/"
750 for key_spec in self.plc_spec['keys']:
751 TestKey(self,key_spec).store_key()
754 def keys_clean(self):
755 "removes keys cached in keys/"
756 utils.system("rm -rf ./keys")
759 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
760 # for later direct access to the nodes
761 def keys_fetch(self):
762 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
764 if not os.path.isdir(dir):
766 vservername=self.vservername
767 vm_root=self.vm_root_in_host()
769 prefix = 'debug_ssh_key'
770 for ext in [ 'pub', 'rsa' ] :
771 src="%(vm_root)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
772 dst="keys/%(vservername)s-debug.%(ext)s"%locals()
773 if self.test_ssh.fetch(src,dst) != 0: overall=False
777 "create sites with PLCAPI"
778 return self.do_sites()
780 def delete_sites (self):
781 "delete sites with PLCAPI"
782 return self.do_sites(action="delete")
784 def do_sites (self,action="add"):
785 for site_spec in self.plc_spec['sites']:
786 test_site = TestSite (self,site_spec)
787 if (action != "add"):
788 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
789 test_site.delete_site()
790 # deleted with the site
791 #test_site.delete_users()
794 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
795 test_site.create_site()
796 test_site.create_users()
799 def delete_all_sites (self):
800 "Delete all sites in PLC, and related objects"
801 print 'auth_root',self.auth_root()
802 sites = self.apiserver.GetSites(self.auth_root(), {}, ['site_id','login_base'])
804 # keep automatic site - otherwise we shoot in our own foot, root_auth is not valid anymore
805 if site['login_base']==self.plc_spec['PLC_SLICE_PREFIX']: continue
806 site_id=site['site_id']
807 print 'Deleting site_id',site_id
808 self.apiserver.DeleteSite(self.auth_root(),site_id)
812 "create nodes with PLCAPI"
813 return self.do_nodes()
814 def delete_nodes (self):
815 "delete nodes with PLCAPI"
816 return self.do_nodes(action="delete")
818 def do_nodes (self,action="add"):
819 for site_spec in self.plc_spec['sites']:
820 test_site = TestSite (self,site_spec)
822 utils.header("Deleting nodes in site %s"%test_site.name())
823 for node_spec in site_spec['nodes']:
824 test_node=TestNode(self,test_site,node_spec)
825 utils.header("Deleting %s"%test_node.name())
826 test_node.delete_node()
828 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
829 for node_spec in site_spec['nodes']:
830 utils.pprint('Creating node %s'%node_spec,node_spec)
831 test_node = TestNode (self,test_site,node_spec)
832 test_node.create_node ()
835 def nodegroups (self):
836 "create nodegroups with PLCAPI"
837 return self.do_nodegroups("add")
838 def delete_nodegroups (self):
839 "delete nodegroups with PLCAPI"
840 return self.do_nodegroups("delete")
844 def translate_timestamp (start,grain,timestamp):
845 if timestamp < TestPlc.YEAR: return start+timestamp*grain
846 else: return timestamp
849 def timestamp_printable (timestamp):
850 return time.strftime('%m-%d %H:%M:%S UTC',time.gmtime(timestamp))
853 "create leases (on reservable nodes only, use e.g. run -c default -c resa)"
855 grain=self.apiserver.GetLeaseGranularity(self.auth_root())
856 print 'API answered grain=',grain
857 start=(now/grain)*grain
859 # find out all nodes that are reservable
860 nodes=self.all_reservable_nodenames()
862 utils.header ("No reservable node found - proceeding without leases")
865 # attach them to the leases as specified in plc_specs
866 # this is where the 'leases' field gets interpreted as relative of absolute
867 for lease_spec in self.plc_spec['leases']:
868 # skip the ones that come with a null slice id
869 if not lease_spec['slice']: continue
870 lease_spec['t_from']=TestPlc.translate_timestamp(start,grain,lease_spec['t_from'])
871 lease_spec['t_until']=TestPlc.translate_timestamp(start,grain,lease_spec['t_until'])
872 lease_addition=self.apiserver.AddLeases(self.auth_root(),nodes,
873 lease_spec['slice'],lease_spec['t_from'],lease_spec['t_until'])
874 if lease_addition['errors']:
875 utils.header("Cannot create leases, %s"%lease_addition['errors'])
878 utils.header('Leases on nodes %r for %s from %d (%s) until %d (%s)'%\
879 (nodes,lease_spec['slice'],
880 lease_spec['t_from'],TestPlc.timestamp_printable(lease_spec['t_from']),
881 lease_spec['t_until'],TestPlc.timestamp_printable(lease_spec['t_until'])))
885 def delete_leases (self):
886 "remove all leases in the myplc side"
887 lease_ids= [ l['lease_id'] for l in self.apiserver.GetLeases(self.auth_root())]
888 utils.header("Cleaning leases %r"%lease_ids)
889 self.apiserver.DeleteLeases(self.auth_root(),lease_ids)
892 def list_leases (self):
893 "list all leases known to the myplc"
894 leases = self.apiserver.GetLeases(self.auth_root())
897 current=l['t_until']>=now
898 if self.options.verbose or current:
899 utils.header("%s %s from %s until %s"%(l['hostname'],l['name'],
900 TestPlc.timestamp_printable(l['t_from']),
901 TestPlc.timestamp_printable(l['t_until'])))
904 # create nodegroups if needed, and populate
905 def do_nodegroups (self, action="add"):
906 # 1st pass to scan contents
908 for site_spec in self.plc_spec['sites']:
909 test_site = TestSite (self,site_spec)
910 for node_spec in site_spec['nodes']:
911 test_node=TestNode (self,test_site,node_spec)
912 if node_spec.has_key('nodegroups'):
913 nodegroupnames=node_spec['nodegroups']
914 if isinstance(nodegroupnames,StringTypes):
915 nodegroupnames = [ nodegroupnames ]
916 for nodegroupname in nodegroupnames:
917 if not groups_dict.has_key(nodegroupname):
918 groups_dict[nodegroupname]=[]
919 groups_dict[nodegroupname].append(test_node.name())
920 auth=self.auth_root()
922 for (nodegroupname,group_nodes) in groups_dict.iteritems():
924 print 'nodegroups:','dealing with nodegroup',nodegroupname,'on nodes',group_nodes
925 # first, check if the nodetagtype is here
926 tag_types = self.apiserver.GetTagTypes(auth,{'tagname':nodegroupname})
928 tag_type_id = tag_types[0]['tag_type_id']
930 tag_type_id = self.apiserver.AddTagType(auth,
931 {'tagname':nodegroupname,
932 'description': 'for nodegroup %s'%nodegroupname,
934 print 'located tag (type)',nodegroupname,'as',tag_type_id
936 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
938 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
939 print 'created nodegroup',nodegroupname,'from tagname',nodegroupname,'and value','yes'
940 # set node tag on all nodes, value='yes'
941 for nodename in group_nodes:
943 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
945 traceback.print_exc()
946 print 'node',nodename,'seems to already have tag',nodegroupname
949 expect_yes = self.apiserver.GetNodeTags(auth,
950 {'hostname':nodename,
951 'tagname':nodegroupname},
952 ['value'])[0]['value']
953 if expect_yes != "yes":
954 print 'Mismatch node tag on node',nodename,'got',expect_yes
957 if not self.options.dry_run:
958 print 'Cannot find tag',nodegroupname,'on node',nodename
962 print 'cleaning nodegroup',nodegroupname
963 self.apiserver.DeleteNodeGroup(auth,nodegroupname)
965 traceback.print_exc()
969 # a list of TestNode objs
970 def all_nodes (self):
972 for site_spec in self.plc_spec['sites']:
973 test_site = TestSite (self,site_spec)
974 for node_spec in site_spec['nodes']:
975 nodes.append(TestNode (self,test_site,node_spec))
978 # return a list of tuples (nodename,qemuname)
979 def all_node_infos (self) :
981 for site_spec in self.plc_spec['sites']:
982 node_infos += [ (node_spec['node_fields']['hostname'],node_spec['host_box']) \
983 for node_spec in site_spec['nodes'] ]
986 def all_nodenames (self): return [ x[0] for x in self.all_node_infos() ]
987 def all_reservable_nodenames (self):
989 for site_spec in self.plc_spec['sites']:
990 for node_spec in site_spec['nodes']:
991 node_fields=node_spec['node_fields']
992 if 'node_type' in node_fields and node_fields['node_type']=='reservable':
993 res.append(node_fields['hostname'])
996 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
997 def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period_seconds=15):
998 if self.options.dry_run:
1002 class CompleterTaskBootState (CompleterTask):
1003 def __init__ (self, test_plc,hostname):
1004 self.test_plc=test_plc
1005 self.hostname=hostname
1006 self.last_boot_state='undef'
1007 def actual_run (self):
1009 node = self.test_plc.apiserver.GetNodes(self.test_plc.auth_root(), [ self.hostname ],
1011 self.last_boot_state = node['boot_state']
1012 return self.last_boot_state == target_boot_state
1016 return "CompleterTaskBootState with node %s"%self.hostname
1017 def failure_message (self):
1018 return "node %s in state %s - expected %s"%(self.hostname,self.last_boot_state,target_boot_state)
1020 timeout = timedelta(minutes=timeout_minutes)
1021 graceout = timedelta(minutes=silent_minutes)
1022 period = timedelta(seconds=period_seconds)
1023 # the nodes that haven't checked yet - start with a full list and shrink over time
1024 utils.header("checking nodes boot state (expected %s)"%target_boot_state)
1025 tasks = [ CompleterTaskBootState (self,hostname) \
1026 for (hostname,_) in self.all_node_infos() ]
1027 return Completer (tasks).run (timeout, graceout, period)
1029 def nodes_booted(self):
1030 return self.nodes_check_boot_state('boot',timeout_minutes=30,silent_minutes=28)
1032 def check_nodes_ssh(self,debug,timeout_minutes,silent_minutes,period_seconds=15):
1033 class CompleterTaskNodeSsh (CompleterTask):
1034 def __init__ (self, hostname, qemuname, boot_state, local_key):
1035 self.hostname=hostname
1036 self.qemuname=qemuname
1037 self.boot_state=boot_state
1038 self.local_key=local_key
1039 def run (self, silent):
1040 command = TestSsh (self.hostname,key=self.local_key).actual_command("hostname;uname -a")
1041 return utils.system (command, silent=silent)==0
1042 def failure_message (self):
1043 return "Cannot reach %s @ %s in %s mode"%(self.hostname, self.qemuname, self.boot_state)
1046 timeout = timedelta(minutes=timeout_minutes)
1047 graceout = timedelta(minutes=silent_minutes)
1048 period = timedelta(seconds=period_seconds)
1049 vservername=self.vservername
1052 local_key = "keys/%(vservername)s-debug.rsa"%locals()
1055 local_key = "keys/key_admin.rsa"
1056 utils.header("checking ssh access to nodes (expected in %s mode)"%message)
1057 node_infos = self.all_node_infos()
1058 tasks = [ CompleterTaskNodeSsh (nodename, qemuname, message, local_key) \
1059 for (nodename,qemuname) in node_infos ]
1060 return Completer (tasks).run (timeout, graceout, period)
1062 def ssh_node_debug(self):
1063 "Tries to ssh into nodes in debug mode with the debug ssh key"
1064 return self.check_nodes_ssh(debug=True,
1065 timeout_minutes=self.ssh_node_debug_timeout,
1066 silent_minutes=self.ssh_node_debug_silent)
1068 def ssh_node_boot(self):
1069 "Tries to ssh into nodes in production mode with the root ssh key"
1070 return self.check_nodes_ssh(debug=False,
1071 timeout_minutes=self.ssh_node_boot_timeout,
1072 silent_minutes=self.ssh_node_boot_silent)
1074 def node_bmlogs(self):
1075 "Checks that there's a non-empty dir. /var/log/bm/raw"
1076 return utils.system(self.actual_command_in_guest("ls /var/log/bm/raw"))==0
1079 def qemu_local_init (self): pass
1081 def bootcd (self): pass
1083 def qemu_local_config (self): pass
1085 def nodestate_reinstall (self): pass
1087 def nodestate_safeboot (self): pass
1089 def nodestate_boot (self): pass
1091 def nodestate_show (self): pass
1093 def qemu_export (self): pass
1095 ### check hooks : invoke scripts from hooks/{node,slice}
1096 def check_hooks_node (self):
1097 return self.locate_first_node().check_hooks()
1098 def check_hooks_sliver (self) :
1099 return self.locate_first_sliver().check_hooks()
1101 def check_hooks (self):
1102 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
1103 return self.check_hooks_node() and self.check_hooks_sliver()
1106 def do_check_initscripts(self):
1107 class CompleterTaskInitscript (CompleterTask):
1108 def __init__ (self, test_sliver, stamp):
1109 self.test_sliver=test_sliver
1111 def actual_run (self):
1112 return self.test_sliver.check_initscript_stamp (self.stamp)
1114 return "initscript checker for %s"%self.test_sliver.name()
1115 def failure_message (self):
1116 return "initscript stamp %s not found in sliver %s"%(self.stamp,self.test_sliver.name())
1119 for slice_spec in self.plc_spec['slices']:
1120 if not slice_spec.has_key('initscriptstamp'):
1122 stamp=slice_spec['initscriptstamp']
1123 slicename=slice_spec['slice_fields']['name']
1124 for nodename in slice_spec['nodenames']:
1125 print 'nodename',nodename,'slicename',slicename,'stamp',stamp
1126 (site,node) = self.locate_node (nodename)
1127 # xxx - passing the wrong site - probably harmless
1128 test_site = TestSite (self,site)
1129 test_slice = TestSlice (self,test_site,slice_spec)
1130 test_node = TestNode (self,test_site,node)
1131 test_sliver = TestSliver (self, test_node, test_slice)
1132 tasks.append ( CompleterTaskInitscript (test_sliver, stamp))
1133 return Completer (tasks).run (timedelta(minutes=5), timedelta(minutes=4), timedelta(seconds=10))
1135 def check_initscripts(self):
1136 "check that the initscripts have triggered"
1137 return self.do_check_initscripts()
1139 def initscripts (self):
1140 "create initscripts with PLCAPI"
1141 for initscript in self.plc_spec['initscripts']:
1142 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
1143 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
1146 def delete_initscripts (self):
1147 "delete initscripts with PLCAPI"
1148 for initscript in self.plc_spec['initscripts']:
1149 initscript_name = initscript['initscript_fields']['name']
1150 print('Attempting to delete %s in plc %s'%(initscript_name,self.plc_spec['name']))
1152 self.apiserver.DeleteInitScript(self.auth_root(),initscript_name)
1153 print initscript_name,'deleted'
1155 print 'deletion went wrong - probably did not exist'
1160 "create slices with PLCAPI"
1161 return self.do_slices(action="add")
1163 def delete_slices (self):
1164 "delete slices with PLCAPI"
1165 return self.do_slices(action="delete")
1167 def fill_slices (self):
1168 "add nodes in slices with PLCAPI"
1169 return self.do_slices(action="fill")
1171 def empty_slices (self):
1172 "remove nodes from slices with PLCAPI"
1173 return self.do_slices(action="empty")
1175 def do_slices (self, action="add"):
1176 for slice in self.plc_spec['slices']:
1177 site_spec = self.locate_site (slice['sitename'])
1178 test_site = TestSite(self,site_spec)
1179 test_slice=TestSlice(self,test_site,slice)
1180 if action == "delete":
1181 test_slice.delete_slice()
1182 elif action=="fill":
1183 test_slice.add_nodes()
1184 elif action=="empty":
1185 test_slice.delete_nodes()
1187 test_slice.create_slice()
1190 @slice_mapper__tasks(20,10,15)
1191 def ssh_slice(self): pass
1192 @slice_mapper__tasks(20,19,15)
1193 def ssh_slice_off (self): pass
1195 # use another name so we can exclude/ignore it from the tests on the nightly command line
1196 def ssh_slice_again(self): return self.ssh_slice()
1197 # note that simply doing ssh_slice_again=ssh_slice would kind od work too
1198 # but for some reason the ignore-wrapping thing would not
1201 def ssh_slice_basics(self): pass
1204 def check_vsys_defaults(self): pass
1207 def keys_clear_known_hosts (self): pass
1209 def plcapi_urls (self):
1210 return PlcapiUrlScanner (self.auth_root(),ip=self.vserverip).scan()
1212 def speed_up_slices (self):
1213 "tweak nodemanager settings on all nodes using a conf file"
1214 # create the template on the server-side
1215 template="%s.nodemanager"%self.name()
1216 template_file = open (template,"w")
1217 template_file.write('OPTIONS="-p 30 -r 11 -d"\n')
1218 template_file.close()
1219 in_vm="/var/www/html/PlanetLabConf/nodemanager"
1220 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1221 self.test_ssh.copy_abs(template,remote)
1223 self.apiserver.AddConfFile (self.auth_root(),
1224 {'dest':'/etc/sysconfig/nodemanager',
1225 'source':'PlanetLabConf/nodemanager',
1226 'postinstall_cmd':'service nm restart',})
1229 def debug_nodemanager (self):
1230 "sets verbose mode for nodemanager, and speeds up cycle even more (needs speed_up_slices first)"
1231 template="%s.nodemanager"%self.name()
1232 template_file = open (template,"w")
1233 template_file.write('OPTIONS="-p 10 -r 6 -v -d"\n')
1234 template_file.close()
1235 in_vm="/var/www/html/PlanetLabConf/nodemanager"
1236 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1237 self.test_ssh.copy_abs(template,remote)
1241 def qemu_start (self) : pass
1244 def timestamp_qemu (self) : pass
1246 # when a spec refers to a node possibly on another plc
1247 def locate_sliver_obj_cross (self, nodename, slicename, other_plcs):
1248 for plc in [ self ] + other_plcs:
1250 return plc.locate_sliver_obj (nodename, slicename)
1253 raise Exception, "Cannot locate sliver %s@%s among all PLCs"%(nodename,slicename)
1255 # implement this one as a cross step so that we can take advantage of different nodes
1256 # in multi-plcs mode
1257 def cross_check_tcp (self, other_plcs):
1258 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
1259 if 'tcp_specs' not in self.plc_spec or not self.plc_spec['tcp_specs']:
1260 utils.header ("check_tcp: no/empty config found")
1262 specs = self.plc_spec['tcp_specs']
1267 s_test_sliver = self.locate_sliver_obj_cross (spec['server_node'],spec['server_slice'],other_plcs)
1268 if not s_test_sliver.run_tcp_server(port,timeout=20):
1272 # idem for the client side
1273 c_test_sliver = self.locate_sliver_obj_cross (spec['client_node'],spec['client_slice'],other_plcs)
1274 # use nodename from locatesd sliver, unless 'client_connect' is set
1275 if 'client_connect' in spec:
1276 destination = spec['client_connect']
1278 destination=s_test_sliver.test_node.name()
1279 if not c_test_sliver.run_tcp_client(destination,port):
1283 # painfully enough, we need to allow for some time as netflow might show up last
1284 def check_system_slice (self):
1285 "all nodes: check that a system slice is alive"
1286 # netflow currently not working in the lxc distro
1287 # drl not built at all in the wtx distro
1288 # if we find either of them we're happy
1289 return self.check_netflow() or self.check_drl()
1292 def check_netflow (self): return self._check_system_slice ('netflow')
1293 def check_drl (self): return self._check_system_slice ('drl')
1295 # we have the slices up already here, so it should not take too long
1296 def _check_system_slice (self, slicename, timeout_minutes=5, period_seconds=15):
1297 class CompleterTaskSystemSlice (CompleterTask):
1298 def __init__ (self, test_node, dry_run):
1299 self.test_node=test_node
1300 self.dry_run=dry_run
1301 def actual_run (self):
1302 return self.test_node._check_system_slice (slicename, dry_run=self.dry_run)
1304 return "System slice %s @ %s"%(slicename, self.test_node.name())
1305 def failure_message (self):
1306 return "COULD not find system slice %s @ %s"%(slicename, self.test_node.name())
1307 timeout = timedelta(minutes=timeout_minutes)
1308 silent = timedelta (0)
1309 period = timedelta (seconds=period_seconds)
1310 tasks = [ CompleterTaskSystemSlice (test_node, self.options.dry_run) \
1311 for test_node in self.all_nodes() ]
1312 return Completer (tasks) . run (timeout, silent, period)
1314 def plcsh_stress_test (self):
1315 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
1316 # install the stress-test in the plc image
1317 location = "/usr/share/plc_api/plcsh_stress_test.py"
1318 remote="%s/%s"%(self.vm_root_in_host(),location)
1319 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1321 command += " -- --check"
1322 if self.options.size == 1:
1323 command += " --tiny"
1324 return ( self.run_in_guest(command) == 0)
1326 # populate runs the same utility without slightly different options
1327 # in particular runs with --preserve (dont cleanup) and without --check
1328 # also it gets run twice, once with the --foreign option for creating fake foreign entries
1330 def sfa_install_all (self):
1331 "yum install sfa sfa-plc sfa-sfatables sfa-client"
1332 return self.yum_install ("sfa sfa-plc sfa-sfatables sfa-client")
1334 def sfa_install_core(self):
1336 return self.yum_install ("sfa")
1338 def sfa_install_plc(self):
1339 "yum install sfa-plc"
1340 return self.yum_install("sfa-plc")
1342 def sfa_install_sfatables(self):
1343 "yum install sfa-sfatables"
1344 return self.yum_install ("sfa-sfatables")
1346 # for some very odd reason, this sometimes fails with the following symptom
1347 # # yum install sfa-client
1348 # Setting up Install Process
1350 # Downloading Packages:
1351 # Running rpm_check_debug
1352 # Running Transaction Test
1353 # Transaction Test Succeeded
1354 # Running Transaction
1355 # Transaction couldn't start:
1356 # installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem
1357 # [('installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem', (9, '/', 69632L))]
1358 # even though in the same context I have
1359 # [2012.05.23--f14-32-sfastd1-1-vplc07] / # df -h
1360 # Filesystem Size Used Avail Use% Mounted on
1361 # /dev/hdv1 806G 264G 501G 35% /
1362 # none 16M 36K 16M 1% /tmp
1364 # so as a workaround, we first try yum install, and then invoke rpm on the cached rpm...
1365 def sfa_install_client(self):
1366 "yum install sfa-client"
1367 first_try=self.yum_install("sfa-client")
1368 if first_try: return True
1369 utils.header ("********** Regular yum failed - special workaround in place, 2nd chance")
1370 (code,cached_rpm_path)=utils.output_of(self.actual_command_in_guest('find /var/cache/yum -name sfa-client\*.rpm'))
1371 utils.header("rpm_path=<<%s>>"%rpm_path)
1373 self.run_in_guest("rpm -i %s"%cached_rpm_path)
1374 return self.yum_check_installed ("sfa-client")
1376 def sfa_dbclean(self):
1377 "thoroughly wipes off the SFA database"
1378 return self.run_in_guest("sfaadmin reg nuke")==0 or \
1379 self.run_in_guest("sfa-nuke.py")==0 or \
1380 self.run_in_guest("sfa-nuke-plc.py")==0
1382 def sfa_fsclean(self):
1383 "cleanup /etc/sfa/trusted_roots and /var/lib/sfa"
1384 self.run_in_guest("rm -rf /etc/sfa/trusted_roots /var/lib/sfa/authorities")
1387 def sfa_plcclean(self):
1388 "cleans the PLC entries that were created as a side effect of running the script"
1390 sfa_spec=self.plc_spec['sfa']
1392 for auth_sfa_spec in sfa_spec['auth_sfa_specs']:
1393 login_base=auth_sfa_spec['login_base']
1394 try: self.apiserver.DeleteSite (self.auth_root(),login_base)
1395 except: print "Site %s already absent from PLC db"%login_base
1397 for spec_name in ['pi_spec','user_spec']:
1398 user_spec=auth_sfa_spec[spec_name]
1399 username=user_spec['email']
1400 try: self.apiserver.DeletePerson(self.auth_root(),username)
1402 # this in fact is expected as sites delete their members
1403 #print "User %s already absent from PLC db"%username
1406 print "REMEMBER TO RUN sfa_import AGAIN"
1409 def sfa_uninstall(self):
1410 "uses rpm to uninstall sfa - ignore result"
1411 self.run_in_guest("rpm -e sfa sfa-sfatables sfa-client sfa-plc")
1412 self.run_in_guest("rm -rf /var/lib/sfa")
1413 self.run_in_guest("rm -rf /etc/sfa")
1414 self.run_in_guest("rm -rf /var/log/sfa_access.log /var/log/sfa_import_plc.log /var/log/sfa.daemon")
1416 self.run_in_guest("rpm -e --noscripts sfa-plc")
1419 ### run unit tests for SFA
1420 # NOTE: for some reason on f14/i386, yum install sfa-tests fails for no reason
1421 # Running Transaction
1422 # Transaction couldn't start:
1423 # installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem
1424 # [('installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem', (9, '/', 208896L))]
1425 # no matter how many Gbs are available on the testplc
1426 # could not figure out what's wrong, so...
1427 # if the yum install phase fails, consider the test is successful
1428 # other combinations will eventually run it hopefully
1429 def sfa_utest(self):
1430 "yum install sfa-tests and run SFA unittests"
1431 self.run_in_guest("yum -y install sfa-tests")
1432 # failed to install - forget it
1433 if self.run_in_guest("rpm -q sfa-tests")!=0:
1434 utils.header("WARNING: SFA unit tests failed to install, ignoring")
1436 return self.run_in_guest("/usr/share/sfa/tests/testAll.py")==0
1440 dirname="conf.%s"%self.plc_spec['name']
1441 if not os.path.isdir(dirname):
1442 utils.system("mkdir -p %s"%dirname)
1443 if not os.path.isdir(dirname):
1444 raise Exception,"Cannot create config dir for plc %s"%self.name()
1447 def conffile(self,filename):
1448 return "%s/%s"%(self.confdir(),filename)
1449 def confsubdir(self,dirname,clean,dry_run=False):
1450 subdirname="%s/%s"%(self.confdir(),dirname)
1452 utils.system("rm -rf %s"%subdirname)
1453 if not os.path.isdir(subdirname):
1454 utils.system("mkdir -p %s"%subdirname)
1455 if not dry_run and not os.path.isdir(subdirname):
1456 raise "Cannot create config subdir %s for plc %s"%(dirname,self.name())
1459 def conffile_clean (self,filename):
1460 filename=self.conffile(filename)
1461 return utils.system("rm -rf %s"%filename)==0
1464 def sfa_configure(self):
1465 "run sfa-config-tty"
1466 tmpname=self.conffile("sfa-config-tty")
1467 fileconf=open(tmpname,'w')
1468 for var in [ 'SFA_REGISTRY_ROOT_AUTH',
1469 'SFA_INTERFACE_HRN',
1470 'SFA_REGISTRY_LEVEL1_AUTH',
1471 'SFA_REGISTRY_HOST',
1472 'SFA_AGGREGATE_HOST',
1482 'SFA_GENERIC_FLAVOUR',
1483 'SFA_AGGREGATE_ENABLED',
1485 if self.plc_spec['sfa'].has_key(var):
1486 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec['sfa'][var]))
1487 # the way plc_config handles booleans just sucks..
1490 if self.plc_spec['sfa'][var]: val='true'
1491 fileconf.write ('e %s\n%s\n'%(var,val))
1492 fileconf.write('w\n')
1493 fileconf.write('R\n')
1494 fileconf.write('q\n')
1496 utils.system('cat %s'%tmpname)
1497 self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
1500 def aggregate_xml_line(self):
1501 port=self.plc_spec['sfa']['neighbours-port']
1502 return '<aggregate addr="%s" hrn="%s" port="%r"/>' % \
1503 (self.vserverip,self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH'],port)
1505 def registry_xml_line(self):
1506 return '<registry addr="%s" hrn="%s" port="12345"/>' % \
1507 (self.vserverip,self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH'])
1510 # a cross step that takes all other plcs in argument
1511 def cross_sfa_configure(self, other_plcs):
1512 "writes aggregates.xml and registries.xml that point to all other PLCs in the test"
1513 # of course with a single plc, other_plcs is an empty list
1516 agg_fname=self.conffile("agg.xml")
1517 file(agg_fname,"w").write("<aggregates>%s</aggregates>\n" % \
1518 " ".join([ plc.aggregate_xml_line() for plc in other_plcs ]))
1519 utils.header ("(Over)wrote %s"%agg_fname)
1520 reg_fname=self.conffile("reg.xml")
1521 file(reg_fname,"w").write("<registries>%s</registries>\n" % \
1522 " ".join([ plc.registry_xml_line() for plc in other_plcs ]))
1523 utils.header ("(Over)wrote %s"%reg_fname)
1524 return self.test_ssh.copy_abs(agg_fname,'/%s/etc/sfa/aggregates.xml'%self.vm_root_in_host())==0 \
1525 and self.test_ssh.copy_abs(reg_fname,'/%s/etc/sfa/registries.xml'%self.vm_root_in_host())==0
1527 def sfa_import(self):
1528 "use sfaadmin to import from plc"
1529 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1530 return self.run_in_guest('sfaadmin reg import_registry')==0
1532 def sfa_start(self):
1534 return self.start_service('sfa')
1537 def sfi_configure(self):
1538 "Create /root/sfi on the plc side for sfi client configuration"
1539 if self.options.dry_run:
1540 utils.header("DRY RUN - skipping step")
1542 sfa_spec=self.plc_spec['sfa']
1543 # cannot use auth_sfa_mapper to pass dir_name
1544 for slice_spec in self.plc_spec['sfa']['auth_sfa_specs']:
1545 test_slice=TestAuthSfa(self,slice_spec)
1546 dir_basename=os.path.basename(test_slice.sfi_path())
1547 dir_name=self.confsubdir("dot-sfi/%s"%dir_basename,clean=True,dry_run=self.options.dry_run)
1548 test_slice.sfi_configure(dir_name)
1549 # push into the remote /root/sfi area
1550 location = test_slice.sfi_path()
1551 remote="%s/%s"%(self.vm_root_in_host(),location)
1552 self.test_ssh.mkdir(remote,abs=True)
1553 # need to strip last level or remote otherwise we get an extra dir level
1554 self.test_ssh.copy_abs(dir_name, os.path.dirname(remote), recursive=True)
1558 def sfi_clean (self):
1559 "clean up /root/sfi on the plc side"
1560 self.run_in_guest("rm -rf /root/sfi")
1564 def sfa_add_site (self): pass
1566 def sfa_add_pi (self): pass
1568 def sfa_add_user(self): pass
1570 def sfa_update_user(self): pass
1572 def sfa_add_slice(self): pass
1574 def sfa_renew_slice(self): pass
1576 def sfa_discover(self): pass
1578 def sfa_create_slice(self): pass
1580 def sfa_check_slice_plc(self): pass
1582 def sfa_update_slice(self): pass
1584 def sfi_list(self): pass
1586 def sfi_show(self): pass
1588 def ssh_slice_sfa(self): pass
1590 def sfa_delete_user(self): pass
1592 def sfa_delete_slice(self): pass
1596 return self.stop_service ('sfa')
1598 def populate (self):
1599 "creates random entries in the PLCAPI"
1600 # install the stress-test in the plc image
1601 location = "/usr/share/plc_api/plcsh_stress_test.py"
1602 remote="%s/%s"%(self.vm_root_in_host(),location)
1603 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1605 command += " -- --preserve --short-names"
1606 local = (self.run_in_guest(command) == 0);
1607 # second run with --foreign
1608 command += ' --foreign'
1609 remote = (self.run_in_guest(command) == 0);
1610 return ( local and remote)
1612 def gather_logs (self):
1613 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1614 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1615 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1616 # (1.c) get the plc's /root/sfi -> logs/sfi.<plcname>/
1617 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1618 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1619 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1621 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1622 self.gather_var_logs ()
1624 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1625 self.gather_pgsql_logs ()
1627 print "-------------------- TestPlc.gather_logs : PLC's /root/sfi/"
1628 self.gather_root_sfi ()
1630 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1631 for site_spec in self.plc_spec['sites']:
1632 test_site = TestSite (self,site_spec)
1633 for node_spec in site_spec['nodes']:
1634 test_node=TestNode(self,test_site,node_spec)
1635 test_node.gather_qemu_logs()
1637 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1638 self.gather_nodes_var_logs()
1640 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1641 self.gather_slivers_var_logs()
1644 def gather_slivers_var_logs(self):
1645 for test_sliver in self.all_sliver_objs():
1646 remote = test_sliver.tar_var_logs()
1647 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
1648 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
1649 utils.system(command)
1652 def gather_var_logs (self):
1653 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
1654 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1655 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
1656 utils.system(command)
1657 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd"%self.name()
1658 utils.system(command)
1660 def gather_pgsql_logs (self):
1661 utils.system("mkdir -p logs/myplc.pgsql-log.%s"%self.name())
1662 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1663 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -"%self.name()
1664 utils.system(command)
1666 def gather_root_sfi (self):
1667 utils.system("mkdir -p logs/sfi.%s"%self.name())
1668 to_plc = self.actual_command_in_guest("tar -C /root/sfi/ -cf - .")
1669 command = to_plc + "| tar -C logs/sfi.%s -xf -"%self.name()
1670 utils.system(command)
1672 def gather_nodes_var_logs (self):
1673 for site_spec in self.plc_spec['sites']:
1674 test_site = TestSite (self,site_spec)
1675 for node_spec in site_spec['nodes']:
1676 test_node=TestNode(self,test_site,node_spec)
1677 test_ssh = TestSsh (test_node.name(),key="keys/key_admin.rsa")
1678 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1679 command = command + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
1680 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
1681 utils.system(command)
1684 # returns the filename to use for sql dump/restore, using options.dbname if set
1685 def dbfile (self, database):
1686 # uses options.dbname if it is found
1688 name=self.options.dbname
1689 if not isinstance(name,StringTypes):
1695 return "/root/%s-%s.sql"%(database,name)
1697 def plc_db_dump(self):
1698 'dump the planetlab5 DB in /root in the PLC - filename has time'
1699 dump=self.dbfile("planetab5")
1700 self.run_in_guest('pg_dump -U pgsqluser planetlab5 -f '+ dump)
1701 utils.header('Dumped planetlab5 database in %s'%dump)
1704 def plc_db_restore(self):
1705 'restore the planetlab5 DB - looks broken, but run -n might help'
1706 dump=self.dbfile("planetab5")
1707 ##stop httpd service
1708 self.run_in_guest('service httpd stop')
1709 # xxx - need another wrapper
1710 self.run_in_guest_piped('echo drop database planetlab5','psql --user=pgsqluser template1')
1711 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab5')
1712 self.run_in_guest('psql -U pgsqluser planetlab5 -f '+dump)
1713 ##starting httpd service
1714 self.run_in_guest('service httpd start')
1716 utils.header('Database restored from ' + dump)
1719 def create_ignore_steps ():
1720 for step in TestPlc.default_steps + TestPlc.other_steps:
1721 # default step can have a plc qualifier
1722 if '@' in step: (step,qualifier)=step.split('@')
1723 # or be defined as forced or ignored by default
1724 for keyword in ['_ignore','_force']:
1725 if step.endswith (keyword): step=step.replace(keyword,'')
1726 if step == SEP or step == SEPSFA : continue
1727 method=getattr(TestPlc,step)
1729 wrapped=ignore_result(method)
1730 # wrapped.__doc__ = method.__doc__ + " (run in ignore-result mode)"
1731 setattr(TestPlc, name, wrapped)
1734 # def ssh_slice_again_ignore (self): pass
1736 # def check_initscripts_ignore (self): pass
1738 def standby_1_through_20(self):
1739 """convenience function to wait for a specified number of minutes"""
1742 def standby_1(): pass
1744 def standby_2(): pass
1746 def standby_3(): pass
1748 def standby_4(): pass
1750 def standby_5(): pass
1752 def standby_6(): pass
1754 def standby_7(): pass
1756 def standby_8(): pass
1758 def standby_9(): pass
1760 def standby_10(): pass
1762 def standby_11(): pass
1764 def standby_12(): pass
1766 def standby_13(): pass
1768 def standby_14(): pass
1770 def standby_15(): pass
1772 def standby_16(): pass
1774 def standby_17(): pass
1776 def standby_18(): pass
1778 def standby_19(): pass
1780 def standby_20(): pass
1782 # convenience for debugging the test logic
1783 def yes (self): return True
1784 def no (self): return False