1 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
2 # Copyright (C) 2010 INRIA
9 from datetime import datetime, timedelta
10 from types import StringTypes
13 from Completer import Completer, CompleterTask
14 from TestSite import TestSite
15 from TestNode import TestNode, CompleterTaskNodeSsh
16 from TestUser import TestUser
17 from TestKey import TestKey
18 from TestSlice import TestSlice
19 from TestSliver import TestSliver
20 from TestBoxQemu import TestBoxQemu
21 from TestSsh import TestSsh
22 from TestApiserver import TestApiserver
23 from TestAuthSfa import TestAuthSfa
24 from PlcapiUrlScanner import PlcapiUrlScanner
26 has_sfa_cache_filename="sfa-cache"
28 # step methods must take (self) and return a boolean (options is a member of the class)
30 def standby(minutes,dry_run):
31 utils.header('Entering StandBy for %d mn'%minutes)
35 time.sleep(60*minutes)
38 def standby_generic (func):
40 minutes=int(func.__name__.split("_")[1])
41 return standby(minutes,self.options.dry_run)
44 def node_mapper (method):
45 def map_on_nodes(self,*args, **kwds):
47 node_method = TestNode.__dict__[method.__name__]
48 for test_node in self.all_nodes():
49 if not node_method(test_node, *args, **kwds): overall=False
51 # maintain __name__ for ignore_result
52 map_on_nodes.__name__=method.__name__
53 # restore the doc text
54 map_on_nodes.__doc__=TestNode.__dict__[method.__name__].__doc__
57 def slice_mapper (method):
58 def map_on_slices(self):
60 slice_method = TestSlice.__dict__[method.__name__]
61 for slice_spec in self.plc_spec['slices']:
62 site_spec = self.locate_site (slice_spec['sitename'])
63 test_site = TestSite(self,site_spec)
64 test_slice=TestSlice(self,test_site,slice_spec)
65 if not slice_method(test_slice,self.options): overall=False
67 # maintain __name__ for ignore_result
68 map_on_slices.__name__=method.__name__
69 # restore the doc text
70 map_on_slices.__doc__=TestSlice.__dict__[method.__name__].__doc__
73 # run a step but return True so that we can go on
74 def ignore_result (method):
76 # ssh_slice_ignore->ssh_slice
77 ref_name=method.__name__.replace('_ignore','').replace('force_','')
78 ref_method=TestPlc.__dict__[ref_name]
79 result=ref_method(self)
80 print "Actual (but ignored) result for %(ref_name)s is %(result)s"%locals()
81 return Ignored (result)
82 name=method.__name__.replace('_ignore','').replace('force_','')
83 ignoring.__name__=name
84 ignoring.__doc__="ignored version of " + name
87 # a variant that expects the TestSlice method to return a list of CompleterTasks that
88 # are then merged into a single Completer run to avoid wating for all the slices
89 # esp. useful when a test fails of course
90 # because we need to pass arguments we use a class instead..
91 class slice_mapper__tasks (object):
92 # could not get this to work with named arguments
93 def __init__ (self,timeout_minutes,silent_minutes,period_seconds):
94 self.timeout=timedelta(minutes=timeout_minutes)
95 self.silent=timedelta(minutes=silent_minutes)
96 self.period=timedelta(seconds=period_seconds)
97 def __call__ (self, method):
99 # compute augmented method name
100 method_name = method.__name__ + "__tasks"
101 # locate in TestSlice
102 slice_method = TestSlice.__dict__[ method_name ]
105 for slice_spec in self.plc_spec['slices']:
106 site_spec = self.locate_site (slice_spec['sitename'])
107 test_site = TestSite(self,site_spec)
108 test_slice=TestSlice(self,test_site,slice_spec)
109 tasks += slice_method (test_slice, self.options)
110 return Completer (tasks).run (decorator_self.timeout, decorator_self.silent, decorator_self.period)
111 # restore the doc text from the TestSlice method even if a bit odd
112 wrappee.__name__ = method.__name__
113 wrappee.__doc__ = slice_method.__doc__
116 def auth_sfa_mapper (method):
119 auth_method = TestAuthSfa.__dict__[method.__name__]
120 for auth_spec in self.plc_spec['sfa']['auth_sfa_specs']:
121 test_auth=TestAuthSfa(self,auth_spec)
122 if not auth_method(test_auth,self.options): overall=False
124 # restore the doc text
125 actual.__doc__=TestAuthSfa.__dict__[method.__name__].__doc__
129 def __init__ (self,result):
139 'plcvm_delete','plcvm_timestamp','plcvm_create', SEP,
140 'plc_install', 'plc_configure', 'plc_start', SEP,
141 'keys_fetch', 'keys_store', 'keys_clear_known_hosts', SEP,
142 'plcapi_urls','speed_up_slices', SEP,
143 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', 'leases', SEP,
144 # slices created under plcsh interactively seem to be fine but these ones don't have the tags
145 # keep this our of the way for now
146 'check_vsys_defaults_ignore', SEP,
147 # run this first off so it's easier to re-run on another qemu box
148 'qemu_kill_mine', SEP,
149 'nodestate_reinstall', 'qemu_local_init','bootcd', 'qemu_local_config', SEP,
150 'qemu_clean_mine', 'qemu_export', 'qemu_start', 'qemu_timestamp', SEP,
151 'sfa_install_all', 'sfa_configure', 'cross_sfa_configure', 'sfa_start', 'sfa_import', SEPSFA,
152 'sfi_configure@1', 'sfa_register_site@1','sfa_register_pi@1', SEPSFA,
153 'sfa_register_user@1', 'sfa_update_user@1', 'sfa_register_slice@1', 'sfa_renew_slice@1', SEPSFA,
154 'sfa_remove_user_from_slice@1','sfi_show_slice_researchers@1',
155 'sfa_insert_user_in_slice@1','sfi_show_slice_researchers@1', SEPSFA,
156 'sfa_discover@1', 'sfa_rspec@1', 'sfa_allocate@1', 'sfa_provision@1', SEPSFA,
157 'sfa_check_slice_plc@1', 'sfa_update_slice@1', SEPSFA,
158 'sfa_rspec_empty@1','sfa_allocate_empty@1', 'sfa_provision_empty@1','sfa_check_slice_plc_empty@1', SEPSFA,
159 'sfi_list@1', 'sfi_show_site@1', 'sfa_utest@1', SEPSFA,
160 # we used to run plcsh_stress_test, and then ssh_node_debug and ssh_node_boot
161 # but as the stress test might take a while, we sometimes missed the debug mode..
162 'probe_kvm_iptables',
163 'ping_node', 'ssh_node_debug', 'plcsh_stress_test@1', SEP,
164 'ssh_node_boot', 'node_bmlogs', 'ssh_slice', 'ssh_slice_basics', 'check_initscripts_ignore', SEP,
165 'ssh_slice_sfa@1', 'sfa_delete_slice@1', 'sfa_delete_user@1', SEPSFA,
166 'cross_check_tcp@1', 'check_system_slice', SEP,
167 # check slices are turned off properly
168 'empty_slices', 'ssh_slice_off', 'slice_fs_deleted_ignore', SEP,
169 # check they are properly re-created with the same name
170 'fill_slices', 'ssh_slice_again', SEP,
171 'gather_logs_force', SEP,
174 'export', 'show_boxes', 'super_speed_up_slices', SEP,
175 'check_hooks', 'plc_stop', 'plcvm_start', 'plcvm_stop', SEP,
176 'delete_initscripts', 'delete_nodegroups','delete_all_sites', SEP,
177 'delete_sites', 'delete_nodes', 'delete_slices', 'keys_clean', SEP,
178 'delete_leases', 'list_leases', SEP,
180 'nodestate_show','nodestate_safeboot','nodestate_boot', SEP,
181 'qemu_list_all', 'qemu_list_mine', 'qemu_kill_all', SEP,
182 'sfa_install_core', 'sfa_install_sfatables', 'sfa_install_plc', 'sfa_install_client', SEPSFA,
183 'sfa_plcclean', 'sfa_dbclean', 'sfa_stop','sfa_uninstall', 'sfi_clean', SEPSFA,
184 'sfa_get_expires', SEPSFA,
185 'plc_db_dump' , 'plc_db_restore', SEP,
186 'check_netflow','check_drl', SEP,
187 'debug_nodemanager', 'slice_fs_present', SEP,
188 'standby_1_through_20','yes','no',SEP,
192 def printable_steps (list):
193 single_line=" ".join(list)+" "
194 return single_line.replace(" "+SEP+" "," \\\n").replace(" "+SEPSFA+" "," \\\n")
196 def valid_step (step):
197 return step != SEP and step != SEPSFA
199 # turn off the sfa-related steps when build has skipped SFA
200 # this was originally for centos5 but is still valid
201 # for up to f12 as recent SFAs with sqlalchemy won't build before f14
203 def _has_sfa_cached (rpms_url):
204 if os.path.isfile(has_sfa_cache_filename):
205 cached=file(has_sfa_cache_filename).read()=="yes"
206 utils.header("build provides SFA (cached):%s"%cached)
208 # warning, we're now building 'sface' so let's be a bit more picky
209 # full builds are expected to return with 0 here
210 utils.header ("Checking if build provides SFA package...")
211 retcod=os.system ("curl --silent %s/ | grep -q sfa-"%rpms_url)==0
212 encoded='yes' if retcod else 'no'
213 file(has_sfa_cache_filename,'w').write(encoded)
217 def check_whether_build_has_sfa (rpms_url):
218 has_sfa=TestPlc._has_sfa_cached(rpms_url)
220 utils.header("build does provide SFA")
222 # move all steps containing 'sfa' from default_steps to other_steps
223 utils.header("SFA package not found - removing steps with sfa or sfi")
224 sfa_steps= [ step for step in TestPlc.default_steps if step.find('sfa')>=0 or step.find("sfi")>=0 ]
225 TestPlc.other_steps += sfa_steps
226 for step in sfa_steps: TestPlc.default_steps.remove(step)
228 def __init__ (self,plc_spec,options):
229 self.plc_spec=plc_spec
231 self.test_ssh=TestSsh(self.plc_spec['host_box'],self.options.buildname)
232 self.vserverip=plc_spec['vserverip']
233 self.vservername=plc_spec['vservername']
234 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
235 self.apiserver=TestApiserver(self.url,options.dry_run)
236 (self.ssh_node_boot_timeout,self.ssh_node_boot_silent)=plc_spec['ssh_node_boot_timers']
237 (self.ssh_node_debug_timeout,self.ssh_node_debug_silent)=plc_spec['ssh_node_debug_timers']
239 def has_addresses_api (self):
240 return self.apiserver.has_method('AddIpAddress')
243 name=self.plc_spec['name']
244 return "%s.%s"%(name,self.vservername)
247 return self.plc_spec['host_box']
250 return self.test_ssh.is_local()
252 # define the API methods on this object through xmlrpc
253 # would help, but not strictly necessary
257 def actual_command_in_guest (self,command, backslash=False):
258 raw1=self.host_to_guest(command)
259 raw2=self.test_ssh.actual_command(raw1,dry_run=self.options.dry_run, backslash=backslash)
262 def start_guest (self):
263 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host(),dry_run=self.options.dry_run))
265 def stop_guest (self):
266 return utils.system(self.test_ssh.actual_command(self.stop_guest_in_host(),dry_run=self.options.dry_run))
268 def run_in_guest (self,command,backslash=False):
269 raw=self.actual_command_in_guest(command,backslash)
270 return utils.system(raw)
272 def run_in_host (self,command):
273 return self.test_ssh.run_in_buildname(command, dry_run=self.options.dry_run)
275 # backslashing turned out so awful at some point that I've turned off auto-backslashing
276 # see e.g. plc_start esp. the version for f14
277 #command gets run in the plc's vm
278 def host_to_guest(self,command):
279 vservername=self.vservername
280 personality=self.options.personality
281 raw="%(personality)s virsh -c lxc:/// lxc-enter-namespace %(vservername)s"%locals()
282 # f14 still needs some extra help
283 if self.options.fcdistro == 'f14':
284 raw +=" -- /usr/bin/env PATH=/bin:/sbin:/usr/bin:/usr/sbin %(command)s" %locals()
286 raw +=" -- /usr/bin/env %(command)s"%locals()
289 # this /vservers thing is legacy...
290 def vm_root_in_host(self):
291 return "/vservers/%s/"%(self.vservername)
293 def vm_timestamp_path (self):
294 return "/vservers/%s/%s.timestamp"%(self.vservername,self.vservername)
296 #start/stop the vserver
297 def start_guest_in_host(self):
298 return "virsh -c lxc:/// start %s"%(self.vservername)
300 def stop_guest_in_host(self):
301 return "virsh -c lxc:/// destroy %s"%(self.vservername)
304 def run_in_guest_piped (self,local,remote):
305 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
307 def yum_check_installed (self, rpms):
308 if isinstance (rpms, list):
310 return self.run_in_guest("rpm -q %s"%rpms)==0
312 # does a yum install in the vs, ignore yum retcod, check with rpm
313 def yum_install (self, rpms):
314 if isinstance (rpms, list):
316 self.run_in_guest("yum -y install %s"%rpms)
317 # yum-complete-transaction comes with yum-utils, that is in vtest.pkgs
318 self.run_in_guest("yum-complete-transaction -y")
319 return self.yum_check_installed (rpms)
321 def auth_root (self):
322 return {'Username':self.plc_spec['settings']['PLC_ROOT_USER'],
323 'AuthMethod':'password',
324 'AuthString':self.plc_spec['settings']['PLC_ROOT_PASSWORD'],
325 'Role' : self.plc_spec['role']
327 def locate_site (self,sitename):
328 for site in self.plc_spec['sites']:
329 if site['site_fields']['name'] == sitename:
331 if site['site_fields']['login_base'] == sitename:
333 raise Exception,"Cannot locate site %s"%sitename
335 def locate_node (self,nodename):
336 for site in self.plc_spec['sites']:
337 for node in site['nodes']:
338 if node['name'] == nodename:
340 raise Exception,"Cannot locate node %s"%nodename
342 def locate_hostname (self,hostname):
343 for site in self.plc_spec['sites']:
344 for node in site['nodes']:
345 if node['node_fields']['hostname'] == hostname:
347 raise Exception,"Cannot locate hostname %s"%hostname
349 def locate_key (self,key_name):
350 for key in self.plc_spec['keys']:
351 if key['key_name'] == key_name:
353 raise Exception,"Cannot locate key %s"%key_name
355 def locate_private_key_from_key_names (self, key_names):
356 # locate the first avail. key
358 for key_name in key_names:
359 key_spec=self.locate_key(key_name)
360 test_key=TestKey(self,key_spec)
361 publickey=test_key.publicpath()
362 privatekey=test_key.privatepath()
363 if os.path.isfile(publickey) and os.path.isfile(privatekey):
365 if found: return privatekey
368 def locate_slice (self, slicename):
369 for slice in self.plc_spec['slices']:
370 if slice['slice_fields']['name'] == slicename:
372 raise Exception,"Cannot locate slice %s"%slicename
374 def all_sliver_objs (self):
376 for slice_spec in self.plc_spec['slices']:
377 slicename = slice_spec['slice_fields']['name']
378 for nodename in slice_spec['nodenames']:
379 result.append(self.locate_sliver_obj (nodename,slicename))
382 def locate_sliver_obj (self,nodename,slicename):
383 (site,node) = self.locate_node(nodename)
384 slice = self.locate_slice (slicename)
386 test_site = TestSite (self, site)
387 test_node = TestNode (self, test_site,node)
388 # xxx the slice site is assumed to be the node site - mhh - probably harmless
389 test_slice = TestSlice (self, test_site, slice)
390 return TestSliver (self, test_node, test_slice)
392 def locate_first_node(self):
393 nodename=self.plc_spec['slices'][0]['nodenames'][0]
394 (site,node) = self.locate_node(nodename)
395 test_site = TestSite (self, site)
396 test_node = TestNode (self, test_site,node)
399 def locate_first_sliver (self):
400 slice_spec=self.plc_spec['slices'][0]
401 slicename=slice_spec['slice_fields']['name']
402 nodename=slice_spec['nodenames'][0]
403 return self.locate_sliver_obj(nodename,slicename)
405 # all different hostboxes used in this plc
406 def get_BoxNodes(self):
407 # maps on sites and nodes, return [ (host_box,test_node) ]
409 for site_spec in self.plc_spec['sites']:
410 test_site = TestSite (self,site_spec)
411 for node_spec in site_spec['nodes']:
412 test_node = TestNode (self, test_site, node_spec)
413 if not test_node.is_real():
414 tuples.append( (test_node.host_box(),test_node) )
415 # transform into a dict { 'host_box' -> [ test_node .. ] }
417 for (box,node) in tuples:
418 if not result.has_key(box):
421 result[box].append(node)
424 # a step for checking this stuff
425 def show_boxes (self):
426 'print summary of nodes location'
427 for (box,nodes) in self.get_BoxNodes().iteritems():
428 print box,":"," + ".join( [ node.name() for node in nodes ] )
431 # make this a valid step
432 def qemu_kill_all(self):
433 'kill all qemu instances on the qemu boxes involved by this setup'
434 # this is the brute force version, kill all qemus on that host box
435 for (box,nodes) in self.get_BoxNodes().iteritems():
436 # pass the first nodename, as we don't push template-qemu on testboxes
437 nodedir=nodes[0].nodedir()
438 TestBoxQemu(box,self.options.buildname).qemu_kill_all(nodedir)
441 # make this a valid step
442 def qemu_list_all(self):
443 'list all qemu instances on the qemu boxes involved by this setup'
444 for (box,nodes) in self.get_BoxNodes().iteritems():
445 # this is the brute force version, kill all qemus on that host box
446 TestBoxQemu(box,self.options.buildname).qemu_list_all()
449 # kill only the qemus related to this test
450 def qemu_list_mine(self):
451 'list qemu instances for our nodes'
452 for (box,nodes) in self.get_BoxNodes().iteritems():
453 # the fine-grain version
458 # kill only the qemus related to this test
459 def qemu_clean_mine(self):
460 'cleanup (rm -rf) qemu instances for our nodes'
461 for (box,nodes) in self.get_BoxNodes().iteritems():
462 # the fine-grain version
467 # kill only the right qemus
468 def qemu_kill_mine(self):
469 'kill the qemu instances for our nodes'
470 for (box,nodes) in self.get_BoxNodes().iteritems():
471 # the fine-grain version
476 #################### display config
478 "show test configuration after localization"
483 # uggly hack to make sure 'run export' only reports about the 1st plc
484 # to avoid confusion - also we use 'inri_slice1' in various aliases..
487 "print cut'n paste-able stuff to export env variables to your shell"
488 # guess local domain from hostname
489 if TestPlc.exported_id>1:
490 print "export GUESTHOSTNAME%d=%s"%(TestPlc.exported_id,self.plc_spec['vservername'])
492 TestPlc.exported_id+=1
493 domain=socket.gethostname().split('.',1)[1]
494 fqdn="%s.%s"%(self.plc_spec['host_box'],domain)
495 print "export BUILD=%s"%self.options.buildname
496 print "export PLCHOSTLXC=%s"%fqdn
497 print "export GUESTNAME=%s"%self.plc_spec['vservername']
498 vplcname=self.plc_spec['vservername'].split('-')[-1]
499 print "export GUESTHOSTNAME=%s.%s"%(vplcname,domain)
500 # find hostname of first node
501 (hostname,qemubox) = self.all_node_infos()[0]
502 print "export KVMHOST=%s.%s"%(qemubox,domain)
503 print "export NODE=%s"%(hostname)
507 always_display_keys=['PLC_WWW_HOST','nodes','sites',]
508 def show_pass (self,passno):
509 for (key,val) in self.plc_spec.iteritems():
510 if not self.options.verbose and key not in TestPlc.always_display_keys: continue
514 self.display_site_spec(site)
515 for node in site['nodes']:
516 self.display_node_spec(node)
517 elif key=='initscripts':
518 for initscript in val:
519 self.display_initscript_spec (initscript)
522 self.display_slice_spec (slice)
525 self.display_key_spec (key)
527 if key not in ['sites','initscripts','slices','keys']:
528 print '+ ',key,':',val
530 def display_site_spec (self,site):
531 print '+ ======== site',site['site_fields']['name']
532 for (k,v) in site.iteritems():
533 if not self.options.verbose and k not in TestPlc.always_display_keys: continue
536 print '+ ','nodes : ',
538 print node['node_fields']['hostname'],'',
544 print user['name'],'',
546 elif k == 'site_fields':
547 print '+ login_base',':',v['login_base']
548 elif k == 'address_fields':
554 def display_initscript_spec (self,initscript):
555 print '+ ======== initscript',initscript['initscript_fields']['name']
557 def display_key_spec (self,key):
558 print '+ ======== key',key['key_name']
560 def display_slice_spec (self,slice):
561 print '+ ======== slice',slice['slice_fields']['name']
562 for (k,v) in slice.iteritems():
575 elif k=='slice_fields':
576 print '+ fields',':',
577 print 'max_nodes=',v['max_nodes'],
582 def display_node_spec (self,node):
583 print "+ node=%s host_box=%s"%(node['name'],node['host_box']),
584 print "hostname=",node['node_fields']['hostname'],
585 print "ip=",node['interface_fields']['ip']
586 if self.options.verbose:
587 utils.pprint("node details",node,depth=3)
589 # another entry point for just showing the boxes involved
590 def display_mapping (self):
591 TestPlc.display_mapping_plc(self.plc_spec)
595 def display_mapping_plc (plc_spec):
596 print '+ MyPLC',plc_spec['name']
597 # WARNING this would not be right for lxc-based PLC's - should be harmless though
598 print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['host_box'],plc_spec['vservername'])
599 print '+\tIP = %s/%s'%(plc_spec['settings']['PLC_API_HOST'],plc_spec['vserverip'])
600 for site_spec in plc_spec['sites']:
601 for node_spec in site_spec['nodes']:
602 TestPlc.display_mapping_node(node_spec)
605 def display_mapping_node (node_spec):
606 print '+ NODE %s'%(node_spec['name'])
607 print '+\tqemu box %s'%node_spec['host_box']
608 print '+\thostname=%s'%node_spec['node_fields']['hostname']
610 # write a timestamp in /vservers/<>.timestamp
611 # cannot be inside the vserver, that causes vserver .. build to cough
612 def plcvm_timestamp (self):
613 "Create a timestamp to remember creation date for this plc"
615 # TODO-lxc check this one
616 # a first approx. is to store the timestamp close to the VM root like vs does
617 stamp_path=self.vm_timestamp_path ()
618 stamp_dir = os.path.dirname (stamp_path)
619 utils.system(self.test_ssh.actual_command("mkdir -p %s"%stamp_dir))
620 return utils.system(self.test_ssh.actual_command("echo %d > %s"%(now,stamp_path)))==0
622 # this is called inconditionnally at the beginning of the test sequence
623 # just in case this is a rerun, so if the vm is not running it's fine
624 def plcvm_delete(self):
625 "vserver delete the test myplc"
626 stamp_path=self.vm_timestamp_path()
627 self.run_in_host("rm -f %s"%stamp_path)
628 self.run_in_host("virsh -c lxc:// destroy %s"%self.vservername)
629 self.run_in_host("virsh -c lxc:// undefine %s"%self.vservername)
630 self.run_in_host("rm -fr /vservers/%s"%self.vservername)
634 # historically the build was being fetched by the tests
635 # now the build pushes itself as a subdir of the tests workdir
636 # so that the tests do not have to worry about extracting the build (svn, git, or whatever)
637 def plcvm_create (self):
638 "vserver creation (no install done)"
639 # push the local build/ dir to the testplc box
641 # a full path for the local calls
642 build_dir=os.path.dirname(sys.argv[0])
643 # sometimes this is empty - set to "." in such a case
644 if not build_dir: build_dir="."
645 build_dir += "/build"
647 # use a standard name - will be relative to remote buildname
649 # remove for safety; do *not* mkdir first, otherwise we end up with build/build/
650 self.test_ssh.rmdir(build_dir)
651 self.test_ssh.copy(build_dir,recursive=True)
652 # the repo url is taken from arch-rpms-url
653 # with the last step (i386) removed
654 repo_url = self.options.arch_rpms_url
655 for level in [ 'arch' ]:
656 repo_url = os.path.dirname(repo_url)
658 # invoke initvm (drop support for vs)
659 script="lbuild-initvm.sh"
661 # pass the vbuild-nightly options to [lv]test-initvm
662 script_options += " -p %s"%self.options.personality
663 script_options += " -d %s"%self.options.pldistro
664 script_options += " -f %s"%self.options.fcdistro
665 script_options += " -r %s"%repo_url
666 vserver_name = self.vservername
668 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
669 script_options += " -n %s"%vserver_hostname
671 print "Cannot reverse lookup %s"%self.vserverip
672 print "This is considered fatal, as this might pollute the test results"
674 create_vserver="%(build_dir)s/%(script)s %(script_options)s %(vserver_name)s"%locals()
675 return self.run_in_host(create_vserver) == 0
678 def plc_install(self):
679 "yum install myplc, noderepo, and the plain bootstrapfs"
681 # workaround for getting pgsql8.2 on centos5
682 if self.options.fcdistro == "centos5":
683 self.run_in_guest("rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-3.noarch.rpm")
686 if self.options.personality == "linux32":
688 elif self.options.personality == "linux64":
691 raise Exception, "Unsupported personality %r"%self.options.personality
692 nodefamily="%s-%s-%s"%(self.options.pldistro,self.options.fcdistro,arch)
695 pkgs_list.append ("slicerepo-%s"%nodefamily)
696 pkgs_list.append ("myplc")
697 pkgs_list.append ("noderepo-%s"%nodefamily)
698 pkgs_list.append ("nodeimage-%s-plain"%nodefamily)
699 pkgs_string=" ".join(pkgs_list)
700 return self.yum_install (pkgs_list)
703 def mod_python(self):
704 """yum install mod_python, useful on f18 and above so as to avoid broken wsgi"""
705 return self.yum_install ( [ 'mod_python' ] )
708 def plc_configure(self):
710 tmpname='%s.plc-config-tty'%(self.name())
711 fileconf=open(tmpname,'w')
712 for (var,value) in self.plc_spec['settings'].iteritems():
713 fileconf.write ('e %s\n%s\n'%(var,value))
714 fileconf.write('w\n')
715 fileconf.write('q\n')
717 utils.system('cat %s'%tmpname)
718 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
719 utils.system('rm %s'%tmpname)
722 # f14 is a bit odd in this respect, although this worked fine in guests up to f18
723 # however using a vplc guest under f20 requires this trick
724 # the symptom is this: service plc start
725 # Starting plc (via systemctl): Failed to get D-Bus connection: \
726 # Failed to connect to socket /org/freedesktop/systemd1/private: Connection refused
727 # weird thing is the doc says f14 uses upstart by default and not systemd
728 # so this sounds kind of harmless
729 def start_service (self,service): return self.start_stop_service (service,'start')
730 def stop_service (self,service): return self.start_stop_service (service,'stop')
732 def start_stop_service (self, service,start_or_stop):
733 "utility to start/stop a service with the special trick for f14"
734 if self.options.fcdistro != 'f14':
735 return self.run_in_guest ("service %s %s"%(service,start_or_stop))==0
737 # patch /sbin/service so it does not reset environment
738 self.run_in_guest ('sed -i -e \\"s,env -i,env,\\" /sbin/service')
739 # this is because our own scripts in turn call service
740 return self.run_in_guest("SYSTEMCTL_SKIP_REDIRECT=true service %s %s"%(service,start_or_stop))==0
744 return self.start_service ('plc')
748 return self.stop_service ('plc')
750 def plcvm_start (self):
751 "start the PLC vserver"
755 def plcvm_stop (self):
756 "stop the PLC vserver"
760 # stores the keys from the config for further use
761 def keys_store(self):
762 "stores test users ssh keys in keys/"
763 for key_spec in self.plc_spec['keys']:
764 TestKey(self,key_spec).store_key()
767 def keys_clean(self):
768 "removes keys cached in keys/"
769 utils.system("rm -rf ./keys")
772 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
773 # for later direct access to the nodes
774 def keys_fetch(self):
775 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
777 if not os.path.isdir(dir):
779 vservername=self.vservername
780 vm_root=self.vm_root_in_host()
782 prefix = 'debug_ssh_key'
783 for ext in [ 'pub', 'rsa' ] :
784 src="%(vm_root)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
785 dst="keys/%(vservername)s-debug.%(ext)s"%locals()
786 if self.test_ssh.fetch(src,dst) != 0: overall=False
790 "create sites with PLCAPI"
791 return self.do_sites()
793 def delete_sites (self):
794 "delete sites with PLCAPI"
795 return self.do_sites(action="delete")
797 def do_sites (self,action="add"):
798 for site_spec in self.plc_spec['sites']:
799 test_site = TestSite (self,site_spec)
800 if (action != "add"):
801 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
802 test_site.delete_site()
803 # deleted with the site
804 #test_site.delete_users()
807 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
808 test_site.create_site()
809 test_site.create_users()
812 def delete_all_sites (self):
813 "Delete all sites in PLC, and related objects"
814 print 'auth_root',self.auth_root()
815 sites = self.apiserver.GetSites(self.auth_root(), {}, ['site_id','login_base'])
817 # keep automatic site - otherwise we shoot in our own foot, root_auth is not valid anymore
818 if site['login_base']==self.plc_spec['settings']['PLC_SLICE_PREFIX']: continue
819 site_id=site['site_id']
820 print 'Deleting site_id',site_id
821 self.apiserver.DeleteSite(self.auth_root(),site_id)
825 "create nodes with PLCAPI"
826 return self.do_nodes()
827 def delete_nodes (self):
828 "delete nodes with PLCAPI"
829 return self.do_nodes(action="delete")
831 def do_nodes (self,action="add"):
832 for site_spec in self.plc_spec['sites']:
833 test_site = TestSite (self,site_spec)
835 utils.header("Deleting nodes in site %s"%test_site.name())
836 for node_spec in site_spec['nodes']:
837 test_node=TestNode(self,test_site,node_spec)
838 utils.header("Deleting %s"%test_node.name())
839 test_node.delete_node()
841 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
842 for node_spec in site_spec['nodes']:
843 utils.pprint('Creating node %s'%node_spec,node_spec)
844 test_node = TestNode (self,test_site,node_spec)
845 test_node.create_node ()
848 def nodegroups (self):
849 "create nodegroups with PLCAPI"
850 return self.do_nodegroups("add")
851 def delete_nodegroups (self):
852 "delete nodegroups with PLCAPI"
853 return self.do_nodegroups("delete")
857 def translate_timestamp (start,grain,timestamp):
858 if timestamp < TestPlc.YEAR: return start+timestamp*grain
859 else: return timestamp
862 def timestamp_printable (timestamp):
863 return time.strftime('%m-%d %H:%M:%S UTC',time.gmtime(timestamp))
866 "create leases (on reservable nodes only, use e.g. run -c default -c resa)"
868 grain=self.apiserver.GetLeaseGranularity(self.auth_root())
869 print 'API answered grain=',grain
870 start=(now/grain)*grain
872 # find out all nodes that are reservable
873 nodes=self.all_reservable_nodenames()
875 utils.header ("No reservable node found - proceeding without leases")
878 # attach them to the leases as specified in plc_specs
879 # this is where the 'leases' field gets interpreted as relative of absolute
880 for lease_spec in self.plc_spec['leases']:
881 # skip the ones that come with a null slice id
882 if not lease_spec['slice']: continue
883 lease_spec['t_from']=TestPlc.translate_timestamp(start,grain,lease_spec['t_from'])
884 lease_spec['t_until']=TestPlc.translate_timestamp(start,grain,lease_spec['t_until'])
885 lease_addition=self.apiserver.AddLeases(self.auth_root(),nodes,
886 lease_spec['slice'],lease_spec['t_from'],lease_spec['t_until'])
887 if lease_addition['errors']:
888 utils.header("Cannot create leases, %s"%lease_addition['errors'])
891 utils.header('Leases on nodes %r for %s from %d (%s) until %d (%s)'%\
892 (nodes,lease_spec['slice'],
893 lease_spec['t_from'],TestPlc.timestamp_printable(lease_spec['t_from']),
894 lease_spec['t_until'],TestPlc.timestamp_printable(lease_spec['t_until'])))
898 def delete_leases (self):
899 "remove all leases in the myplc side"
900 lease_ids= [ l['lease_id'] for l in self.apiserver.GetLeases(self.auth_root())]
901 utils.header("Cleaning leases %r"%lease_ids)
902 self.apiserver.DeleteLeases(self.auth_root(),lease_ids)
905 def list_leases (self):
906 "list all leases known to the myplc"
907 leases = self.apiserver.GetLeases(self.auth_root())
910 current=l['t_until']>=now
911 if self.options.verbose or current:
912 utils.header("%s %s from %s until %s"%(l['hostname'],l['name'],
913 TestPlc.timestamp_printable(l['t_from']),
914 TestPlc.timestamp_printable(l['t_until'])))
917 # create nodegroups if needed, and populate
918 def do_nodegroups (self, action="add"):
919 # 1st pass to scan contents
921 for site_spec in self.plc_spec['sites']:
922 test_site = TestSite (self,site_spec)
923 for node_spec in site_spec['nodes']:
924 test_node=TestNode (self,test_site,node_spec)
925 if node_spec.has_key('nodegroups'):
926 nodegroupnames=node_spec['nodegroups']
927 if isinstance(nodegroupnames,StringTypes):
928 nodegroupnames = [ nodegroupnames ]
929 for nodegroupname in nodegroupnames:
930 if not groups_dict.has_key(nodegroupname):
931 groups_dict[nodegroupname]=[]
932 groups_dict[nodegroupname].append(test_node.name())
933 auth=self.auth_root()
935 for (nodegroupname,group_nodes) in groups_dict.iteritems():
937 print 'nodegroups:','dealing with nodegroup',nodegroupname,'on nodes',group_nodes
938 # first, check if the nodetagtype is here
939 tag_types = self.apiserver.GetTagTypes(auth,{'tagname':nodegroupname})
941 tag_type_id = tag_types[0]['tag_type_id']
943 tag_type_id = self.apiserver.AddTagType(auth,
944 {'tagname':nodegroupname,
945 'description': 'for nodegroup %s'%nodegroupname,
947 print 'located tag (type)',nodegroupname,'as',tag_type_id
949 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
951 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
952 print 'created nodegroup',nodegroupname,'from tagname',nodegroupname,'and value','yes'
953 # set node tag on all nodes, value='yes'
954 for nodename in group_nodes:
956 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
958 traceback.print_exc()
959 print 'node',nodename,'seems to already have tag',nodegroupname
962 expect_yes = self.apiserver.GetNodeTags(auth,
963 {'hostname':nodename,
964 'tagname':nodegroupname},
965 ['value'])[0]['value']
966 if expect_yes != "yes":
967 print 'Mismatch node tag on node',nodename,'got',expect_yes
970 if not self.options.dry_run:
971 print 'Cannot find tag',nodegroupname,'on node',nodename
975 print 'cleaning nodegroup',nodegroupname
976 self.apiserver.DeleteNodeGroup(auth,nodegroupname)
978 traceback.print_exc()
982 # a list of TestNode objs
983 def all_nodes (self):
985 for site_spec in self.plc_spec['sites']:
986 test_site = TestSite (self,site_spec)
987 for node_spec in site_spec['nodes']:
988 nodes.append(TestNode (self,test_site,node_spec))
991 # return a list of tuples (nodename,qemuname)
992 def all_node_infos (self) :
994 for site_spec in self.plc_spec['sites']:
995 node_infos += [ (node_spec['node_fields']['hostname'],node_spec['host_box']) \
996 for node_spec in site_spec['nodes'] ]
999 def all_nodenames (self): return [ x[0] for x in self.all_node_infos() ]
1000 def all_reservable_nodenames (self):
1002 for site_spec in self.plc_spec['sites']:
1003 for node_spec in site_spec['nodes']:
1004 node_fields=node_spec['node_fields']
1005 if 'node_type' in node_fields and node_fields['node_type']=='reservable':
1006 res.append(node_fields['hostname'])
1009 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
1010 def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period_seconds=15):
1011 if self.options.dry_run:
1015 class CompleterTaskBootState (CompleterTask):
1016 def __init__ (self, test_plc,hostname):
1017 self.test_plc=test_plc
1018 self.hostname=hostname
1019 self.last_boot_state='undef'
1020 def actual_run (self):
1022 node = self.test_plc.apiserver.GetNodes(self.test_plc.auth_root(), [ self.hostname ],
1024 self.last_boot_state = node['boot_state']
1025 return self.last_boot_state == target_boot_state
1029 return "CompleterTaskBootState with node %s"%self.hostname
1030 def failure_epilogue (self):
1031 print "node %s in state %s - expected %s"%(self.hostname,self.last_boot_state,target_boot_state)
1033 timeout = timedelta(minutes=timeout_minutes)
1034 graceout = timedelta(minutes=silent_minutes)
1035 period = timedelta(seconds=period_seconds)
1036 # the nodes that haven't checked yet - start with a full list and shrink over time
1037 utils.header("checking nodes boot state (expected %s)"%target_boot_state)
1038 tasks = [ CompleterTaskBootState (self,hostname) \
1039 for (hostname,_) in self.all_node_infos() ]
1040 return Completer (tasks).run (timeout, graceout, period)
1042 def nodes_booted(self):
1043 return self.nodes_check_boot_state('boot',timeout_minutes=30,silent_minutes=28)
1045 def probe_kvm_iptables (self):
1046 (_,kvmbox) = self.all_node_infos()[0]
1047 TestSsh(kvmbox).run("iptables-save")
1051 def check_nodes_ping(self,timeout_seconds=120,period_seconds=10):
1052 class CompleterTaskPingNode (CompleterTask):
1053 def __init__ (self, hostname):
1054 self.hostname=hostname
1055 def run(self,silent):
1056 command="ping -c 1 -w 1 %s >& /dev/null"%self.hostname
1057 return utils.system (command, silent=silent)==0
1058 def failure_epilogue (self):
1059 print "Cannot ping node with name %s"%self.hostname
1060 timeout=timedelta (seconds=timeout_seconds)
1062 period=timedelta (seconds=period_seconds)
1063 node_infos = self.all_node_infos()
1064 tasks = [ CompleterTaskPingNode (h) for (h,_) in node_infos ]
1065 return Completer (tasks).run (timeout, graceout, period)
1067 # ping node before we try to reach ssh, helpful for troubleshooting failing bootCDs
1068 def ping_node (self):
1070 return self.check_nodes_ping ()
1072 def check_nodes_ssh(self,debug,timeout_minutes,silent_minutes,period_seconds=15):
1074 timeout = timedelta(minutes=timeout_minutes)
1075 graceout = timedelta(minutes=silent_minutes)
1076 period = timedelta(seconds=period_seconds)
1077 vservername=self.vservername
1080 local_key = "keys/%(vservername)s-debug.rsa"%locals()
1083 local_key = "keys/key_admin.rsa"
1084 utils.header("checking ssh access to nodes (expected in %s mode)"%message)
1085 node_infos = self.all_node_infos()
1086 tasks = [ CompleterTaskNodeSsh (nodename, qemuname, local_key, boot_state=message) \
1087 for (nodename,qemuname) in node_infos ]
1088 return Completer (tasks).run (timeout, graceout, period)
1090 def ssh_node_debug(self):
1091 "Tries to ssh into nodes in debug mode with the debug ssh key"
1092 return self.check_nodes_ssh(debug=True,
1093 timeout_minutes=self.ssh_node_debug_timeout,
1094 silent_minutes=self.ssh_node_debug_silent)
1096 def ssh_node_boot(self):
1097 "Tries to ssh into nodes in production mode with the root ssh key"
1098 return self.check_nodes_ssh(debug=False,
1099 timeout_minutes=self.ssh_node_boot_timeout,
1100 silent_minutes=self.ssh_node_boot_silent)
1102 def node_bmlogs(self):
1103 "Checks that there's a non-empty dir. /var/log/bm/raw"
1104 return utils.system(self.actual_command_in_guest("ls /var/log/bm/raw"))==0
1107 def qemu_local_init (self): pass
1109 def bootcd (self): pass
1111 def qemu_local_config (self): pass
1113 def nodestate_reinstall (self): pass
1115 def nodestate_safeboot (self): pass
1117 def nodestate_boot (self): pass
1119 def nodestate_show (self): pass
1121 def qemu_export (self): pass
1123 ### check hooks : invoke scripts from hooks/{node,slice}
1124 def check_hooks_node (self):
1125 return self.locate_first_node().check_hooks()
1126 def check_hooks_sliver (self) :
1127 return self.locate_first_sliver().check_hooks()
1129 def check_hooks (self):
1130 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
1131 return self.check_hooks_node() and self.check_hooks_sliver()
1134 def do_check_initscripts(self):
1135 class CompleterTaskInitscript (CompleterTask):
1136 def __init__ (self, test_sliver, stamp):
1137 self.test_sliver=test_sliver
1139 def actual_run (self):
1140 return self.test_sliver.check_initscript_stamp (self.stamp)
1142 return "initscript checker for %s"%self.test_sliver.name()
1143 def failure_epilogue (self):
1144 print "initscript stamp %s not found in sliver %s"%(self.stamp,self.test_sliver.name())
1147 for slice_spec in self.plc_spec['slices']:
1148 if not slice_spec.has_key('initscriptstamp'):
1150 stamp=slice_spec['initscriptstamp']
1151 slicename=slice_spec['slice_fields']['name']
1152 for nodename in slice_spec['nodenames']:
1153 print 'nodename',nodename,'slicename',slicename,'stamp',stamp
1154 (site,node) = self.locate_node (nodename)
1155 # xxx - passing the wrong site - probably harmless
1156 test_site = TestSite (self,site)
1157 test_slice = TestSlice (self,test_site,slice_spec)
1158 test_node = TestNode (self,test_site,node)
1159 test_sliver = TestSliver (self, test_node, test_slice)
1160 tasks.append ( CompleterTaskInitscript (test_sliver, stamp))
1161 return Completer (tasks).run (timedelta(minutes=5), timedelta(minutes=4), timedelta(seconds=10))
1163 def check_initscripts(self):
1164 "check that the initscripts have triggered"
1165 return self.do_check_initscripts()
1167 def initscripts (self):
1168 "create initscripts with PLCAPI"
1169 for initscript in self.plc_spec['initscripts']:
1170 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
1171 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
1174 def delete_initscripts (self):
1175 "delete initscripts with PLCAPI"
1176 for initscript in self.plc_spec['initscripts']:
1177 initscript_name = initscript['initscript_fields']['name']
1178 print('Attempting to delete %s in plc %s'%(initscript_name,self.plc_spec['name']))
1180 self.apiserver.DeleteInitScript(self.auth_root(),initscript_name)
1181 print initscript_name,'deleted'
1183 print 'deletion went wrong - probably did not exist'
1188 "create slices with PLCAPI"
1189 return self.do_slices(action="add")
1191 def delete_slices (self):
1192 "delete slices with PLCAPI"
1193 return self.do_slices(action="delete")
1195 def fill_slices (self):
1196 "add nodes in slices with PLCAPI"
1197 return self.do_slices(action="fill")
1199 def empty_slices (self):
1200 "remove nodes from slices with PLCAPI"
1201 return self.do_slices(action="empty")
1203 def do_slices (self, action="add"):
1204 for slice in self.plc_spec['slices']:
1205 site_spec = self.locate_site (slice['sitename'])
1206 test_site = TestSite(self,site_spec)
1207 test_slice=TestSlice(self,test_site,slice)
1208 if action == "delete":
1209 test_slice.delete_slice()
1210 elif action=="fill":
1211 test_slice.add_nodes()
1212 elif action=="empty":
1213 test_slice.delete_nodes()
1215 test_slice.create_slice()
1218 @slice_mapper__tasks(20,10,15)
1219 def ssh_slice(self): pass
1220 @slice_mapper__tasks(20,19,15)
1221 def ssh_slice_off (self): pass
1222 @slice_mapper__tasks(1,1,15)
1223 def slice_fs_present(self): pass
1224 @slice_mapper__tasks(1,1,15)
1225 def slice_fs_deleted(self): pass
1227 # use another name so we can exclude/ignore it from the tests on the nightly command line
1228 def ssh_slice_again(self): return self.ssh_slice()
1229 # note that simply doing ssh_slice_again=ssh_slice would kind of work too
1230 # but for some reason the ignore-wrapping thing would not
1233 def ssh_slice_basics(self): pass
1235 def check_vsys_defaults(self): pass
1238 def keys_clear_known_hosts (self): pass
1240 def plcapi_urls (self):
1241 return PlcapiUrlScanner (self.auth_root(),ip=self.vserverip).scan()
1243 def speed_up_slices (self):
1244 "tweak nodemanager cycle (wait time) to 30+/-10 s"
1245 return self._speed_up_slices (30,10)
1246 def super_speed_up_slices (self):
1247 "dev mode: tweak nodemanager cycle (wait time) to 5+/-1 s"
1248 return self._speed_up_slices (5,1)
1250 def _speed_up_slices (self, p, r):
1251 # create the template on the server-side
1252 template="%s.nodemanager"%self.name()
1253 template_file = open (template,"w")
1254 template_file.write('OPTIONS="-p %s -r %s -d"\n'%(p,r))
1255 template_file.close()
1256 in_vm="/var/www/html/PlanetLabConf/nodemanager"
1257 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1258 self.test_ssh.copy_abs(template,remote)
1260 if not self.apiserver.GetConfFiles (self.auth_root(),
1261 {'dest':'/etc/sysconfig/nodemanager'}):
1262 self.apiserver.AddConfFile (self.auth_root(),
1263 {'dest':'/etc/sysconfig/nodemanager',
1264 'source':'PlanetLabConf/nodemanager',
1265 'postinstall_cmd':'service nm restart',})
1268 def debug_nodemanager (self):
1269 "sets verbose mode for nodemanager, and speeds up cycle even more (needs speed_up_slices first)"
1270 template="%s.nodemanager"%self.name()
1271 template_file = open (template,"w")
1272 template_file.write('OPTIONS="-p 10 -r 6 -v -d"\n')
1273 template_file.close()
1274 in_vm="/var/www/html/PlanetLabConf/nodemanager"
1275 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1276 self.test_ssh.copy_abs(template,remote)
1280 def qemu_start (self) : pass
1283 def qemu_timestamp (self) : pass
1285 # when a spec refers to a node possibly on another plc
1286 def locate_sliver_obj_cross (self, nodename, slicename, other_plcs):
1287 for plc in [ self ] + other_plcs:
1289 return plc.locate_sliver_obj (nodename, slicename)
1292 raise Exception, "Cannot locate sliver %s@%s among all PLCs"%(nodename,slicename)
1294 # implement this one as a cross step so that we can take advantage of different nodes
1295 # in multi-plcs mode
1296 def cross_check_tcp (self, other_plcs):
1297 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
1298 if 'tcp_specs' not in self.plc_spec or not self.plc_spec['tcp_specs']:
1299 utils.header ("check_tcp: no/empty config found")
1301 specs = self.plc_spec['tcp_specs']
1306 s_test_sliver = self.locate_sliver_obj_cross (spec['server_node'],spec['server_slice'],other_plcs)
1307 if not s_test_sliver.run_tcp_server(port,timeout=20):
1311 # idem for the client side
1312 c_test_sliver = self.locate_sliver_obj_cross (spec['client_node'],spec['client_slice'],other_plcs)
1313 # use nodename from locatesd sliver, unless 'client_connect' is set
1314 if 'client_connect' in spec:
1315 destination = spec['client_connect']
1317 destination=s_test_sliver.test_node.name()
1318 if not c_test_sliver.run_tcp_client(destination,port):
1322 # painfully enough, we need to allow for some time as netflow might show up last
1323 def check_system_slice (self):
1324 "all nodes: check that a system slice is alive"
1325 # netflow currently not working in the lxc distro
1326 # drl not built at all in the wtx distro
1327 # if we find either of them we're happy
1328 return self.check_netflow() or self.check_drl()
1331 def check_netflow (self): return self._check_system_slice ('netflow')
1332 def check_drl (self): return self._check_system_slice ('drl')
1334 # we have the slices up already here, so it should not take too long
1335 def _check_system_slice (self, slicename, timeout_minutes=5, period_seconds=15):
1336 class CompleterTaskSystemSlice (CompleterTask):
1337 def __init__ (self, test_node, dry_run):
1338 self.test_node=test_node
1339 self.dry_run=dry_run
1340 def actual_run (self):
1341 return self.test_node._check_system_slice (slicename, dry_run=self.dry_run)
1343 return "System slice %s @ %s"%(slicename, self.test_node.name())
1344 def failure_epilogue (self):
1345 print "COULD not find system slice %s @ %s"%(slicename, self.test_node.name())
1346 timeout = timedelta(minutes=timeout_minutes)
1347 silent = timedelta (0)
1348 period = timedelta (seconds=period_seconds)
1349 tasks = [ CompleterTaskSystemSlice (test_node, self.options.dry_run) \
1350 for test_node in self.all_nodes() ]
1351 return Completer (tasks) . run (timeout, silent, period)
1353 def plcsh_stress_test (self):
1354 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
1355 # install the stress-test in the plc image
1356 location = "/usr/share/plc_api/plcsh_stress_test.py"
1357 remote="%s/%s"%(self.vm_root_in_host(),location)
1358 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1360 command += " -- --check"
1361 if self.options.size == 1:
1362 command += " --tiny"
1363 return ( self.run_in_guest(command) == 0)
1365 # populate runs the same utility without slightly different options
1366 # in particular runs with --preserve (dont cleanup) and without --check
1367 # also it gets run twice, once with the --foreign option for creating fake foreign entries
1369 def sfa_install_all (self):
1370 "yum install sfa sfa-plc sfa-sfatables sfa-client"
1371 return self.yum_install ("sfa sfa-plc sfa-sfatables sfa-client")
1373 def sfa_install_core(self):
1375 return self.yum_install ("sfa")
1377 def sfa_install_plc(self):
1378 "yum install sfa-plc"
1379 return self.yum_install("sfa-plc")
1381 def sfa_install_sfatables(self):
1382 "yum install sfa-sfatables"
1383 return self.yum_install ("sfa-sfatables")
1385 # for some very odd reason, this sometimes fails with the following symptom
1386 # # yum install sfa-client
1387 # Setting up Install Process
1389 # Downloading Packages:
1390 # Running rpm_check_debug
1391 # Running Transaction Test
1392 # Transaction Test Succeeded
1393 # Running Transaction
1394 # Transaction couldn't start:
1395 # installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem
1396 # [('installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem', (9, '/', 69632L))]
1397 # even though in the same context I have
1398 # [2012.05.23--f14-32-sfastd1-1-vplc07] / # df -h
1399 # Filesystem Size Used Avail Use% Mounted on
1400 # /dev/hdv1 806G 264G 501G 35% /
1401 # none 16M 36K 16M 1% /tmp
1403 # so as a workaround, we first try yum install, and then invoke rpm on the cached rpm...
1404 def sfa_install_client(self):
1405 "yum install sfa-client"
1406 first_try=self.yum_install("sfa-client")
1407 if first_try: return True
1408 utils.header ("********** Regular yum failed - special workaround in place, 2nd chance")
1409 (code,cached_rpm_path)=utils.output_of(self.actual_command_in_guest('find /var/cache/yum -name sfa-client\*.rpm'))
1410 utils.header("rpm_path=<<%s>>"%rpm_path)
1412 self.run_in_guest("rpm -i %s"%cached_rpm_path)
1413 return self.yum_check_installed ("sfa-client")
1415 def sfa_dbclean(self):
1416 "thoroughly wipes off the SFA database"
1417 return self.run_in_guest("sfaadmin reg nuke")==0 or \
1418 self.run_in_guest("sfa-nuke.py")==0 or \
1419 self.run_in_guest("sfa-nuke-plc.py")==0
1421 def sfa_fsclean(self):
1422 "cleanup /etc/sfa/trusted_roots and /var/lib/sfa"
1423 self.run_in_guest("rm -rf /etc/sfa/trusted_roots /var/lib/sfa/authorities")
1426 def sfa_plcclean(self):
1427 "cleans the PLC entries that were created as a side effect of running the script"
1429 sfa_spec=self.plc_spec['sfa']
1431 for auth_sfa_spec in sfa_spec['auth_sfa_specs']:
1432 login_base=auth_sfa_spec['login_base']
1433 try: self.apiserver.DeleteSite (self.auth_root(),login_base)
1434 except: print "Site %s already absent from PLC db"%login_base
1436 for spec_name in ['pi_spec','user_spec']:
1437 user_spec=auth_sfa_spec[spec_name]
1438 username=user_spec['email']
1439 try: self.apiserver.DeletePerson(self.auth_root(),username)
1441 # this in fact is expected as sites delete their members
1442 #print "User %s already absent from PLC db"%username
1445 print "REMEMBER TO RUN sfa_import AGAIN"
1448 def sfa_uninstall(self):
1449 "uses rpm to uninstall sfa - ignore result"
1450 self.run_in_guest("rpm -e sfa sfa-sfatables sfa-client sfa-plc")
1451 self.run_in_guest("rm -rf /var/lib/sfa")
1452 self.run_in_guest("rm -rf /etc/sfa")
1453 self.run_in_guest("rm -rf /var/log/sfa_access.log /var/log/sfa_import_plc.log /var/log/sfa.daemon")
1455 self.run_in_guest("rpm -e --noscripts sfa-plc")
1458 ### run unit tests for SFA
1459 # NOTE: for some reason on f14/i386, yum install sfa-tests fails for no reason
1460 # Running Transaction
1461 # Transaction couldn't start:
1462 # installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem
1463 # [('installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem', (9, '/', 208896L))]
1464 # no matter how many Gbs are available on the testplc
1465 # could not figure out what's wrong, so...
1466 # if the yum install phase fails, consider the test is successful
1467 # other combinations will eventually run it hopefully
1468 def sfa_utest(self):
1469 "yum install sfa-tests and run SFA unittests"
1470 self.run_in_guest("yum -y install sfa-tests")
1471 # failed to install - forget it
1472 if self.run_in_guest("rpm -q sfa-tests")!=0:
1473 utils.header("WARNING: SFA unit tests failed to install, ignoring")
1475 return self.run_in_guest("/usr/share/sfa/tests/testAll.py")==0
1479 dirname="conf.%s"%self.plc_spec['name']
1480 if not os.path.isdir(dirname):
1481 utils.system("mkdir -p %s"%dirname)
1482 if not os.path.isdir(dirname):
1483 raise Exception,"Cannot create config dir for plc %s"%self.name()
1486 def conffile(self,filename):
1487 return "%s/%s"%(self.confdir(),filename)
1488 def confsubdir(self,dirname,clean,dry_run=False):
1489 subdirname="%s/%s"%(self.confdir(),dirname)
1491 utils.system("rm -rf %s"%subdirname)
1492 if not os.path.isdir(subdirname):
1493 utils.system("mkdir -p %s"%subdirname)
1494 if not dry_run and not os.path.isdir(subdirname):
1495 raise "Cannot create config subdir %s for plc %s"%(dirname,self.name())
1498 def conffile_clean (self,filename):
1499 filename=self.conffile(filename)
1500 return utils.system("rm -rf %s"%filename)==0
1503 def sfa_configure(self):
1504 "run sfa-config-tty"
1505 tmpname=self.conffile("sfa-config-tty")
1506 fileconf=open(tmpname,'w')
1507 for (var,value) in self.plc_spec['sfa']['settings'].iteritems():
1508 fileconf.write ('e %s\n%s\n'%(var,value))
1509 # # the way plc_config handles booleans just sucks..
1512 # if self.plc_spec['sfa'][var]: val='true'
1513 # fileconf.write ('e %s\n%s\n'%(var,val))
1514 fileconf.write('w\n')
1515 fileconf.write('R\n')
1516 fileconf.write('q\n')
1518 utils.system('cat %s'%tmpname)
1519 self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
1522 def aggregate_xml_line(self):
1523 port=self.plc_spec['sfa']['neighbours-port']
1524 return '<aggregate addr="%s" hrn="%s" port="%r"/>' % \
1525 (self.vserverip,self.plc_spec['sfa']['settings']['SFA_REGISTRY_ROOT_AUTH'],port)
1527 def registry_xml_line(self):
1528 return '<registry addr="%s" hrn="%s" port="12345"/>' % \
1529 (self.vserverip,self.plc_spec['sfa']['settings']['SFA_REGISTRY_ROOT_AUTH'])
1532 # a cross step that takes all other plcs in argument
1533 def cross_sfa_configure(self, other_plcs):
1534 "writes aggregates.xml and registries.xml that point to all other PLCs in the test"
1535 # of course with a single plc, other_plcs is an empty list
1538 agg_fname=self.conffile("agg.xml")
1539 file(agg_fname,"w").write("<aggregates>%s</aggregates>\n" % \
1540 " ".join([ plc.aggregate_xml_line() for plc in other_plcs ]))
1541 utils.header ("(Over)wrote %s"%agg_fname)
1542 reg_fname=self.conffile("reg.xml")
1543 file(reg_fname,"w").write("<registries>%s</registries>\n" % \
1544 " ".join([ plc.registry_xml_line() for plc in other_plcs ]))
1545 utils.header ("(Over)wrote %s"%reg_fname)
1546 return self.test_ssh.copy_abs(agg_fname,'/%s/etc/sfa/aggregates.xml'%self.vm_root_in_host())==0 \
1547 and self.test_ssh.copy_abs(reg_fname,'/%s/etc/sfa/registries.xml'%self.vm_root_in_host())==0
1549 def sfa_import(self):
1550 "use sfaadmin to import from plc"
1551 auth=self.plc_spec['sfa']['settings']['SFA_REGISTRY_ROOT_AUTH']
1552 return self.run_in_guest('sfaadmin reg import_registry')==0
1554 def sfa_start(self):
1556 return self.start_service('sfa')
1559 def sfi_configure(self):
1560 "Create /root/sfi on the plc side for sfi client configuration"
1561 if self.options.dry_run:
1562 utils.header("DRY RUN - skipping step")
1564 sfa_spec=self.plc_spec['sfa']
1565 # cannot use auth_sfa_mapper to pass dir_name
1566 for slice_spec in self.plc_spec['sfa']['auth_sfa_specs']:
1567 test_slice=TestAuthSfa(self,slice_spec)
1568 dir_basename=os.path.basename(test_slice.sfi_path())
1569 dir_name=self.confsubdir("dot-sfi/%s"%dir_basename,clean=True,dry_run=self.options.dry_run)
1570 test_slice.sfi_configure(dir_name)
1571 # push into the remote /root/sfi area
1572 location = test_slice.sfi_path()
1573 remote="%s/%s"%(self.vm_root_in_host(),location)
1574 self.test_ssh.mkdir(remote,abs=True)
1575 # need to strip last level or remote otherwise we get an extra dir level
1576 self.test_ssh.copy_abs(dir_name, os.path.dirname(remote), recursive=True)
1580 def sfi_clean (self):
1581 "clean up /root/sfi on the plc side"
1582 self.run_in_guest("rm -rf /root/sfi")
1585 def sfa_rspec_empty(self):
1586 "expose a static empty rspec (ships with the tests module) in the sfi directory"
1587 filename="empty-rspec.xml"
1589 for slice_spec in self.plc_spec['sfa']['auth_sfa_specs']:
1590 test_slice=TestAuthSfa(self,slice_spec)
1591 in_vm = test_slice.sfi_path()
1592 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1593 if self.test_ssh.copy_abs (filename, remote) !=0: overall=False
1597 def sfa_register_site (self): pass
1599 def sfa_register_pi (self): pass
1601 def sfa_register_user(self): pass
1603 def sfa_update_user(self): pass
1605 def sfa_register_slice(self): pass
1607 def sfa_renew_slice(self): pass
1609 def sfa_get_expires(self): pass
1611 def sfa_discover(self): pass
1613 def sfa_rspec(self): pass
1615 def sfa_allocate(self): pass
1617 def sfa_allocate_empty(self): pass
1619 def sfa_provision(self): pass
1621 def sfa_provision_empty(self): pass
1623 def sfa_check_slice_plc(self): pass
1625 def sfa_check_slice_plc_empty(self): pass
1627 def sfa_update_slice(self): pass
1629 def sfa_remove_user_from_slice(self): pass
1631 def sfa_insert_user_in_slice(self): pass
1633 def sfi_list(self): pass
1635 def sfi_show_site(self): pass
1637 def sfi_show_slice(self): pass
1639 def sfi_show_slice_researchers(self): pass
1641 def ssh_slice_sfa(self): pass
1643 def sfa_delete_user(self): pass
1645 def sfa_delete_slice(self): pass
1649 return self.stop_service ('sfa')
1651 def populate (self):
1652 "creates random entries in the PLCAPI"
1653 # install the stress-test in the plc image
1654 location = "/usr/share/plc_api/plcsh_stress_test.py"
1655 remote="%s/%s"%(self.vm_root_in_host(),location)
1656 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1658 command += " -- --preserve --short-names"
1659 local = (self.run_in_guest(command) == 0);
1660 # second run with --foreign
1661 command += ' --foreign'
1662 remote = (self.run_in_guest(command) == 0);
1663 return ( local and remote)
1665 def gather_logs (self):
1666 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1667 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1668 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1669 # (1.c) get the plc's /root/sfi -> logs/sfi.<plcname>/
1670 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1671 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1672 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1674 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1675 self.gather_var_logs ()
1677 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1678 self.gather_pgsql_logs ()
1680 print "-------------------- TestPlc.gather_logs : PLC's /root/sfi/"
1681 self.gather_root_sfi ()
1683 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1684 for site_spec in self.plc_spec['sites']:
1685 test_site = TestSite (self,site_spec)
1686 for node_spec in site_spec['nodes']:
1687 test_node=TestNode(self,test_site,node_spec)
1688 test_node.gather_qemu_logs()
1690 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1691 self.gather_nodes_var_logs()
1693 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1694 self.gather_slivers_var_logs()
1697 def gather_slivers_var_logs(self):
1698 for test_sliver in self.all_sliver_objs():
1699 remote = test_sliver.tar_var_logs()
1700 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
1701 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
1702 utils.system(command)
1705 def gather_var_logs (self):
1706 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
1707 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1708 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
1709 utils.system(command)
1710 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd"%self.name()
1711 utils.system(command)
1713 def gather_pgsql_logs (self):
1714 utils.system("mkdir -p logs/myplc.pgsql-log.%s"%self.name())
1715 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1716 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -"%self.name()
1717 utils.system(command)
1719 def gather_root_sfi (self):
1720 utils.system("mkdir -p logs/sfi.%s"%self.name())
1721 to_plc = self.actual_command_in_guest("tar -C /root/sfi/ -cf - .")
1722 command = to_plc + "| tar -C logs/sfi.%s -xf -"%self.name()
1723 utils.system(command)
1725 def gather_nodes_var_logs (self):
1726 for site_spec in self.plc_spec['sites']:
1727 test_site = TestSite (self,site_spec)
1728 for node_spec in site_spec['nodes']:
1729 test_node=TestNode(self,test_site,node_spec)
1730 test_ssh = TestSsh (test_node.name(),key="keys/key_admin.rsa")
1731 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1732 command = command + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
1733 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
1734 utils.system(command)
1737 # returns the filename to use for sql dump/restore, using options.dbname if set
1738 def dbfile (self, database):
1739 # uses options.dbname if it is found
1741 name=self.options.dbname
1742 if not isinstance(name,StringTypes):
1748 return "/root/%s-%s.sql"%(database,name)
1750 def plc_db_dump(self):
1751 'dump the planetlab5 DB in /root in the PLC - filename has time'
1752 dump=self.dbfile("planetab5")
1753 self.run_in_guest('pg_dump -U pgsqluser planetlab5 -f '+ dump)
1754 utils.header('Dumped planetlab5 database in %s'%dump)
1757 def plc_db_restore(self):
1758 'restore the planetlab5 DB - looks broken, but run -n might help'
1759 dump=self.dbfile("planetab5")
1760 ##stop httpd service
1761 self.run_in_guest('service httpd stop')
1762 # xxx - need another wrapper
1763 self.run_in_guest_piped('echo drop database planetlab5','psql --user=pgsqluser template1')
1764 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab5')
1765 self.run_in_guest('psql -U pgsqluser planetlab5 -f '+dump)
1766 ##starting httpd service
1767 self.run_in_guest('service httpd start')
1769 utils.header('Database restored from ' + dump)
1772 def create_ignore_steps ():
1773 for step in TestPlc.default_steps + TestPlc.other_steps:
1774 # default step can have a plc qualifier
1775 if '@' in step: (step,qualifier)=step.split('@')
1776 # or be defined as forced or ignored by default
1777 for keyword in ['_ignore','_force']:
1778 if step.endswith (keyword): step=step.replace(keyword,'')
1779 if step == SEP or step == SEPSFA : continue
1780 method=getattr(TestPlc,step)
1782 wrapped=ignore_result(method)
1783 # wrapped.__doc__ = method.__doc__ + " (run in ignore-result mode)"
1784 setattr(TestPlc, name, wrapped)
1787 # def ssh_slice_again_ignore (self): pass
1789 # def check_initscripts_ignore (self): pass
1791 def standby_1_through_20(self):
1792 """convenience function to wait for a specified number of minutes"""
1795 def standby_1(): pass
1797 def standby_2(): pass
1799 def standby_3(): pass
1801 def standby_4(): pass
1803 def standby_5(): pass
1805 def standby_6(): pass
1807 def standby_7(): pass
1809 def standby_8(): pass
1811 def standby_9(): pass
1813 def standby_10(): pass
1815 def standby_11(): pass
1817 def standby_12(): pass
1819 def standby_13(): pass
1821 def standby_14(): pass
1823 def standby_15(): pass
1825 def standby_16(): pass
1827 def standby_17(): pass
1829 def standby_18(): pass
1831 def standby_19(): pass
1833 def standby_20(): pass
1835 # convenience for debugging the test logic
1836 def yes (self): return True
1837 def no (self): return False