1 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
2 # Copyright (C) 2010 INRIA
9 from datetime import datetime, timedelta
10 from types import StringTypes
13 from Completer import Completer, CompleterTask
14 from TestSite import TestSite
15 from TestNode import TestNode, CompleterTaskNodeSsh
16 from TestUser import TestUser
17 from TestKey import TestKey
18 from TestSlice import TestSlice
19 from TestSliver import TestSliver
20 from TestBoxQemu import TestBoxQemu
21 from TestSsh import TestSsh
22 from TestApiserver import TestApiserver
23 from TestAuthSfa import TestAuthSfa
24 from PlcapiUrlScanner import PlcapiUrlScanner
26 has_sfa_cache_filename="sfa-cache"
28 # step methods must take (self) and return a boolean (options is a member of the class)
30 def standby(minutes,dry_run):
31 utils.header('Entering StandBy for %d mn'%minutes)
35 time.sleep(60*minutes)
38 def standby_generic (func):
40 minutes=int(func.__name__.split("_")[1])
41 return standby(minutes,self.options.dry_run)
44 def node_mapper (method):
45 def map_on_nodes(self,*args, **kwds):
47 node_method = TestNode.__dict__[method.__name__]
48 for test_node in self.all_nodes():
49 if not node_method(test_node, *args, **kwds): overall=False
51 # maintain __name__ for ignore_result
52 map_on_nodes.__name__=method.__name__
53 # restore the doc text
54 map_on_nodes.__doc__=TestNode.__dict__[method.__name__].__doc__
57 def slice_mapper (method):
58 def map_on_slices(self):
60 slice_method = TestSlice.__dict__[method.__name__]
61 for slice_spec in self.plc_spec['slices']:
62 site_spec = self.locate_site (slice_spec['sitename'])
63 test_site = TestSite(self,site_spec)
64 test_slice=TestSlice(self,test_site,slice_spec)
65 if not slice_method(test_slice,self.options): overall=False
67 # maintain __name__ for ignore_result
68 map_on_slices.__name__=method.__name__
69 # restore the doc text
70 map_on_slices.__doc__=TestSlice.__dict__[method.__name__].__doc__
73 # run a step but return True so that we can go on
74 def ignore_result (method):
76 # ssh_slice_ignore->ssh_slice
77 ref_name=method.__name__.replace('_ignore','').replace('force_','')
78 ref_method=TestPlc.__dict__[ref_name]
79 result=ref_method(self)
80 print "Actual (but ignored) result for %(ref_name)s is %(result)s"%locals()
81 return Ignored (result)
82 name=method.__name__.replace('_ignore','').replace('force_','')
83 ignoring.__name__=name
84 ignoring.__doc__="ignored version of " + name
87 # a variant that expects the TestSlice method to return a list of CompleterTasks that
88 # are then merged into a single Completer run to avoid wating for all the slices
89 # esp. useful when a test fails of course
90 # because we need to pass arguments we use a class instead..
91 class slice_mapper__tasks (object):
92 # could not get this to work with named arguments
93 def __init__ (self,timeout_minutes,silent_minutes,period_seconds):
94 self.timeout=timedelta(minutes=timeout_minutes)
95 self.silent=timedelta(minutes=silent_minutes)
96 self.period=timedelta(seconds=period_seconds)
97 def __call__ (self, method):
99 # compute augmented method name
100 method_name = method.__name__ + "__tasks"
101 # locate in TestSlice
102 slice_method = TestSlice.__dict__[ method_name ]
105 for slice_spec in self.plc_spec['slices']:
106 site_spec = self.locate_site (slice_spec['sitename'])
107 test_site = TestSite(self,site_spec)
108 test_slice=TestSlice(self,test_site,slice_spec)
109 tasks += slice_method (test_slice, self.options)
110 return Completer (tasks).run (decorator_self.timeout, decorator_self.silent, decorator_self.period)
111 # restore the doc text from the TestSlice method even if a bit odd
112 wrappee.__name__ = method.__name__
113 wrappee.__doc__ = slice_method.__doc__
116 def auth_sfa_mapper (method):
119 auth_method = TestAuthSfa.__dict__[method.__name__]
120 for auth_spec in self.plc_spec['sfa']['auth_sfa_specs']:
121 test_auth=TestAuthSfa(self,auth_spec)
122 if not auth_method(test_auth,self.options): overall=False
124 # restore the doc text
125 actual.__doc__=TestAuthSfa.__dict__[method.__name__].__doc__
129 def __init__ (self,result):
139 'plcvm_delete','plcvm_timestamp','plcvm_create', SEP,
140 'plc_install', 'plc_configure', 'plc_start', SEP,
141 'keys_fetch', 'keys_store', 'keys_clear_known_hosts', SEP,
142 'plcapi_urls','speed_up_slices', SEP,
143 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', 'leases', SEP,
144 # slices created under plcsh interactively seem to be fine but these ones don't have the tags
145 # keep this our of the way for now
146 'check_vsys_defaults_ignore', SEP,
147 # run this first off so it's easier to re-run on another qemu box
148 'qemu_kill_mine', SEP,
149 'nodestate_reinstall', 'qemu_local_init','bootcd', 'qemu_local_config', SEP,
150 'qemu_clean_mine', 'qemu_export', 'qemu_start', 'qemu_timestamp', SEP,
151 'sfa_install_all', 'sfa_configure', 'cross_sfa_configure', 'sfa_start', 'sfa_import', SEPSFA,
152 'sfi_configure@1', 'sfa_register_site@1','sfa_register_pi@1', SEPSFA,
153 'sfa_register_user@1', 'sfa_update_user@1', 'sfa_register_slice@1', 'sfa_renew_slice@1', SEPSFA,
154 'sfa_remove_user_from_slice@1','sfi_show_slice_researchers@1',
155 'sfa_insert_user_in_slice@1','sfi_show_slice_researchers@1', SEPSFA,
156 'sfa_discover@1', 'sfa_rspec@1', 'sfa_allocate@1', 'sfa_provision@1', SEPSFA,
157 'sfa_check_slice_plc@1', 'sfa_update_slice@1', SEPSFA,
158 'sfi_list@1', 'sfi_show_site@1', 'sfa_utest@1', SEPSFA,
159 # we used to run plcsh_stress_test, and then ssh_node_debug and ssh_node_boot
160 # but as the stress test might take a while, we sometimes missed the debug mode..
161 'probe_kvm_iptables',
162 'ping_node', 'ssh_node_debug', 'plcsh_stress_test@1', SEP,
163 'ssh_node_boot', 'node_bmlogs', 'ssh_slice', 'ssh_slice_basics', 'check_initscripts_ignore', SEP,
164 'ssh_slice_sfa@1', SEPSFA,
165 'sfa_rspec_empty@1', 'sfa_allocate_empty@1', 'sfa_provision_empty@1','sfa_check_slice_plc_empty@1', SEPSFA,
166 'sfa_delete_slice@1', 'sfa_delete_user@1', SEPSFA,
167 'cross_check_tcp@1', 'check_system_slice', SEP,
168 # check slices are turned off properly
169 'empty_slices', 'ssh_slice_off', 'slice_fs_deleted_ignore', SEP,
170 # check they are properly re-created with the same name
171 'fill_slices', 'ssh_slice_again', SEP,
172 'gather_logs_force', SEP,
175 'export', 'show_boxes', 'super_speed_up_slices', SEP,
176 'check_hooks', 'plc_stop', 'plcvm_start', 'plcvm_stop', SEP,
177 'delete_initscripts', 'delete_nodegroups','delete_all_sites', SEP,
178 'delete_sites', 'delete_nodes', 'delete_slices', 'keys_clean', SEP,
179 'delete_leases', 'list_leases', SEP,
181 'nodestate_show','nodestate_safeboot','nodestate_boot', SEP,
182 'qemu_list_all', 'qemu_list_mine', 'qemu_kill_all', SEP,
183 'sfa_install_core', 'sfa_install_sfatables', 'sfa_install_plc', 'sfa_install_client', SEPSFA,
184 'sfa_plcclean', 'sfa_dbclean', 'sfa_stop','sfa_uninstall', 'sfi_clean', SEPSFA,
185 'sfa_get_expires', SEPSFA,
186 'plc_db_dump' , 'plc_db_restore', SEP,
187 'check_netflow','check_drl', SEP,
188 'debug_nodemanager', 'slice_fs_present', SEP,
189 'standby_1_through_20','yes','no',SEP,
193 def printable_steps (list):
194 single_line=" ".join(list)+" "
195 return single_line.replace(" "+SEP+" "," \\\n").replace(" "+SEPSFA+" "," \\\n")
197 def valid_step (step):
198 return step != SEP and step != SEPSFA
200 # turn off the sfa-related steps when build has skipped SFA
201 # this was originally for centos5 but is still valid
202 # for up to f12 as recent SFAs with sqlalchemy won't build before f14
204 def _has_sfa_cached (rpms_url):
205 if os.path.isfile(has_sfa_cache_filename):
206 cached=file(has_sfa_cache_filename).read()=="yes"
207 utils.header("build provides SFA (cached):%s"%cached)
209 # warning, we're now building 'sface' so let's be a bit more picky
210 # full builds are expected to return with 0 here
211 utils.header ("Checking if build provides SFA package...")
212 retcod=os.system ("curl --silent %s/ | grep -q sfa-"%rpms_url)==0
213 encoded='yes' if retcod else 'no'
214 file(has_sfa_cache_filename,'w').write(encoded)
218 def check_whether_build_has_sfa (rpms_url):
219 has_sfa=TestPlc._has_sfa_cached(rpms_url)
221 utils.header("build does provide SFA")
223 # move all steps containing 'sfa' from default_steps to other_steps
224 utils.header("SFA package not found - removing steps with sfa or sfi")
225 sfa_steps= [ step for step in TestPlc.default_steps if step.find('sfa')>=0 or step.find("sfi")>=0 ]
226 TestPlc.other_steps += sfa_steps
227 for step in sfa_steps: TestPlc.default_steps.remove(step)
229 def __init__ (self,plc_spec,options):
230 self.plc_spec=plc_spec
232 self.test_ssh=TestSsh(self.plc_spec['host_box'],self.options.buildname)
233 self.vserverip=plc_spec['vserverip']
234 self.vservername=plc_spec['vservername']
235 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
236 self.apiserver=TestApiserver(self.url,options.dry_run)
237 (self.ssh_node_boot_timeout,self.ssh_node_boot_silent)=plc_spec['ssh_node_boot_timers']
238 (self.ssh_node_debug_timeout,self.ssh_node_debug_silent)=plc_spec['ssh_node_debug_timers']
240 def has_addresses_api (self):
241 return self.apiserver.has_method('AddIpAddress')
244 name=self.plc_spec['name']
245 return "%s.%s"%(name,self.vservername)
248 return self.plc_spec['host_box']
251 return self.test_ssh.is_local()
253 # define the API methods on this object through xmlrpc
254 # would help, but not strictly necessary
258 def actual_command_in_guest (self,command, backslash=False):
259 raw1=self.host_to_guest(command)
260 raw2=self.test_ssh.actual_command(raw1,dry_run=self.options.dry_run, backslash=backslash)
263 def start_guest (self):
264 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host(),dry_run=self.options.dry_run))
266 def stop_guest (self):
267 return utils.system(self.test_ssh.actual_command(self.stop_guest_in_host(),dry_run=self.options.dry_run))
269 def run_in_guest (self,command,backslash=False):
270 raw=self.actual_command_in_guest(command,backslash)
271 return utils.system(raw)
273 def run_in_host (self,command):
274 return self.test_ssh.run_in_buildname(command, dry_run=self.options.dry_run)
276 # backslashing turned out so awful at some point that I've turned off auto-backslashing
277 # see e.g. plc_start esp. the version for f14
278 #command gets run in the plc's vm
279 def host_to_guest(self,command):
280 vservername=self.vservername
281 personality=self.options.personality
282 raw="%(personality)s virsh -c lxc:/// lxc-enter-namespace %(vservername)s"%locals()
283 # f14 still needs some extra help
284 if self.options.fcdistro == 'f14':
285 raw +=" -- /usr/bin/env PATH=/bin:/sbin:/usr/bin:/usr/sbin %(command)s" %locals()
287 raw +=" -- /usr/bin/env %(command)s"%locals()
290 # this /vservers thing is legacy...
291 def vm_root_in_host(self):
292 return "/vservers/%s/"%(self.vservername)
294 def vm_timestamp_path (self):
295 return "/vservers/%s/%s.timestamp"%(self.vservername,self.vservername)
297 #start/stop the vserver
298 def start_guest_in_host(self):
299 return "virsh -c lxc:/// start %s"%(self.vservername)
301 def stop_guest_in_host(self):
302 return "virsh -c lxc:/// destroy %s"%(self.vservername)
305 def run_in_guest_piped (self,local,remote):
306 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
308 def yum_check_installed (self, rpms):
309 if isinstance (rpms, list):
311 return self.run_in_guest("rpm -q %s"%rpms)==0
313 # does a yum install in the vs, ignore yum retcod, check with rpm
314 def yum_install (self, rpms):
315 if isinstance (rpms, list):
317 self.run_in_guest("yum -y install %s"%rpms)
318 # yum-complete-transaction comes with yum-utils, that is in vtest.pkgs
319 self.run_in_guest("yum-complete-transaction -y")
320 return self.yum_check_installed (rpms)
322 def auth_root (self):
323 return {'Username':self.plc_spec['settings']['PLC_ROOT_USER'],
324 'AuthMethod':'password',
325 'AuthString':self.plc_spec['settings']['PLC_ROOT_PASSWORD'],
326 'Role' : self.plc_spec['role']
328 def locate_site (self,sitename):
329 for site in self.plc_spec['sites']:
330 if site['site_fields']['name'] == sitename:
332 if site['site_fields']['login_base'] == sitename:
334 raise Exception,"Cannot locate site %s"%sitename
336 def locate_node (self,nodename):
337 for site in self.plc_spec['sites']:
338 for node in site['nodes']:
339 if node['name'] == nodename:
341 raise Exception,"Cannot locate node %s"%nodename
343 def locate_hostname (self,hostname):
344 for site in self.plc_spec['sites']:
345 for node in site['nodes']:
346 if node['node_fields']['hostname'] == hostname:
348 raise Exception,"Cannot locate hostname %s"%hostname
350 def locate_key (self,key_name):
351 for key in self.plc_spec['keys']:
352 if key['key_name'] == key_name:
354 raise Exception,"Cannot locate key %s"%key_name
356 def locate_private_key_from_key_names (self, key_names):
357 # locate the first avail. key
359 for key_name in key_names:
360 key_spec=self.locate_key(key_name)
361 test_key=TestKey(self,key_spec)
362 publickey=test_key.publicpath()
363 privatekey=test_key.privatepath()
364 if os.path.isfile(publickey) and os.path.isfile(privatekey):
366 if found: return privatekey
369 def locate_slice (self, slicename):
370 for slice in self.plc_spec['slices']:
371 if slice['slice_fields']['name'] == slicename:
373 raise Exception,"Cannot locate slice %s"%slicename
375 def all_sliver_objs (self):
377 for slice_spec in self.plc_spec['slices']:
378 slicename = slice_spec['slice_fields']['name']
379 for nodename in slice_spec['nodenames']:
380 result.append(self.locate_sliver_obj (nodename,slicename))
383 def locate_sliver_obj (self,nodename,slicename):
384 (site,node) = self.locate_node(nodename)
385 slice = self.locate_slice (slicename)
387 test_site = TestSite (self, site)
388 test_node = TestNode (self, test_site,node)
389 # xxx the slice site is assumed to be the node site - mhh - probably harmless
390 test_slice = TestSlice (self, test_site, slice)
391 return TestSliver (self, test_node, test_slice)
393 def locate_first_node(self):
394 nodename=self.plc_spec['slices'][0]['nodenames'][0]
395 (site,node) = self.locate_node(nodename)
396 test_site = TestSite (self, site)
397 test_node = TestNode (self, test_site,node)
400 def locate_first_sliver (self):
401 slice_spec=self.plc_spec['slices'][0]
402 slicename=slice_spec['slice_fields']['name']
403 nodename=slice_spec['nodenames'][0]
404 return self.locate_sliver_obj(nodename,slicename)
406 # all different hostboxes used in this plc
407 def get_BoxNodes(self):
408 # maps on sites and nodes, return [ (host_box,test_node) ]
410 for site_spec in self.plc_spec['sites']:
411 test_site = TestSite (self,site_spec)
412 for node_spec in site_spec['nodes']:
413 test_node = TestNode (self, test_site, node_spec)
414 if not test_node.is_real():
415 tuples.append( (test_node.host_box(),test_node) )
416 # transform into a dict { 'host_box' -> [ test_node .. ] }
418 for (box,node) in tuples:
419 if not result.has_key(box):
422 result[box].append(node)
425 # a step for checking this stuff
426 def show_boxes (self):
427 'print summary of nodes location'
428 for (box,nodes) in self.get_BoxNodes().iteritems():
429 print box,":"," + ".join( [ node.name() for node in nodes ] )
432 # make this a valid step
433 def qemu_kill_all(self):
434 'kill all qemu instances on the qemu boxes involved by this setup'
435 # this is the brute force version, kill all qemus on that host box
436 for (box,nodes) in self.get_BoxNodes().iteritems():
437 # pass the first nodename, as we don't push template-qemu on testboxes
438 nodedir=nodes[0].nodedir()
439 TestBoxQemu(box,self.options.buildname).qemu_kill_all(nodedir)
442 # make this a valid step
443 def qemu_list_all(self):
444 'list all qemu instances on the qemu boxes involved by this setup'
445 for (box,nodes) in self.get_BoxNodes().iteritems():
446 # this is the brute force version, kill all qemus on that host box
447 TestBoxQemu(box,self.options.buildname).qemu_list_all()
450 # kill only the qemus related to this test
451 def qemu_list_mine(self):
452 'list qemu instances for our nodes'
453 for (box,nodes) in self.get_BoxNodes().iteritems():
454 # the fine-grain version
459 # kill only the qemus related to this test
460 def qemu_clean_mine(self):
461 'cleanup (rm -rf) qemu instances for our nodes'
462 for (box,nodes) in self.get_BoxNodes().iteritems():
463 # the fine-grain version
468 # kill only the right qemus
469 def qemu_kill_mine(self):
470 'kill the qemu instances for our nodes'
471 for (box,nodes) in self.get_BoxNodes().iteritems():
472 # the fine-grain version
477 #################### display config
479 "show test configuration after localization"
484 # uggly hack to make sure 'run export' only reports about the 1st plc
485 # to avoid confusion - also we use 'inri_slice1' in various aliases..
488 "print cut'n paste-able stuff to export env variables to your shell"
489 # guess local domain from hostname
490 if TestPlc.exported_id>1:
491 print "export GUESTHOSTNAME%d=%s"%(TestPlc.exported_id,self.plc_spec['vservername'])
493 TestPlc.exported_id+=1
494 domain=socket.gethostname().split('.',1)[1]
495 fqdn="%s.%s"%(self.plc_spec['host_box'],domain)
496 print "export BUILD=%s"%self.options.buildname
497 print "export PLCHOSTLXC=%s"%fqdn
498 print "export GUESTNAME=%s"%self.plc_spec['vservername']
499 vplcname=self.plc_spec['vservername'].split('-')[-1]
500 print "export GUESTHOSTNAME=%s.%s"%(vplcname,domain)
501 # find hostname of first node
502 (hostname,qemubox) = self.all_node_infos()[0]
503 print "export KVMHOST=%s.%s"%(qemubox,domain)
504 print "export NODE=%s"%(hostname)
508 always_display_keys=['PLC_WWW_HOST','nodes','sites',]
509 def show_pass (self,passno):
510 for (key,val) in self.plc_spec.iteritems():
511 if not self.options.verbose and key not in TestPlc.always_display_keys: continue
515 self.display_site_spec(site)
516 for node in site['nodes']:
517 self.display_node_spec(node)
518 elif key=='initscripts':
519 for initscript in val:
520 self.display_initscript_spec (initscript)
523 self.display_slice_spec (slice)
526 self.display_key_spec (key)
528 if key not in ['sites','initscripts','slices','keys']:
529 print '+ ',key,':',val
531 def display_site_spec (self,site):
532 print '+ ======== site',site['site_fields']['name']
533 for (k,v) in site.iteritems():
534 if not self.options.verbose and k not in TestPlc.always_display_keys: continue
537 print '+ ','nodes : ',
539 print node['node_fields']['hostname'],'',
545 print user['name'],'',
547 elif k == 'site_fields':
548 print '+ login_base',':',v['login_base']
549 elif k == 'address_fields':
555 def display_initscript_spec (self,initscript):
556 print '+ ======== initscript',initscript['initscript_fields']['name']
558 def display_key_spec (self,key):
559 print '+ ======== key',key['key_name']
561 def display_slice_spec (self,slice):
562 print '+ ======== slice',slice['slice_fields']['name']
563 for (k,v) in slice.iteritems():
576 elif k=='slice_fields':
577 print '+ fields',':',
578 print 'max_nodes=',v['max_nodes'],
583 def display_node_spec (self,node):
584 print "+ node=%s host_box=%s"%(node['name'],node['host_box']),
585 print "hostname=",node['node_fields']['hostname'],
586 print "ip=",node['interface_fields']['ip']
587 if self.options.verbose:
588 utils.pprint("node details",node,depth=3)
590 # another entry point for just showing the boxes involved
591 def display_mapping (self):
592 TestPlc.display_mapping_plc(self.plc_spec)
596 def display_mapping_plc (plc_spec):
597 print '+ MyPLC',plc_spec['name']
598 # WARNING this would not be right for lxc-based PLC's - should be harmless though
599 print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['host_box'],plc_spec['vservername'])
600 print '+\tIP = %s/%s'%(plc_spec['settings']['PLC_API_HOST'],plc_spec['vserverip'])
601 for site_spec in plc_spec['sites']:
602 for node_spec in site_spec['nodes']:
603 TestPlc.display_mapping_node(node_spec)
606 def display_mapping_node (node_spec):
607 print '+ NODE %s'%(node_spec['name'])
608 print '+\tqemu box %s'%node_spec['host_box']
609 print '+\thostname=%s'%node_spec['node_fields']['hostname']
611 # write a timestamp in /vservers/<>.timestamp
612 # cannot be inside the vserver, that causes vserver .. build to cough
613 def plcvm_timestamp (self):
614 "Create a timestamp to remember creation date for this plc"
616 # TODO-lxc check this one
617 # a first approx. is to store the timestamp close to the VM root like vs does
618 stamp_path=self.vm_timestamp_path ()
619 stamp_dir = os.path.dirname (stamp_path)
620 utils.system(self.test_ssh.actual_command("mkdir -p %s"%stamp_dir))
621 return utils.system(self.test_ssh.actual_command("echo %d > %s"%(now,stamp_path)))==0
623 # this is called inconditionnally at the beginning of the test sequence
624 # just in case this is a rerun, so if the vm is not running it's fine
625 def plcvm_delete(self):
626 "vserver delete the test myplc"
627 stamp_path=self.vm_timestamp_path()
628 self.run_in_host("rm -f %s"%stamp_path)
629 self.run_in_host("virsh -c lxc:// destroy %s"%self.vservername)
630 self.run_in_host("virsh -c lxc:// undefine %s"%self.vservername)
631 self.run_in_host("rm -fr /vservers/%s"%self.vservername)
635 # historically the build was being fetched by the tests
636 # now the build pushes itself as a subdir of the tests workdir
637 # so that the tests do not have to worry about extracting the build (svn, git, or whatever)
638 def plcvm_create (self):
639 "vserver creation (no install done)"
640 # push the local build/ dir to the testplc box
642 # a full path for the local calls
643 build_dir=os.path.dirname(sys.argv[0])
644 # sometimes this is empty - set to "." in such a case
645 if not build_dir: build_dir="."
646 build_dir += "/build"
648 # use a standard name - will be relative to remote buildname
650 # remove for safety; do *not* mkdir first, otherwise we end up with build/build/
651 self.test_ssh.rmdir(build_dir)
652 self.test_ssh.copy(build_dir,recursive=True)
653 # the repo url is taken from arch-rpms-url
654 # with the last step (i386) removed
655 repo_url = self.options.arch_rpms_url
656 for level in [ 'arch' ]:
657 repo_url = os.path.dirname(repo_url)
659 # invoke initvm (drop support for vs)
660 script="lbuild-initvm.sh"
662 # pass the vbuild-nightly options to [lv]test-initvm
663 script_options += " -p %s"%self.options.personality
664 script_options += " -d %s"%self.options.pldistro
665 script_options += " -f %s"%self.options.fcdistro
666 script_options += " -r %s"%repo_url
667 vserver_name = self.vservername
669 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
670 script_options += " -n %s"%vserver_hostname
672 print "Cannot reverse lookup %s"%self.vserverip
673 print "This is considered fatal, as this might pollute the test results"
675 create_vserver="%(build_dir)s/%(script)s %(script_options)s %(vserver_name)s"%locals()
676 return self.run_in_host(create_vserver) == 0
679 def plc_install(self):
680 "yum install myplc, noderepo, and the plain bootstrapfs"
682 # workaround for getting pgsql8.2 on centos5
683 if self.options.fcdistro == "centos5":
684 self.run_in_guest("rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-3.noarch.rpm")
687 if self.options.personality == "linux32":
689 elif self.options.personality == "linux64":
692 raise Exception, "Unsupported personality %r"%self.options.personality
693 nodefamily="%s-%s-%s"%(self.options.pldistro,self.options.fcdistro,arch)
696 pkgs_list.append ("slicerepo-%s"%nodefamily)
697 pkgs_list.append ("myplc")
698 pkgs_list.append ("noderepo-%s"%nodefamily)
699 pkgs_list.append ("nodeimage-%s-plain"%nodefamily)
700 pkgs_string=" ".join(pkgs_list)
701 return self.yum_install (pkgs_list)
704 def mod_python(self):
705 """yum install mod_python, useful on f18 and above so as to avoid broken wsgi"""
706 return self.yum_install ( [ 'mod_python' ] )
709 def plc_configure(self):
711 tmpname='%s.plc-config-tty'%(self.name())
712 fileconf=open(tmpname,'w')
713 for (var,value) in self.plc_spec['settings'].iteritems():
714 fileconf.write ('e %s\n%s\n'%(var,value))
715 fileconf.write('w\n')
716 fileconf.write('q\n')
718 utils.system('cat %s'%tmpname)
719 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
720 utils.system('rm %s'%tmpname)
723 # f14 is a bit odd in this respect, although this worked fine in guests up to f18
724 # however using a vplc guest under f20 requires this trick
725 # the symptom is this: service plc start
726 # Starting plc (via systemctl): Failed to get D-Bus connection: \
727 # Failed to connect to socket /org/freedesktop/systemd1/private: Connection refused
728 # weird thing is the doc says f14 uses upstart by default and not systemd
729 # so this sounds kind of harmless
730 def start_service (self,service): return self.start_stop_service (service,'start')
731 def stop_service (self,service): return self.start_stop_service (service,'stop')
733 def start_stop_service (self, service,start_or_stop):
734 "utility to start/stop a service with the special trick for f14"
735 if self.options.fcdistro != 'f14':
736 return self.run_in_guest ("service %s %s"%(service,start_or_stop))==0
738 # patch /sbin/service so it does not reset environment
739 self.run_in_guest ('sed -i -e \\"s,env -i,env,\\" /sbin/service')
740 # this is because our own scripts in turn call service
741 return self.run_in_guest("SYSTEMCTL_SKIP_REDIRECT=true service %s %s"%(service,start_or_stop))==0
745 return self.start_service ('plc')
749 return self.stop_service ('plc')
751 def plcvm_start (self):
752 "start the PLC vserver"
756 def plcvm_stop (self):
757 "stop the PLC vserver"
761 # stores the keys from the config for further use
762 def keys_store(self):
763 "stores test users ssh keys in keys/"
764 for key_spec in self.plc_spec['keys']:
765 TestKey(self,key_spec).store_key()
768 def keys_clean(self):
769 "removes keys cached in keys/"
770 utils.system("rm -rf ./keys")
773 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
774 # for later direct access to the nodes
775 def keys_fetch(self):
776 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
778 if not os.path.isdir(dir):
780 vservername=self.vservername
781 vm_root=self.vm_root_in_host()
783 prefix = 'debug_ssh_key'
784 for ext in [ 'pub', 'rsa' ] :
785 src="%(vm_root)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
786 dst="keys/%(vservername)s-debug.%(ext)s"%locals()
787 if self.test_ssh.fetch(src,dst) != 0: overall=False
791 "create sites with PLCAPI"
792 return self.do_sites()
794 def delete_sites (self):
795 "delete sites with PLCAPI"
796 return self.do_sites(action="delete")
798 def do_sites (self,action="add"):
799 for site_spec in self.plc_spec['sites']:
800 test_site = TestSite (self,site_spec)
801 if (action != "add"):
802 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
803 test_site.delete_site()
804 # deleted with the site
805 #test_site.delete_users()
808 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
809 test_site.create_site()
810 test_site.create_users()
813 def delete_all_sites (self):
814 "Delete all sites in PLC, and related objects"
815 print 'auth_root',self.auth_root()
816 sites = self.apiserver.GetSites(self.auth_root(), {}, ['site_id','login_base'])
818 # keep automatic site - otherwise we shoot in our own foot, root_auth is not valid anymore
819 if site['login_base']==self.plc_spec['settings']['PLC_SLICE_PREFIX']: continue
820 site_id=site['site_id']
821 print 'Deleting site_id',site_id
822 self.apiserver.DeleteSite(self.auth_root(),site_id)
826 "create nodes with PLCAPI"
827 return self.do_nodes()
828 def delete_nodes (self):
829 "delete nodes with PLCAPI"
830 return self.do_nodes(action="delete")
832 def do_nodes (self,action="add"):
833 for site_spec in self.plc_spec['sites']:
834 test_site = TestSite (self,site_spec)
836 utils.header("Deleting nodes in site %s"%test_site.name())
837 for node_spec in site_spec['nodes']:
838 test_node=TestNode(self,test_site,node_spec)
839 utils.header("Deleting %s"%test_node.name())
840 test_node.delete_node()
842 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
843 for node_spec in site_spec['nodes']:
844 utils.pprint('Creating node %s'%node_spec,node_spec)
845 test_node = TestNode (self,test_site,node_spec)
846 test_node.create_node ()
849 def nodegroups (self):
850 "create nodegroups with PLCAPI"
851 return self.do_nodegroups("add")
852 def delete_nodegroups (self):
853 "delete nodegroups with PLCAPI"
854 return self.do_nodegroups("delete")
858 def translate_timestamp (start,grain,timestamp):
859 if timestamp < TestPlc.YEAR: return start+timestamp*grain
860 else: return timestamp
863 def timestamp_printable (timestamp):
864 return time.strftime('%m-%d %H:%M:%S UTC',time.gmtime(timestamp))
867 "create leases (on reservable nodes only, use e.g. run -c default -c resa)"
869 grain=self.apiserver.GetLeaseGranularity(self.auth_root())
870 print 'API answered grain=',grain
871 start=(now/grain)*grain
873 # find out all nodes that are reservable
874 nodes=self.all_reservable_nodenames()
876 utils.header ("No reservable node found - proceeding without leases")
879 # attach them to the leases as specified in plc_specs
880 # this is where the 'leases' field gets interpreted as relative of absolute
881 for lease_spec in self.plc_spec['leases']:
882 # skip the ones that come with a null slice id
883 if not lease_spec['slice']: continue
884 lease_spec['t_from']=TestPlc.translate_timestamp(start,grain,lease_spec['t_from'])
885 lease_spec['t_until']=TestPlc.translate_timestamp(start,grain,lease_spec['t_until'])
886 lease_addition=self.apiserver.AddLeases(self.auth_root(),nodes,
887 lease_spec['slice'],lease_spec['t_from'],lease_spec['t_until'])
888 if lease_addition['errors']:
889 utils.header("Cannot create leases, %s"%lease_addition['errors'])
892 utils.header('Leases on nodes %r for %s from %d (%s) until %d (%s)'%\
893 (nodes,lease_spec['slice'],
894 lease_spec['t_from'],TestPlc.timestamp_printable(lease_spec['t_from']),
895 lease_spec['t_until'],TestPlc.timestamp_printable(lease_spec['t_until'])))
899 def delete_leases (self):
900 "remove all leases in the myplc side"
901 lease_ids= [ l['lease_id'] for l in self.apiserver.GetLeases(self.auth_root())]
902 utils.header("Cleaning leases %r"%lease_ids)
903 self.apiserver.DeleteLeases(self.auth_root(),lease_ids)
906 def list_leases (self):
907 "list all leases known to the myplc"
908 leases = self.apiserver.GetLeases(self.auth_root())
911 current=l['t_until']>=now
912 if self.options.verbose or current:
913 utils.header("%s %s from %s until %s"%(l['hostname'],l['name'],
914 TestPlc.timestamp_printable(l['t_from']),
915 TestPlc.timestamp_printable(l['t_until'])))
918 # create nodegroups if needed, and populate
919 def do_nodegroups (self, action="add"):
920 # 1st pass to scan contents
922 for site_spec in self.plc_spec['sites']:
923 test_site = TestSite (self,site_spec)
924 for node_spec in site_spec['nodes']:
925 test_node=TestNode (self,test_site,node_spec)
926 if node_spec.has_key('nodegroups'):
927 nodegroupnames=node_spec['nodegroups']
928 if isinstance(nodegroupnames,StringTypes):
929 nodegroupnames = [ nodegroupnames ]
930 for nodegroupname in nodegroupnames:
931 if not groups_dict.has_key(nodegroupname):
932 groups_dict[nodegroupname]=[]
933 groups_dict[nodegroupname].append(test_node.name())
934 auth=self.auth_root()
936 for (nodegroupname,group_nodes) in groups_dict.iteritems():
938 print 'nodegroups:','dealing with nodegroup',nodegroupname,'on nodes',group_nodes
939 # first, check if the nodetagtype is here
940 tag_types = self.apiserver.GetTagTypes(auth,{'tagname':nodegroupname})
942 tag_type_id = tag_types[0]['tag_type_id']
944 tag_type_id = self.apiserver.AddTagType(auth,
945 {'tagname':nodegroupname,
946 'description': 'for nodegroup %s'%nodegroupname,
948 print 'located tag (type)',nodegroupname,'as',tag_type_id
950 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
952 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
953 print 'created nodegroup',nodegroupname,'from tagname',nodegroupname,'and value','yes'
954 # set node tag on all nodes, value='yes'
955 for nodename in group_nodes:
957 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
959 traceback.print_exc()
960 print 'node',nodename,'seems to already have tag',nodegroupname
963 expect_yes = self.apiserver.GetNodeTags(auth,
964 {'hostname':nodename,
965 'tagname':nodegroupname},
966 ['value'])[0]['value']
967 if expect_yes != "yes":
968 print 'Mismatch node tag on node',nodename,'got',expect_yes
971 if not self.options.dry_run:
972 print 'Cannot find tag',nodegroupname,'on node',nodename
976 print 'cleaning nodegroup',nodegroupname
977 self.apiserver.DeleteNodeGroup(auth,nodegroupname)
979 traceback.print_exc()
983 # a list of TestNode objs
984 def all_nodes (self):
986 for site_spec in self.plc_spec['sites']:
987 test_site = TestSite (self,site_spec)
988 for node_spec in site_spec['nodes']:
989 nodes.append(TestNode (self,test_site,node_spec))
992 # return a list of tuples (nodename,qemuname)
993 def all_node_infos (self) :
995 for site_spec in self.plc_spec['sites']:
996 node_infos += [ (node_spec['node_fields']['hostname'],node_spec['host_box']) \
997 for node_spec in site_spec['nodes'] ]
1000 def all_nodenames (self): return [ x[0] for x in self.all_node_infos() ]
1001 def all_reservable_nodenames (self):
1003 for site_spec in self.plc_spec['sites']:
1004 for node_spec in site_spec['nodes']:
1005 node_fields=node_spec['node_fields']
1006 if 'node_type' in node_fields and node_fields['node_type']=='reservable':
1007 res.append(node_fields['hostname'])
1010 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
1011 def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period_seconds=15):
1012 if self.options.dry_run:
1016 class CompleterTaskBootState (CompleterTask):
1017 def __init__ (self, test_plc,hostname):
1018 self.test_plc=test_plc
1019 self.hostname=hostname
1020 self.last_boot_state='undef'
1021 def actual_run (self):
1023 node = self.test_plc.apiserver.GetNodes(self.test_plc.auth_root(), [ self.hostname ],
1025 self.last_boot_state = node['boot_state']
1026 return self.last_boot_state == target_boot_state
1030 return "CompleterTaskBootState with node %s"%self.hostname
1031 def failure_epilogue (self):
1032 print "node %s in state %s - expected %s"%(self.hostname,self.last_boot_state,target_boot_state)
1034 timeout = timedelta(minutes=timeout_minutes)
1035 graceout = timedelta(minutes=silent_minutes)
1036 period = timedelta(seconds=period_seconds)
1037 # the nodes that haven't checked yet - start with a full list and shrink over time
1038 utils.header("checking nodes boot state (expected %s)"%target_boot_state)
1039 tasks = [ CompleterTaskBootState (self,hostname) \
1040 for (hostname,_) in self.all_node_infos() ]
1041 return Completer (tasks).run (timeout, graceout, period)
1043 def nodes_booted(self):
1044 return self.nodes_check_boot_state('boot',timeout_minutes=30,silent_minutes=28)
1046 def probe_kvm_iptables (self):
1047 (_,kvmbox) = self.all_node_infos()[0]
1048 TestSsh(kvmbox).run("iptables-save")
1052 def check_nodes_ping(self,timeout_seconds=120,period_seconds=10):
1053 class CompleterTaskPingNode (CompleterTask):
1054 def __init__ (self, hostname):
1055 self.hostname=hostname
1056 def run(self,silent):
1057 command="ping -c 1 -w 1 %s >& /dev/null"%self.hostname
1058 return utils.system (command, silent=silent)==0
1059 def failure_epilogue (self):
1060 print "Cannot ping node with name %s"%self.hostname
1061 timeout=timedelta (seconds=timeout_seconds)
1063 period=timedelta (seconds=period_seconds)
1064 node_infos = self.all_node_infos()
1065 tasks = [ CompleterTaskPingNode (h) for (h,_) in node_infos ]
1066 return Completer (tasks).run (timeout, graceout, period)
1068 # ping node before we try to reach ssh, helpful for troubleshooting failing bootCDs
1069 def ping_node (self):
1071 return self.check_nodes_ping ()
1073 def check_nodes_ssh(self,debug,timeout_minutes,silent_minutes,period_seconds=15):
1075 timeout = timedelta(minutes=timeout_minutes)
1076 graceout = timedelta(minutes=silent_minutes)
1077 period = timedelta(seconds=period_seconds)
1078 vservername=self.vservername
1081 local_key = "keys/%(vservername)s-debug.rsa"%locals()
1084 local_key = "keys/key_admin.rsa"
1085 utils.header("checking ssh access to nodes (expected in %s mode)"%message)
1086 node_infos = self.all_node_infos()
1087 tasks = [ CompleterTaskNodeSsh (nodename, qemuname, local_key, boot_state=message) \
1088 for (nodename,qemuname) in node_infos ]
1089 return Completer (tasks).run (timeout, graceout, period)
1091 def ssh_node_debug(self):
1092 "Tries to ssh into nodes in debug mode with the debug ssh key"
1093 return self.check_nodes_ssh(debug=True,
1094 timeout_minutes=self.ssh_node_debug_timeout,
1095 silent_minutes=self.ssh_node_debug_silent)
1097 def ssh_node_boot(self):
1098 "Tries to ssh into nodes in production mode with the root ssh key"
1099 return self.check_nodes_ssh(debug=False,
1100 timeout_minutes=self.ssh_node_boot_timeout,
1101 silent_minutes=self.ssh_node_boot_silent)
1103 def node_bmlogs(self):
1104 "Checks that there's a non-empty dir. /var/log/bm/raw"
1105 return utils.system(self.actual_command_in_guest("ls /var/log/bm/raw"))==0
1108 def qemu_local_init (self): pass
1110 def bootcd (self): pass
1112 def qemu_local_config (self): pass
1114 def nodestate_reinstall (self): pass
1116 def nodestate_safeboot (self): pass
1118 def nodestate_boot (self): pass
1120 def nodestate_show (self): pass
1122 def qemu_export (self): pass
1124 ### check hooks : invoke scripts from hooks/{node,slice}
1125 def check_hooks_node (self):
1126 return self.locate_first_node().check_hooks()
1127 def check_hooks_sliver (self) :
1128 return self.locate_first_sliver().check_hooks()
1130 def check_hooks (self):
1131 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
1132 return self.check_hooks_node() and self.check_hooks_sliver()
1135 def do_check_initscripts(self):
1136 class CompleterTaskInitscript (CompleterTask):
1137 def __init__ (self, test_sliver, stamp):
1138 self.test_sliver=test_sliver
1140 def actual_run (self):
1141 return self.test_sliver.check_initscript_stamp (self.stamp)
1143 return "initscript checker for %s"%self.test_sliver.name()
1144 def failure_epilogue (self):
1145 print "initscript stamp %s not found in sliver %s"%(self.stamp,self.test_sliver.name())
1148 for slice_spec in self.plc_spec['slices']:
1149 if not slice_spec.has_key('initscriptstamp'):
1151 stamp=slice_spec['initscriptstamp']
1152 slicename=slice_spec['slice_fields']['name']
1153 for nodename in slice_spec['nodenames']:
1154 print 'nodename',nodename,'slicename',slicename,'stamp',stamp
1155 (site,node) = self.locate_node (nodename)
1156 # xxx - passing the wrong site - probably harmless
1157 test_site = TestSite (self,site)
1158 test_slice = TestSlice (self,test_site,slice_spec)
1159 test_node = TestNode (self,test_site,node)
1160 test_sliver = TestSliver (self, test_node, test_slice)
1161 tasks.append ( CompleterTaskInitscript (test_sliver, stamp))
1162 return Completer (tasks).run (timedelta(minutes=5), timedelta(minutes=4), timedelta(seconds=10))
1164 def check_initscripts(self):
1165 "check that the initscripts have triggered"
1166 return self.do_check_initscripts()
1168 def initscripts (self):
1169 "create initscripts with PLCAPI"
1170 for initscript in self.plc_spec['initscripts']:
1171 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
1172 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
1175 def delete_initscripts (self):
1176 "delete initscripts with PLCAPI"
1177 for initscript in self.plc_spec['initscripts']:
1178 initscript_name = initscript['initscript_fields']['name']
1179 print('Attempting to delete %s in plc %s'%(initscript_name,self.plc_spec['name']))
1181 self.apiserver.DeleteInitScript(self.auth_root(),initscript_name)
1182 print initscript_name,'deleted'
1184 print 'deletion went wrong - probably did not exist'
1189 "create slices with PLCAPI"
1190 return self.do_slices(action="add")
1192 def delete_slices (self):
1193 "delete slices with PLCAPI"
1194 return self.do_slices(action="delete")
1196 def fill_slices (self):
1197 "add nodes in slices with PLCAPI"
1198 return self.do_slices(action="fill")
1200 def empty_slices (self):
1201 "remove nodes from slices with PLCAPI"
1202 return self.do_slices(action="empty")
1204 def do_slices (self, action="add"):
1205 for slice in self.plc_spec['slices']:
1206 site_spec = self.locate_site (slice['sitename'])
1207 test_site = TestSite(self,site_spec)
1208 test_slice=TestSlice(self,test_site,slice)
1209 if action == "delete":
1210 test_slice.delete_slice()
1211 elif action=="fill":
1212 test_slice.add_nodes()
1213 elif action=="empty":
1214 test_slice.delete_nodes()
1216 test_slice.create_slice()
1219 @slice_mapper__tasks(20,10,15)
1220 def ssh_slice(self): pass
1221 @slice_mapper__tasks(20,19,15)
1222 def ssh_slice_off (self): pass
1223 @slice_mapper__tasks(1,1,15)
1224 def slice_fs_present(self): pass
1225 @slice_mapper__tasks(1,1,15)
1226 def slice_fs_deleted(self): pass
1228 # use another name so we can exclude/ignore it from the tests on the nightly command line
1229 def ssh_slice_again(self): return self.ssh_slice()
1230 # note that simply doing ssh_slice_again=ssh_slice would kind of work too
1231 # but for some reason the ignore-wrapping thing would not
1234 def ssh_slice_basics(self): pass
1236 def check_vsys_defaults(self): pass
1239 def keys_clear_known_hosts (self): pass
1241 def plcapi_urls (self):
1242 return PlcapiUrlScanner (self.auth_root(),ip=self.vserverip).scan()
1244 def speed_up_slices (self):
1245 "tweak nodemanager cycle (wait time) to 30+/-10 s"
1246 return self._speed_up_slices (30,10)
1247 def super_speed_up_slices (self):
1248 "dev mode: tweak nodemanager cycle (wait time) to 5+/-1 s"
1249 return self._speed_up_slices (5,1)
1251 def _speed_up_slices (self, p, r):
1252 # create the template on the server-side
1253 template="%s.nodemanager"%self.name()
1254 template_file = open (template,"w")
1255 template_file.write('OPTIONS="-p %s -r %s -d"\n'%(p,r))
1256 template_file.close()
1257 in_vm="/var/www/html/PlanetLabConf/nodemanager"
1258 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1259 self.test_ssh.copy_abs(template,remote)
1261 if not self.apiserver.GetConfFiles (self.auth_root(),
1262 {'dest':'/etc/sysconfig/nodemanager'}):
1263 self.apiserver.AddConfFile (self.auth_root(),
1264 {'dest':'/etc/sysconfig/nodemanager',
1265 'source':'PlanetLabConf/nodemanager',
1266 'postinstall_cmd':'service nm restart',})
1269 def debug_nodemanager (self):
1270 "sets verbose mode for nodemanager, and speeds up cycle even more (needs speed_up_slices first)"
1271 template="%s.nodemanager"%self.name()
1272 template_file = open (template,"w")
1273 template_file.write('OPTIONS="-p 10 -r 6 -v -d"\n')
1274 template_file.close()
1275 in_vm="/var/www/html/PlanetLabConf/nodemanager"
1276 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1277 self.test_ssh.copy_abs(template,remote)
1281 def qemu_start (self) : pass
1284 def qemu_timestamp (self) : pass
1286 # when a spec refers to a node possibly on another plc
1287 def locate_sliver_obj_cross (self, nodename, slicename, other_plcs):
1288 for plc in [ self ] + other_plcs:
1290 return plc.locate_sliver_obj (nodename, slicename)
1293 raise Exception, "Cannot locate sliver %s@%s among all PLCs"%(nodename,slicename)
1295 # implement this one as a cross step so that we can take advantage of different nodes
1296 # in multi-plcs mode
1297 def cross_check_tcp (self, other_plcs):
1298 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
1299 if 'tcp_specs' not in self.plc_spec or not self.plc_spec['tcp_specs']:
1300 utils.header ("check_tcp: no/empty config found")
1302 specs = self.plc_spec['tcp_specs']
1307 s_test_sliver = self.locate_sliver_obj_cross (spec['server_node'],spec['server_slice'],other_plcs)
1308 if not s_test_sliver.run_tcp_server(port,timeout=20):
1312 # idem for the client side
1313 c_test_sliver = self.locate_sliver_obj_cross (spec['client_node'],spec['client_slice'],other_plcs)
1314 # use nodename from locatesd sliver, unless 'client_connect' is set
1315 if 'client_connect' in spec:
1316 destination = spec['client_connect']
1318 destination=s_test_sliver.test_node.name()
1319 if not c_test_sliver.run_tcp_client(destination,port):
1323 # painfully enough, we need to allow for some time as netflow might show up last
1324 def check_system_slice (self):
1325 "all nodes: check that a system slice is alive"
1326 # netflow currently not working in the lxc distro
1327 # drl not built at all in the wtx distro
1328 # if we find either of them we're happy
1329 return self.check_netflow() or self.check_drl()
1332 def check_netflow (self): return self._check_system_slice ('netflow')
1333 def check_drl (self): return self._check_system_slice ('drl')
1335 # we have the slices up already here, so it should not take too long
1336 def _check_system_slice (self, slicename, timeout_minutes=5, period_seconds=15):
1337 class CompleterTaskSystemSlice (CompleterTask):
1338 def __init__ (self, test_node, dry_run):
1339 self.test_node=test_node
1340 self.dry_run=dry_run
1341 def actual_run (self):
1342 return self.test_node._check_system_slice (slicename, dry_run=self.dry_run)
1344 return "System slice %s @ %s"%(slicename, self.test_node.name())
1345 def failure_epilogue (self):
1346 print "COULD not find system slice %s @ %s"%(slicename, self.test_node.name())
1347 timeout = timedelta(minutes=timeout_minutes)
1348 silent = timedelta (0)
1349 period = timedelta (seconds=period_seconds)
1350 tasks = [ CompleterTaskSystemSlice (test_node, self.options.dry_run) \
1351 for test_node in self.all_nodes() ]
1352 return Completer (tasks) . run (timeout, silent, period)
1354 def plcsh_stress_test (self):
1355 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
1356 # install the stress-test in the plc image
1357 location = "/usr/share/plc_api/plcsh_stress_test.py"
1358 remote="%s/%s"%(self.vm_root_in_host(),location)
1359 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1361 command += " -- --check"
1362 if self.options.size == 1:
1363 command += " --tiny"
1364 return ( self.run_in_guest(command) == 0)
1366 # populate runs the same utility without slightly different options
1367 # in particular runs with --preserve (dont cleanup) and without --check
1368 # also it gets run twice, once with the --foreign option for creating fake foreign entries
1370 def sfa_install_all (self):
1371 "yum install sfa sfa-plc sfa-sfatables sfa-client"
1372 return self.yum_install ("sfa sfa-plc sfa-sfatables sfa-client")
1374 def sfa_install_core(self):
1376 return self.yum_install ("sfa")
1378 def sfa_install_plc(self):
1379 "yum install sfa-plc"
1380 return self.yum_install("sfa-plc")
1382 def sfa_install_sfatables(self):
1383 "yum install sfa-sfatables"
1384 return self.yum_install ("sfa-sfatables")
1386 # for some very odd reason, this sometimes fails with the following symptom
1387 # # yum install sfa-client
1388 # Setting up Install Process
1390 # Downloading Packages:
1391 # Running rpm_check_debug
1392 # Running Transaction Test
1393 # Transaction Test Succeeded
1394 # Running Transaction
1395 # Transaction couldn't start:
1396 # installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem
1397 # [('installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem', (9, '/', 69632L))]
1398 # even though in the same context I have
1399 # [2012.05.23--f14-32-sfastd1-1-vplc07] / # df -h
1400 # Filesystem Size Used Avail Use% Mounted on
1401 # /dev/hdv1 806G 264G 501G 35% /
1402 # none 16M 36K 16M 1% /tmp
1404 # so as a workaround, we first try yum install, and then invoke rpm on the cached rpm...
1405 def sfa_install_client(self):
1406 "yum install sfa-client"
1407 first_try=self.yum_install("sfa-client")
1408 if first_try: return True
1409 utils.header ("********** Regular yum failed - special workaround in place, 2nd chance")
1410 (code,cached_rpm_path)=utils.output_of(self.actual_command_in_guest('find /var/cache/yum -name sfa-client\*.rpm'))
1411 utils.header("rpm_path=<<%s>>"%rpm_path)
1413 self.run_in_guest("rpm -i %s"%cached_rpm_path)
1414 return self.yum_check_installed ("sfa-client")
1416 def sfa_dbclean(self):
1417 "thoroughly wipes off the SFA database"
1418 return self.run_in_guest("sfaadmin reg nuke")==0 or \
1419 self.run_in_guest("sfa-nuke.py")==0 or \
1420 self.run_in_guest("sfa-nuke-plc.py")==0
1422 def sfa_fsclean(self):
1423 "cleanup /etc/sfa/trusted_roots and /var/lib/sfa"
1424 self.run_in_guest("rm -rf /etc/sfa/trusted_roots /var/lib/sfa/authorities")
1427 def sfa_plcclean(self):
1428 "cleans the PLC entries that were created as a side effect of running the script"
1430 sfa_spec=self.plc_spec['sfa']
1432 for auth_sfa_spec in sfa_spec['auth_sfa_specs']:
1433 login_base=auth_sfa_spec['login_base']
1434 try: self.apiserver.DeleteSite (self.auth_root(),login_base)
1435 except: print "Site %s already absent from PLC db"%login_base
1437 for spec_name in ['pi_spec','user_spec']:
1438 user_spec=auth_sfa_spec[spec_name]
1439 username=user_spec['email']
1440 try: self.apiserver.DeletePerson(self.auth_root(),username)
1442 # this in fact is expected as sites delete their members
1443 #print "User %s already absent from PLC db"%username
1446 print "REMEMBER TO RUN sfa_import AGAIN"
1449 def sfa_uninstall(self):
1450 "uses rpm to uninstall sfa - ignore result"
1451 self.run_in_guest("rpm -e sfa sfa-sfatables sfa-client sfa-plc")
1452 self.run_in_guest("rm -rf /var/lib/sfa")
1453 self.run_in_guest("rm -rf /etc/sfa")
1454 self.run_in_guest("rm -rf /var/log/sfa_access.log /var/log/sfa_import_plc.log /var/log/sfa.daemon")
1456 self.run_in_guest("rpm -e --noscripts sfa-plc")
1459 ### run unit tests for SFA
1460 # NOTE: for some reason on f14/i386, yum install sfa-tests fails for no reason
1461 # Running Transaction
1462 # Transaction couldn't start:
1463 # installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem
1464 # [('installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem', (9, '/', 208896L))]
1465 # no matter how many Gbs are available on the testplc
1466 # could not figure out what's wrong, so...
1467 # if the yum install phase fails, consider the test is successful
1468 # other combinations will eventually run it hopefully
1469 def sfa_utest(self):
1470 "yum install sfa-tests and run SFA unittests"
1471 self.run_in_guest("yum -y install sfa-tests")
1472 # failed to install - forget it
1473 if self.run_in_guest("rpm -q sfa-tests")!=0:
1474 utils.header("WARNING: SFA unit tests failed to install, ignoring")
1476 return self.run_in_guest("/usr/share/sfa/tests/testAll.py")==0
1480 dirname="conf.%s"%self.plc_spec['name']
1481 if not os.path.isdir(dirname):
1482 utils.system("mkdir -p %s"%dirname)
1483 if not os.path.isdir(dirname):
1484 raise Exception,"Cannot create config dir for plc %s"%self.name()
1487 def conffile(self,filename):
1488 return "%s/%s"%(self.confdir(),filename)
1489 def confsubdir(self,dirname,clean,dry_run=False):
1490 subdirname="%s/%s"%(self.confdir(),dirname)
1492 utils.system("rm -rf %s"%subdirname)
1493 if not os.path.isdir(subdirname):
1494 utils.system("mkdir -p %s"%subdirname)
1495 if not dry_run and not os.path.isdir(subdirname):
1496 raise "Cannot create config subdir %s for plc %s"%(dirname,self.name())
1499 def conffile_clean (self,filename):
1500 filename=self.conffile(filename)
1501 return utils.system("rm -rf %s"%filename)==0
1504 def sfa_configure(self):
1505 "run sfa-config-tty"
1506 tmpname=self.conffile("sfa-config-tty")
1507 fileconf=open(tmpname,'w')
1508 for (var,value) in self.plc_spec['sfa']['settings'].iteritems():
1509 fileconf.write ('e %s\n%s\n'%(var,value))
1510 # # the way plc_config handles booleans just sucks..
1513 # if self.plc_spec['sfa'][var]: val='true'
1514 # fileconf.write ('e %s\n%s\n'%(var,val))
1515 fileconf.write('w\n')
1516 fileconf.write('R\n')
1517 fileconf.write('q\n')
1519 utils.system('cat %s'%tmpname)
1520 self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
1523 def aggregate_xml_line(self):
1524 port=self.plc_spec['sfa']['neighbours-port']
1525 return '<aggregate addr="%s" hrn="%s" port="%r"/>' % \
1526 (self.vserverip,self.plc_spec['sfa']['settings']['SFA_REGISTRY_ROOT_AUTH'],port)
1528 def registry_xml_line(self):
1529 return '<registry addr="%s" hrn="%s" port="12345"/>' % \
1530 (self.vserverip,self.plc_spec['sfa']['settings']['SFA_REGISTRY_ROOT_AUTH'])
1533 # a cross step that takes all other plcs in argument
1534 def cross_sfa_configure(self, other_plcs):
1535 "writes aggregates.xml and registries.xml that point to all other PLCs in the test"
1536 # of course with a single plc, other_plcs is an empty list
1539 agg_fname=self.conffile("agg.xml")
1540 file(agg_fname,"w").write("<aggregates>%s</aggregates>\n" % \
1541 " ".join([ plc.aggregate_xml_line() for plc in other_plcs ]))
1542 utils.header ("(Over)wrote %s"%agg_fname)
1543 reg_fname=self.conffile("reg.xml")
1544 file(reg_fname,"w").write("<registries>%s</registries>\n" % \
1545 " ".join([ plc.registry_xml_line() for plc in other_plcs ]))
1546 utils.header ("(Over)wrote %s"%reg_fname)
1547 return self.test_ssh.copy_abs(agg_fname,'/%s/etc/sfa/aggregates.xml'%self.vm_root_in_host())==0 \
1548 and self.test_ssh.copy_abs(reg_fname,'/%s/etc/sfa/registries.xml'%self.vm_root_in_host())==0
1550 def sfa_import(self):
1551 "use sfaadmin to import from plc"
1552 auth=self.plc_spec['sfa']['settings']['SFA_REGISTRY_ROOT_AUTH']
1553 return self.run_in_guest('sfaadmin reg import_registry')==0
1555 def sfa_start(self):
1557 return self.start_service('sfa')
1560 def sfi_configure(self):
1561 "Create /root/sfi on the plc side for sfi client configuration"
1562 if self.options.dry_run:
1563 utils.header("DRY RUN - skipping step")
1565 sfa_spec=self.plc_spec['sfa']
1566 # cannot use auth_sfa_mapper to pass dir_name
1567 for slice_spec in self.plc_spec['sfa']['auth_sfa_specs']:
1568 test_slice=TestAuthSfa(self,slice_spec)
1569 dir_basename=os.path.basename(test_slice.sfi_path())
1570 dir_name=self.confsubdir("dot-sfi/%s"%dir_basename,clean=True,dry_run=self.options.dry_run)
1571 test_slice.sfi_configure(dir_name)
1572 # push into the remote /root/sfi area
1573 location = test_slice.sfi_path()
1574 remote="%s/%s"%(self.vm_root_in_host(),location)
1575 self.test_ssh.mkdir(remote,abs=True)
1576 # need to strip last level or remote otherwise we get an extra dir level
1577 self.test_ssh.copy_abs(dir_name, os.path.dirname(remote), recursive=True)
1581 def sfi_clean (self):
1582 "clean up /root/sfi on the plc side"
1583 self.run_in_guest("rm -rf /root/sfi")
1586 def sfa_rspec_empty(self):
1587 "expose a static empty rspec (ships with the tests module) in the sfi directory"
1588 filename="empty-rspec.xml"
1590 for slice_spec in self.plc_spec['sfa']['auth_sfa_specs']:
1591 test_slice=TestAuthSfa(self,slice_spec)
1592 in_vm = test_slice.sfi_path()
1593 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1594 if self.test_ssh.copy_abs (filename, remote) !=0: overall=False
1598 def sfa_register_site (self): pass
1600 def sfa_register_pi (self): pass
1602 def sfa_register_user(self): pass
1604 def sfa_update_user(self): pass
1606 def sfa_register_slice(self): pass
1608 def sfa_renew_slice(self): pass
1610 def sfa_get_expires(self): pass
1612 def sfa_discover(self): pass
1614 def sfa_rspec(self): pass
1616 def sfa_allocate(self): pass
1618 def sfa_allocate_empty(self): pass
1620 def sfa_provision(self): pass
1622 def sfa_provision_empty(self): pass
1624 def sfa_check_slice_plc(self): pass
1626 def sfa_check_slice_plc_empty(self): pass
1628 def sfa_update_slice(self): pass
1630 def sfa_remove_user_from_slice(self): pass
1632 def sfa_insert_user_in_slice(self): pass
1634 def sfi_list(self): pass
1636 def sfi_show_site(self): pass
1638 def sfi_show_slice(self): pass
1640 def sfi_show_slice_researchers(self): pass
1642 def ssh_slice_sfa(self): pass
1644 def sfa_delete_user(self): pass
1646 def sfa_delete_slice(self): pass
1650 return self.stop_service ('sfa')
1652 def populate (self):
1653 "creates random entries in the PLCAPI"
1654 # install the stress-test in the plc image
1655 location = "/usr/share/plc_api/plcsh_stress_test.py"
1656 remote="%s/%s"%(self.vm_root_in_host(),location)
1657 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1659 command += " -- --preserve --short-names"
1660 local = (self.run_in_guest(command) == 0);
1661 # second run with --foreign
1662 command += ' --foreign'
1663 remote = (self.run_in_guest(command) == 0);
1664 return ( local and remote)
1666 def gather_logs (self):
1667 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1668 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1669 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1670 # (1.c) get the plc's /root/sfi -> logs/sfi.<plcname>/
1671 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1672 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1673 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1675 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1676 self.gather_var_logs ()
1678 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1679 self.gather_pgsql_logs ()
1681 print "-------------------- TestPlc.gather_logs : PLC's /root/sfi/"
1682 self.gather_root_sfi ()
1684 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1685 for site_spec in self.plc_spec['sites']:
1686 test_site = TestSite (self,site_spec)
1687 for node_spec in site_spec['nodes']:
1688 test_node=TestNode(self,test_site,node_spec)
1689 test_node.gather_qemu_logs()
1691 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1692 self.gather_nodes_var_logs()
1694 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1695 self.gather_slivers_var_logs()
1698 def gather_slivers_var_logs(self):
1699 for test_sliver in self.all_sliver_objs():
1700 remote = test_sliver.tar_var_logs()
1701 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
1702 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
1703 utils.system(command)
1706 def gather_var_logs (self):
1707 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
1708 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1709 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
1710 utils.system(command)
1711 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd"%self.name()
1712 utils.system(command)
1714 def gather_pgsql_logs (self):
1715 utils.system("mkdir -p logs/myplc.pgsql-log.%s"%self.name())
1716 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1717 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -"%self.name()
1718 utils.system(command)
1720 def gather_root_sfi (self):
1721 utils.system("mkdir -p logs/sfi.%s"%self.name())
1722 to_plc = self.actual_command_in_guest("tar -C /root/sfi/ -cf - .")
1723 command = to_plc + "| tar -C logs/sfi.%s -xf -"%self.name()
1724 utils.system(command)
1726 def gather_nodes_var_logs (self):
1727 for site_spec in self.plc_spec['sites']:
1728 test_site = TestSite (self,site_spec)
1729 for node_spec in site_spec['nodes']:
1730 test_node=TestNode(self,test_site,node_spec)
1731 test_ssh = TestSsh (test_node.name(),key="keys/key_admin.rsa")
1732 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1733 command = command + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
1734 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
1735 utils.system(command)
1738 # returns the filename to use for sql dump/restore, using options.dbname if set
1739 def dbfile (self, database):
1740 # uses options.dbname if it is found
1742 name=self.options.dbname
1743 if not isinstance(name,StringTypes):
1749 return "/root/%s-%s.sql"%(database,name)
1751 def plc_db_dump(self):
1752 'dump the planetlab5 DB in /root in the PLC - filename has time'
1753 dump=self.dbfile("planetab5")
1754 self.run_in_guest('pg_dump -U pgsqluser planetlab5 -f '+ dump)
1755 utils.header('Dumped planetlab5 database in %s'%dump)
1758 def plc_db_restore(self):
1759 'restore the planetlab5 DB - looks broken, but run -n might help'
1760 dump=self.dbfile("planetab5")
1761 ##stop httpd service
1762 self.run_in_guest('service httpd stop')
1763 # xxx - need another wrapper
1764 self.run_in_guest_piped('echo drop database planetlab5','psql --user=pgsqluser template1')
1765 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab5')
1766 self.run_in_guest('psql -U pgsqluser planetlab5 -f '+dump)
1767 ##starting httpd service
1768 self.run_in_guest('service httpd start')
1770 utils.header('Database restored from ' + dump)
1773 def create_ignore_steps ():
1774 for step in TestPlc.default_steps + TestPlc.other_steps:
1775 # default step can have a plc qualifier
1776 if '@' in step: (step,qualifier)=step.split('@')
1777 # or be defined as forced or ignored by default
1778 for keyword in ['_ignore','_force']:
1779 if step.endswith (keyword): step=step.replace(keyword,'')
1780 if step == SEP or step == SEPSFA : continue
1781 method=getattr(TestPlc,step)
1783 wrapped=ignore_result(method)
1784 # wrapped.__doc__ = method.__doc__ + " (run in ignore-result mode)"
1785 setattr(TestPlc, name, wrapped)
1788 # def ssh_slice_again_ignore (self): pass
1790 # def check_initscripts_ignore (self): pass
1792 def standby_1_through_20(self):
1793 """convenience function to wait for a specified number of minutes"""
1796 def standby_1(): pass
1798 def standby_2(): pass
1800 def standby_3(): pass
1802 def standby_4(): pass
1804 def standby_5(): pass
1806 def standby_6(): pass
1808 def standby_7(): pass
1810 def standby_8(): pass
1812 def standby_9(): pass
1814 def standby_10(): pass
1816 def standby_11(): pass
1818 def standby_12(): pass
1820 def standby_13(): pass
1822 def standby_14(): pass
1824 def standby_15(): pass
1826 def standby_16(): pass
1828 def standby_17(): pass
1830 def standby_18(): pass
1832 def standby_19(): pass
1834 def standby_20(): pass
1836 # convenience for debugging the test logic
1837 def yes (self): return True
1838 def no (self): return False