1 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
2 # Copyright (C) 2010 INRIA
9 from datetime import datetime, timedelta
10 from types import StringTypes
13 from TestSite import TestSite
14 from TestNode import TestNode
15 from TestUser import TestUser
16 from TestKey import TestKey
17 from TestSlice import TestSlice
18 from TestSliver import TestSliver
19 from TestBoxQemu import TestBoxQemu
20 from TestSsh import TestSsh
21 from TestApiserver import TestApiserver
22 from TestAuthSfa import TestAuthSfa
23 from PlcapiUrlScanner import PlcapiUrlScanner
24 from Completer import Completer, CompleterTask
26 # step methods must take (self) and return a boolean (options is a member of the class)
28 def standby(minutes,dry_run):
29 utils.header('Entering StandBy for %d mn'%minutes)
33 time.sleep(60*minutes)
36 def standby_generic (func):
38 minutes=int(func.__name__.split("_")[1])
39 return standby(minutes,self.options.dry_run)
42 def node_mapper (method):
43 def actual(self,*args, **kwds):
45 node_method = TestNode.__dict__[method.__name__]
46 for test_node in self.all_nodes():
47 if not node_method(test_node, *args, **kwds): overall=False
49 # restore the doc text
50 actual.__doc__=TestNode.__dict__[method.__name__].__doc__
53 def slice_mapper (method):
56 slice_method = TestSlice.__dict__[method.__name__]
57 for slice_spec in self.plc_spec['slices']:
58 site_spec = self.locate_site (slice_spec['sitename'])
59 test_site = TestSite(self,site_spec)
60 test_slice=TestSlice(self,test_site,slice_spec)
61 if not slice_method(test_slice,self.options): overall=False
63 # restore the doc text
64 actual.__doc__=TestSlice.__dict__[method.__name__].__doc__
67 def auth_sfa_mapper (method):
70 auth_method = TestAuthSfa.__dict__[method.__name__]
71 for auth_spec in self.plc_spec['sfa']['auth_sfa_specs']:
72 test_auth=TestAuthSfa(self,auth_spec)
73 if not auth_method(test_auth,self.options): overall=False
75 # restore the doc text
76 actual.__doc__=TestAuthSfa.__dict__[method.__name__].__doc__
86 'vs_delete','timestamp_vs','vs_create', SEP,
87 'plc_install', 'mod_python', 'plc_configure', 'plc_start', SEP,
88 'keys_fetch', 'keys_store', 'keys_clear_known_hosts', SEP,
89 'plcapi_urls','speed_up_slices', SEP,
90 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', 'leases', SEP,
91 # slices created under plcsh interactively seem to be fine but these ones don't have the tags
92 # keep this our of the way for now
93 # 'check_vsys_defaults', SEP,
94 'nodestate_reinstall', 'qemu_local_init','bootcd', 'qemu_local_config', SEP,
95 'qemu_kill_mine','qemu_clean_mine', 'qemu_export', 'qemu_start', 'timestamp_qemu', SEP,
96 'sfa_install_all', 'sfa_configure', 'cross_sfa_configure', 'sfa_start', 'sfa_import', SEPSFA,
97 'sfi_configure@1', 'sfa_add_site@1','sfa_add_pi@1', SEPSFA,
98 'sfa_add_user@1', 'sfa_update_user@1', 'sfa_add_slice@1', 'sfa_renew_slice@1', SEPSFA,
99 'sfa_discover@1', 'sfa_create_slice@1', 'sfa_check_slice_plc@1', 'sfa_update_slice@1', SEPSFA,
100 'sfi_list@1', 'sfi_show@1', 'sfa_utest@1', SEPSFA,
101 # we used to run plcsh_stress_test, and then ssh_node_debug and ssh_node_boot
102 # but as the stress test might take a while, we sometimes missed the debug mode..
103 'ssh_node_debug@1', 'plcsh_stress_test@1', SEP,
104 'ssh_node_boot@1', 'node_bmlogs@1', 'ssh_slice', 'ssh_slice_basics', 'check_initscripts', SEP,
105 'ssh_slice_sfa@1', 'sfa_delete_slice@1', 'sfa_delete_user@1', SEPSFA,
106 'cross_check_tcp@1', 'check_system_slice', SEP,
107 'empty_slices', 'ssh_slice_off', 'fill_slices', SEP,
108 'force_gather_logs', SEP,
111 'export', 'show_boxes', SEP,
112 'check_hooks', 'plc_stop', 'vs_start', 'vs_stop', SEP,
113 'delete_initscripts', 'delete_nodegroups','delete_all_sites', SEP,
114 'delete_sites', 'delete_nodes', 'delete_slices', 'keys_clean', SEP,
115 'delete_leases', 'list_leases', SEP,
117 'nodestate_show','nodestate_safeboot','nodestate_boot', SEP,
118 'qemu_list_all', 'qemu_list_mine', 'qemu_kill_all', SEP,
119 'sfa_install_core', 'sfa_install_sfatables', 'sfa_install_plc', 'sfa_install_client', SEPSFA,
120 'sfa_plcclean', 'sfa_dbclean', 'sfa_stop','sfa_uninstall', 'sfi_clean', SEPSFA,
121 'plc_db_dump' , 'plc_db_restore', SEP,
122 'check_netflow','check_drl', SEP,
123 'debug_nodemanager', SEP,
124 'standby_1_through_20',SEP,
128 def printable_steps (list):
129 single_line=" ".join(list)+" "
130 return single_line.replace(" "+SEP+" "," \\\n").replace(" "+SEPSFA+" "," \\\n")
132 def valid_step (step):
133 return step != SEP and step != SEPSFA
135 # turn off the sfa-related steps when build has skipped SFA
136 # this was originally for centos5 but is still valid
137 # for up to f12 as recent SFAs with sqlalchemy won't build before f14
139 def check_whether_build_has_sfa (rpms_url):
140 utils.header ("Checking if build provides SFA package...")
141 # warning, we're now building 'sface' so let's be a bit more picky
142 retcod=os.system ("curl --silent %s/ | grep -q sfa-"%rpms_url)
143 # full builds are expected to return with 0 here
145 utils.header("build does provide SFA")
147 # move all steps containing 'sfa' from default_steps to other_steps
148 utils.header("SFA package not found - removing steps with sfa or sfi")
149 sfa_steps= [ step for step in TestPlc.default_steps if step.find('sfa')>=0 or step.find("sfi")>=0 ]
150 TestPlc.other_steps += sfa_steps
151 for step in sfa_steps: TestPlc.default_steps.remove(step)
153 def __init__ (self,plc_spec,options):
154 self.plc_spec=plc_spec
156 self.test_ssh=TestSsh(self.plc_spec['host_box'],self.options.buildname)
157 self.vserverip=plc_spec['vserverip']
158 self.vservername=plc_spec['vservername']
159 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
160 self.apiserver=TestApiserver(self.url,options.dry_run)
161 (self.ssh_node_boot_timeout,self.ssh_node_boot_silent)=plc_spec['ssh_node_boot_timers']
162 (self.ssh_node_debug_timeout,self.ssh_node_debug_silent)=plc_spec['ssh_node_debug_timers']
164 def has_addresses_api (self):
165 return self.apiserver.has_method('AddIpAddress')
168 name=self.plc_spec['name']
169 return "%s.%s"%(name,self.vservername)
172 return self.plc_spec['host_box']
175 return self.test_ssh.is_local()
177 # define the API methods on this object through xmlrpc
178 # would help, but not strictly necessary
182 def actual_command_in_guest (self,command):
183 return self.test_ssh.actual_command(self.host_to_guest(command),dry_run=self.options.dry_run)
185 def start_guest (self):
186 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host(),dry_run=self.options.dry_run))
188 def stop_guest (self):
189 return utils.system(self.test_ssh.actual_command(self.stop_guest_in_host(),dry_run=self.options.dry_run))
191 def run_in_guest (self,command):
192 return utils.system(self.actual_command_in_guest(command))
194 def run_in_host (self,command):
195 return self.test_ssh.run_in_buildname(command, dry_run=self.options.dry_run)
197 #command gets run in the plc's vm
198 def host_to_guest(self,command):
199 if self.options.plcs_use_lxc:
200 return "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s %s"%(self.vserverip,command)
202 return "vserver %s exec %s"%(self.vservername,command)
204 def vm_root_in_host(self):
205 if self.options.plcs_use_lxc:
206 return "/vservers/%s/rootfs/"%(self.vservername)
208 return "/vservers/%s"%(self.vservername)
210 def vm_timestamp_path (self):
211 if self.options.plcs_use_lxc:
212 return "/vservers/%s/%s.timestamp"%(self.vservername,self.vservername)
214 return "/vservers/%s.timestamp"%(self.vservername)
216 #start/stop the vserver
217 def start_guest_in_host(self):
218 if self.options.plcs_use_lxc:
219 return "virsh -c lxc:// start %s"%(self.vservername)
221 return "vserver %s start"%(self.vservername)
223 def stop_guest_in_host(self):
224 if self.options.plcs_use_lxc:
225 return "virsh -c lxc:// destroy %s"%(self.vservername)
227 return "vserver %s stop"%(self.vservername)
230 def run_in_guest_piped (self,local,remote):
231 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
233 def yum_check_installed (self, rpms):
234 if isinstance (rpms, list):
236 return self.run_in_guest("rpm -q %s"%rpms)==0
238 # does a yum install in the vs, ignore yum retcod, check with rpm
239 def yum_install (self, rpms):
240 if isinstance (rpms, list):
242 self.run_in_guest("yum -y install %s"%rpms)
243 # yum-complete-transaction comes with yum-utils, that is in vtest.pkgs
244 self.run_in_guest("yum-complete-transaction -y")
245 return self.yum_check_installed (rpms)
247 def auth_root (self):
248 return {'Username':self.plc_spec['PLC_ROOT_USER'],
249 'AuthMethod':'password',
250 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
251 'Role' : self.plc_spec['role']
253 def locate_site (self,sitename):
254 for site in self.plc_spec['sites']:
255 if site['site_fields']['name'] == sitename:
257 if site['site_fields']['login_base'] == sitename:
259 raise Exception,"Cannot locate site %s"%sitename
261 def locate_node (self,nodename):
262 for site in self.plc_spec['sites']:
263 for node in site['nodes']:
264 if node['name'] == nodename:
266 raise Exception,"Cannot locate node %s"%nodename
268 def locate_hostname (self,hostname):
269 for site in self.plc_spec['sites']:
270 for node in site['nodes']:
271 if node['node_fields']['hostname'] == hostname:
273 raise Exception,"Cannot locate hostname %s"%hostname
275 def locate_key (self,key_name):
276 for key in self.plc_spec['keys']:
277 if key['key_name'] == key_name:
279 raise Exception,"Cannot locate key %s"%key_name
281 def locate_private_key_from_key_names (self, key_names):
282 # locate the first avail. key
284 for key_name in key_names:
285 key_spec=self.locate_key(key_name)
286 test_key=TestKey(self,key_spec)
287 publickey=test_key.publicpath()
288 privatekey=test_key.privatepath()
289 if os.path.isfile(publickey) and os.path.isfile(privatekey):
291 if found: return privatekey
294 def locate_slice (self, slicename):
295 for slice in self.plc_spec['slices']:
296 if slice['slice_fields']['name'] == slicename:
298 raise Exception,"Cannot locate slice %s"%slicename
300 def all_sliver_objs (self):
302 for slice_spec in self.plc_spec['slices']:
303 slicename = slice_spec['slice_fields']['name']
304 for nodename in slice_spec['nodenames']:
305 result.append(self.locate_sliver_obj (nodename,slicename))
308 def locate_sliver_obj (self,nodename,slicename):
309 (site,node) = self.locate_node(nodename)
310 slice = self.locate_slice (slicename)
312 test_site = TestSite (self, site)
313 test_node = TestNode (self, test_site,node)
314 # xxx the slice site is assumed to be the node site - mhh - probably harmless
315 test_slice = TestSlice (self, test_site, slice)
316 return TestSliver (self, test_node, test_slice)
318 def locate_first_node(self):
319 nodename=self.plc_spec['slices'][0]['nodenames'][0]
320 (site,node) = self.locate_node(nodename)
321 test_site = TestSite (self, site)
322 test_node = TestNode (self, test_site,node)
325 def locate_first_sliver (self):
326 slice_spec=self.plc_spec['slices'][0]
327 slicename=slice_spec['slice_fields']['name']
328 nodename=slice_spec['nodenames'][0]
329 return self.locate_sliver_obj(nodename,slicename)
331 # all different hostboxes used in this plc
332 def get_BoxNodes(self):
333 # maps on sites and nodes, return [ (host_box,test_node) ]
335 for site_spec in self.plc_spec['sites']:
336 test_site = TestSite (self,site_spec)
337 for node_spec in site_spec['nodes']:
338 test_node = TestNode (self, test_site, node_spec)
339 if not test_node.is_real():
340 tuples.append( (test_node.host_box(),test_node) )
341 # transform into a dict { 'host_box' -> [ test_node .. ] }
343 for (box,node) in tuples:
344 if not result.has_key(box):
347 result[box].append(node)
350 # a step for checking this stuff
351 def show_boxes (self):
352 'print summary of nodes location'
353 for (box,nodes) in self.get_BoxNodes().iteritems():
354 print box,":"," + ".join( [ node.name() for node in nodes ] )
357 # make this a valid step
358 def qemu_kill_all(self):
359 'kill all qemu instances on the qemu boxes involved by this setup'
360 # this is the brute force version, kill all qemus on that host box
361 for (box,nodes) in self.get_BoxNodes().iteritems():
362 # pass the first nodename, as we don't push template-qemu on testboxes
363 nodedir=nodes[0].nodedir()
364 TestBoxQemu(box,self.options.buildname).qemu_kill_all(nodedir)
367 # make this a valid step
368 def qemu_list_all(self):
369 'list all qemu instances on the qemu boxes involved by this setup'
370 for (box,nodes) in self.get_BoxNodes().iteritems():
371 # this is the brute force version, kill all qemus on that host box
372 TestBoxQemu(box,self.options.buildname).qemu_list_all()
375 # kill only the qemus related to this test
376 def qemu_list_mine(self):
377 'list qemu instances for our nodes'
378 for (box,nodes) in self.get_BoxNodes().iteritems():
379 # the fine-grain version
384 # kill only the qemus related to this test
385 def qemu_clean_mine(self):
386 'cleanup (rm -rf) qemu instances for our nodes'
387 for (box,nodes) in self.get_BoxNodes().iteritems():
388 # the fine-grain version
393 # kill only the right qemus
394 def qemu_kill_mine(self):
395 'kill the qemu instances for our nodes'
396 for (box,nodes) in self.get_BoxNodes().iteritems():
397 # the fine-grain version
402 #################### display config
404 "show test configuration after localization"
409 # uggly hack to make sure 'run export' only reports about the 1st plc
410 # to avoid confusion - also we use 'inri_slice1' in various aliases..
413 "print cut'n paste-able stuff to export env variables to your shell"
414 # guess local domain from hostname
415 if TestPlc.exported_id>1:
416 print "export GUESTHOSTNAME%d=%s"%(TestPlc.exported_id,self.plc_spec['vservername'])
418 TestPlc.exported_id+=1
419 domain=socket.gethostname().split('.',1)[1]
420 fqdn="%s.%s"%(self.plc_spec['host_box'],domain)
421 print "export BUILD=%s"%self.options.buildname
422 if self.options.plcs_use_lxc:
423 print "export PLCHOSTLXC=%s"%fqdn
425 print "export PLCHOSTVS=%s"%fqdn
426 print "export GUESTNAME=%s"%self.plc_spec['vservername']
427 vplcname=self.plc_spec['vservername'].split('-')[-1]
428 print "export GUESTHOSTNAME=%s.%s"%(vplcname,domain)
429 # find hostname of first node
430 (hostname,qemubox) = self.all_node_infos()[0]
431 print "export KVMHOST=%s.%s"%(qemubox,domain)
432 print "export NODE=%s"%(hostname)
436 always_display_keys=['PLC_WWW_HOST','nodes','sites',]
437 def show_pass (self,passno):
438 for (key,val) in self.plc_spec.iteritems():
439 if not self.options.verbose and key not in TestPlc.always_display_keys: continue
443 self.display_site_spec(site)
444 for node in site['nodes']:
445 self.display_node_spec(node)
446 elif key=='initscripts':
447 for initscript in val:
448 self.display_initscript_spec (initscript)
451 self.display_slice_spec (slice)
454 self.display_key_spec (key)
456 if key not in ['sites','initscripts','slices','keys', 'sfa']:
457 print '+ ',key,':',val
459 def display_site_spec (self,site):
460 print '+ ======== site',site['site_fields']['name']
461 for (k,v) in site.iteritems():
462 if not self.options.verbose and k not in TestPlc.always_display_keys: continue
465 print '+ ','nodes : ',
467 print node['node_fields']['hostname'],'',
473 print user['name'],'',
475 elif k == 'site_fields':
476 print '+ login_base',':',v['login_base']
477 elif k == 'address_fields':
483 def display_initscript_spec (self,initscript):
484 print '+ ======== initscript',initscript['initscript_fields']['name']
486 def display_key_spec (self,key):
487 print '+ ======== key',key['key_name']
489 def display_slice_spec (self,slice):
490 print '+ ======== slice',slice['slice_fields']['name']
491 for (k,v) in slice.iteritems():
504 elif k=='slice_fields':
505 print '+ fields',':',
506 print 'max_nodes=',v['max_nodes'],
511 def display_node_spec (self,node):
512 print "+ node=%s host_box=%s"%(node['name'],node['host_box']),
513 print "hostname=",node['node_fields']['hostname'],
514 print "ip=",node['interface_fields']['ip']
515 if self.options.verbose:
516 utils.pprint("node details",node,depth=3)
518 # another entry point for just showing the boxes involved
519 def display_mapping (self):
520 TestPlc.display_mapping_plc(self.plc_spec)
524 def display_mapping_plc (plc_spec):
525 print '+ MyPLC',plc_spec['name']
526 # WARNING this would not be right for lxc-based PLC's - should be harmless though
527 print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['host_box'],plc_spec['vservername'])
528 print '+\tIP = %s/%s'%(plc_spec['PLC_API_HOST'],plc_spec['vserverip'])
529 for site_spec in plc_spec['sites']:
530 for node_spec in site_spec['nodes']:
531 TestPlc.display_mapping_node(node_spec)
534 def display_mapping_node (node_spec):
535 print '+ NODE %s'%(node_spec['name'])
536 print '+\tqemu box %s'%node_spec['host_box']
537 print '+\thostname=%s'%node_spec['node_fields']['hostname']
539 # write a timestamp in /vservers/<>.timestamp
540 # cannot be inside the vserver, that causes vserver .. build to cough
541 def timestamp_vs (self):
542 "Create a timestamp to remember creation date for this plc"
544 # TODO-lxc check this one
545 # a first approx. is to store the timestamp close to the VM root like vs does
546 stamp_path=self.vm_timestamp_path ()
547 stamp_dir = os.path.dirname (stamp_path)
548 utils.system(self.test_ssh.actual_command("mkdir -p %s"%stamp_dir))
549 return utils.system(self.test_ssh.actual_command("echo %d > %s"%(now,stamp_path)))==0
551 # this is called inconditionnally at the beginning of the test sequence
552 # just in case this is a rerun, so if the vm is not running it's fine
554 "vserver delete the test myplc"
555 stamp_path=self.vm_timestamp_path()
556 self.run_in_host("rm -f %s"%stamp_path)
557 if self.options.plcs_use_lxc:
558 self.run_in_host("virsh -c lxc:// destroy %s"%self.vservername)
559 self.run_in_host("virsh -c lxc:// undefine %s"%self.vservername)
560 self.run_in_host("rm -fr /vservers/%s"%self.vservername)
563 self.run_in_host("vserver --silent %s delete"%self.vservername)
567 # historically the build was being fetched by the tests
568 # now the build pushes itself as a subdir of the tests workdir
569 # so that the tests do not have to worry about extracting the build (svn, git, or whatever)
570 def vs_create (self):
571 "vserver creation (no install done)"
572 # push the local build/ dir to the testplc box
574 # a full path for the local calls
575 build_dir=os.path.dirname(sys.argv[0])
576 # sometimes this is empty - set to "." in such a case
577 if not build_dir: build_dir="."
578 build_dir += "/build"
580 # use a standard name - will be relative to remote buildname
582 # remove for safety; do *not* mkdir first, otherwise we end up with build/build/
583 self.test_ssh.rmdir(build_dir)
584 self.test_ssh.copy(build_dir,recursive=True)
585 # the repo url is taken from arch-rpms-url
586 # with the last step (i386) removed
587 repo_url = self.options.arch_rpms_url
588 for level in [ 'arch' ]:
589 repo_url = os.path.dirname(repo_url)
590 # pass the vbuild-nightly options to vtest-init-vserver
592 test_env_options += " -p %s"%self.options.personality
593 test_env_options += " -d %s"%self.options.pldistro
594 test_env_options += " -f %s"%self.options.fcdistro
595 if self.options.plcs_use_lxc:
596 script="vtest-init-lxc.sh"
598 script="vtest-init-vserver.sh"
599 vserver_name = self.vservername
600 vserver_options="--netdev eth0 --interface %s"%self.vserverip
602 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
603 vserver_options += " --hostname %s"%vserver_hostname
605 print "Cannot reverse lookup %s"%self.vserverip
606 print "This is considered fatal, as this might pollute the test results"
608 create_vserver="%(build_dir)s/%(script)s %(test_env_options)s %(vserver_name)s %(repo_url)s -- %(vserver_options)s"%locals()
609 return self.run_in_host(create_vserver) == 0
612 def plc_install(self):
613 "yum install myplc, noderepo, and the plain bootstrapfs"
615 # workaround for getting pgsql8.2 on centos5
616 if self.options.fcdistro == "centos5":
617 self.run_in_guest("rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-3.noarch.rpm")
620 if self.options.personality == "linux32":
622 elif self.options.personality == "linux64":
625 raise Exception, "Unsupported personality %r"%self.options.personality
626 nodefamily="%s-%s-%s"%(self.options.pldistro,self.options.fcdistro,arch)
629 pkgs_list.append ("slicerepo-%s"%nodefamily)
630 pkgs_list.append ("myplc")
631 pkgs_list.append ("noderepo-%s"%nodefamily)
632 pkgs_list.append ("nodeimage-%s-plain"%nodefamily)
633 pkgs_string=" ".join(pkgs_list)
634 return self.yum_install (pkgs_list)
637 def mod_python(self):
638 """yum install mod_python, useful on f18 and above so as to avoid broken wsgi"""
639 return self.yum_install ( [ 'mod_python' ] )
642 def plc_configure(self):
644 tmpname='%s.plc-config-tty'%(self.name())
645 fileconf=open(tmpname,'w')
646 for var in [ 'PLC_NAME',
651 'PLC_MAIL_SUPPORT_ADDRESS',
654 # Above line was added for integrating SFA Testing
660 'PLC_RESERVATION_GRANULARITY',
662 'PLC_OMF_XMPP_SERVER',
665 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
666 fileconf.write('w\n')
667 fileconf.write('q\n')
669 utils.system('cat %s'%tmpname)
670 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
671 utils.system('rm %s'%tmpname)
676 self.run_in_guest('service plc start')
681 self.run_in_guest('service plc stop')
685 "start the PLC vserver"
690 "stop the PLC vserver"
694 # stores the keys from the config for further use
695 def keys_store(self):
696 "stores test users ssh keys in keys/"
697 for key_spec in self.plc_spec['keys']:
698 TestKey(self,key_spec).store_key()
701 def keys_clean(self):
702 "removes keys cached in keys/"
703 utils.system("rm -rf ./keys")
706 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
707 # for later direct access to the nodes
708 def keys_fetch(self):
709 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
711 if not os.path.isdir(dir):
713 vservername=self.vservername
714 vm_root=self.vm_root_in_host()
716 prefix = 'debug_ssh_key'
717 for ext in [ 'pub', 'rsa' ] :
718 src="%(vm_root)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
719 dst="keys/%(vservername)s-debug.%(ext)s"%locals()
720 if self.test_ssh.fetch(src,dst) != 0: overall=False
724 "create sites with PLCAPI"
725 return self.do_sites()
727 def delete_sites (self):
728 "delete sites with PLCAPI"
729 return self.do_sites(action="delete")
731 def do_sites (self,action="add"):
732 for site_spec in self.plc_spec['sites']:
733 test_site = TestSite (self,site_spec)
734 if (action != "add"):
735 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
736 test_site.delete_site()
737 # deleted with the site
738 #test_site.delete_users()
741 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
742 test_site.create_site()
743 test_site.create_users()
746 def delete_all_sites (self):
747 "Delete all sites in PLC, and related objects"
748 print 'auth_root',self.auth_root()
749 sites = self.apiserver.GetSites(self.auth_root(), {}, ['site_id','login_base'])
751 # keep automatic site - otherwise we shoot in our own foot, root_auth is not valid anymore
752 if site['login_base']==self.plc_spec['PLC_SLICE_PREFIX']: continue
753 site_id=site['site_id']
754 print 'Deleting site_id',site_id
755 self.apiserver.DeleteSite(self.auth_root(),site_id)
759 "create nodes with PLCAPI"
760 return self.do_nodes()
761 def delete_nodes (self):
762 "delete nodes with PLCAPI"
763 return self.do_nodes(action="delete")
765 def do_nodes (self,action="add"):
766 for site_spec in self.plc_spec['sites']:
767 test_site = TestSite (self,site_spec)
769 utils.header("Deleting nodes in site %s"%test_site.name())
770 for node_spec in site_spec['nodes']:
771 test_node=TestNode(self,test_site,node_spec)
772 utils.header("Deleting %s"%test_node.name())
773 test_node.delete_node()
775 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
776 for node_spec in site_spec['nodes']:
777 utils.pprint('Creating node %s'%node_spec,node_spec)
778 test_node = TestNode (self,test_site,node_spec)
779 test_node.create_node ()
782 def nodegroups (self):
783 "create nodegroups with PLCAPI"
784 return self.do_nodegroups("add")
785 def delete_nodegroups (self):
786 "delete nodegroups with PLCAPI"
787 return self.do_nodegroups("delete")
791 def translate_timestamp (start,grain,timestamp):
792 if timestamp < TestPlc.YEAR: return start+timestamp*grain
793 else: return timestamp
796 def timestamp_printable (timestamp):
797 return time.strftime('%m-%d %H:%M:%S UTC',time.gmtime(timestamp))
800 "create leases (on reservable nodes only, use e.g. run -c default -c resa)"
802 grain=self.apiserver.GetLeaseGranularity(self.auth_root())
803 print 'API answered grain=',grain
804 start=(now/grain)*grain
806 # find out all nodes that are reservable
807 nodes=self.all_reservable_nodenames()
809 utils.header ("No reservable node found - proceeding without leases")
812 # attach them to the leases as specified in plc_specs
813 # this is where the 'leases' field gets interpreted as relative of absolute
814 for lease_spec in self.plc_spec['leases']:
815 # skip the ones that come with a null slice id
816 if not lease_spec['slice']: continue
817 lease_spec['t_from']=TestPlc.translate_timestamp(start,grain,lease_spec['t_from'])
818 lease_spec['t_until']=TestPlc.translate_timestamp(start,grain,lease_spec['t_until'])
819 lease_addition=self.apiserver.AddLeases(self.auth_root(),nodes,
820 lease_spec['slice'],lease_spec['t_from'],lease_spec['t_until'])
821 if lease_addition['errors']:
822 utils.header("Cannot create leases, %s"%lease_addition['errors'])
825 utils.header('Leases on nodes %r for %s from %d (%s) until %d (%s)'%\
826 (nodes,lease_spec['slice'],
827 lease_spec['t_from'],TestPlc.timestamp_printable(lease_spec['t_from']),
828 lease_spec['t_until'],TestPlc.timestamp_printable(lease_spec['t_until'])))
832 def delete_leases (self):
833 "remove all leases in the myplc side"
834 lease_ids= [ l['lease_id'] for l in self.apiserver.GetLeases(self.auth_root())]
835 utils.header("Cleaning leases %r"%lease_ids)
836 self.apiserver.DeleteLeases(self.auth_root(),lease_ids)
839 def list_leases (self):
840 "list all leases known to the myplc"
841 leases = self.apiserver.GetLeases(self.auth_root())
844 current=l['t_until']>=now
845 if self.options.verbose or current:
846 utils.header("%s %s from %s until %s"%(l['hostname'],l['name'],
847 TestPlc.timestamp_printable(l['t_from']),
848 TestPlc.timestamp_printable(l['t_until'])))
851 # create nodegroups if needed, and populate
852 def do_nodegroups (self, action="add"):
853 # 1st pass to scan contents
855 for site_spec in self.plc_spec['sites']:
856 test_site = TestSite (self,site_spec)
857 for node_spec in site_spec['nodes']:
858 test_node=TestNode (self,test_site,node_spec)
859 if node_spec.has_key('nodegroups'):
860 nodegroupnames=node_spec['nodegroups']
861 if isinstance(nodegroupnames,StringTypes):
862 nodegroupnames = [ nodegroupnames ]
863 for nodegroupname in nodegroupnames:
864 if not groups_dict.has_key(nodegroupname):
865 groups_dict[nodegroupname]=[]
866 groups_dict[nodegroupname].append(test_node.name())
867 auth=self.auth_root()
869 for (nodegroupname,group_nodes) in groups_dict.iteritems():
871 print 'nodegroups:','dealing with nodegroup',nodegroupname,'on nodes',group_nodes
872 # first, check if the nodetagtype is here
873 tag_types = self.apiserver.GetTagTypes(auth,{'tagname':nodegroupname})
875 tag_type_id = tag_types[0]['tag_type_id']
877 tag_type_id = self.apiserver.AddTagType(auth,
878 {'tagname':nodegroupname,
879 'description': 'for nodegroup %s'%nodegroupname,
881 print 'located tag (type)',nodegroupname,'as',tag_type_id
883 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
885 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
886 print 'created nodegroup',nodegroupname,'from tagname',nodegroupname,'and value','yes'
887 # set node tag on all nodes, value='yes'
888 for nodename in group_nodes:
890 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
892 traceback.print_exc()
893 print 'node',nodename,'seems to already have tag',nodegroupname
896 expect_yes = self.apiserver.GetNodeTags(auth,
897 {'hostname':nodename,
898 'tagname':nodegroupname},
899 ['value'])[0]['value']
900 if expect_yes != "yes":
901 print 'Mismatch node tag on node',nodename,'got',expect_yes
904 if not self.options.dry_run:
905 print 'Cannot find tag',nodegroupname,'on node',nodename
909 print 'cleaning nodegroup',nodegroupname
910 self.apiserver.DeleteNodeGroup(auth,nodegroupname)
912 traceback.print_exc()
916 # a list of TestNode objs
917 def all_nodes (self):
919 for site_spec in self.plc_spec['sites']:
920 test_site = TestSite (self,site_spec)
921 for node_spec in site_spec['nodes']:
922 nodes.append(TestNode (self,test_site,node_spec))
925 # return a list of tuples (nodename,qemuname)
926 def all_node_infos (self) :
928 for site_spec in self.plc_spec['sites']:
929 node_infos += [ (node_spec['node_fields']['hostname'],node_spec['host_box']) \
930 for node_spec in site_spec['nodes'] ]
933 def all_nodenames (self): return [ x[0] for x in self.all_node_infos() ]
934 def all_reservable_nodenames (self):
936 for site_spec in self.plc_spec['sites']:
937 for node_spec in site_spec['nodes']:
938 node_fields=node_spec['node_fields']
939 if 'node_type' in node_fields and node_fields['node_type']=='reservable':
940 res.append(node_fields['hostname'])
943 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
944 def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period_seconds=15):
945 if self.options.dry_run:
949 class CompleterTaskBootState (CompleterTask):
950 def __init__ (self, test_plc,hostname):
951 self.test_plc=test_plc
952 self.hostname=hostname
953 self.last_boot_state='undef'
954 def actual_run (self):
956 node = self.test_plc.apiserver.GetNodes(self.test_plc.auth_root(), [ self.hostname ],
958 self.last_boot_state = node['boot_state']
959 return self.last_boot_state == target_boot_state
963 return "CompleterTaskBootState with node %s"%self.hostname
964 def failure_message (self):
965 return "node %s in state %s - expected %s"%(self.hostname,self.last_boot_state,target_boot_state)
967 timeout = timedelta(minutes=timeout_minutes)
968 graceout = timedelta(minutes=silent_minutes)
969 period = timedelta(seconds=period_seconds)
970 # the nodes that haven't checked yet - start with a full list and shrink over time
971 utils.header("checking nodes boot state (expected %s)"%target_boot_state)
972 tasks = [ CompleterTaskBootState (self,hostname) \
973 for (hostname,_) in self.all_node_infos() ]
974 return Completer (tasks).run (timeout, graceout, period)
976 def nodes_booted(self):
977 return self.nodes_check_boot_state('boot',timeout_minutes=30,silent_minutes=28)
979 def check_nodes_ssh(self,debug,timeout_minutes,silent_minutes,period_seconds=15):
980 class CompleterTaskNodeSsh (CompleterTask):
981 def __init__ (self, hostname, qemuname, boot_state, local_key):
982 self.hostname=hostname
983 self.qemuname=qemuname
984 self.boot_state=boot_state
985 self.local_key=local_key
986 def run (self, silent):
987 command = TestSsh (self.hostname,key=self.local_key).actual_command("hostname;uname -a")
988 return utils.system (command, silent=silent)==0
989 def failure_message (self):
990 return "Cannot reach %s @ %s in %s mode"%(self.hostname, self.qemuname, self.boot_state)
993 timeout = timedelta(minutes=timeout_minutes)
994 graceout = timedelta(minutes=silent_minutes)
995 period = timedelta(seconds=period_seconds)
996 vservername=self.vservername
999 local_key = "keys/%(vservername)s-debug.rsa"%locals()
1002 local_key = "keys/key_admin.rsa"
1003 utils.header("checking ssh access to nodes (expected in %s mode)"%message)
1004 node_infos = self.all_node_infos()
1005 tasks = [ CompleterTaskNodeSsh (nodename, qemuname, message, local_key) \
1006 for (nodename,qemuname) in node_infos ]
1007 return Completer (tasks).run (timeout, graceout, period)
1009 def ssh_node_debug(self):
1010 "Tries to ssh into nodes in debug mode with the debug ssh key"
1011 return self.check_nodes_ssh(debug=True,
1012 timeout_minutes=self.ssh_node_debug_timeout,
1013 silent_minutes=self.ssh_node_debug_silent)
1015 def ssh_node_boot(self):
1016 "Tries to ssh into nodes in production mode with the root ssh key"
1017 return self.check_nodes_ssh(debug=False,
1018 timeout_minutes=self.ssh_node_boot_timeout,
1019 silent_minutes=self.ssh_node_boot_silent)
1021 def node_bmlogs(self):
1022 "Checks that there's a non-empty dir. /var/log/bm/raw"
1023 return utils.system(self.actual_command_in_guest("ls /var/log/bm/raw"))==0
1026 def qemu_local_init (self): pass
1028 def bootcd (self): pass
1030 def qemu_local_config (self): pass
1032 def nodestate_reinstall (self): pass
1034 def nodestate_safeboot (self): pass
1036 def nodestate_boot (self): pass
1038 def nodestate_show (self): pass
1040 def qemu_export (self): pass
1042 ### check hooks : invoke scripts from hooks/{node,slice}
1043 def check_hooks_node (self):
1044 return self.locate_first_node().check_hooks()
1045 def check_hooks_sliver (self) :
1046 return self.locate_first_sliver().check_hooks()
1048 def check_hooks (self):
1049 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
1050 return self.check_hooks_node() and self.check_hooks_sliver()
1053 def do_check_initscripts(self):
1054 class CompleterTaskInitscript (CompleterTask):
1055 def __init__ (self, test_sliver, stamp):
1056 self.test_sliver=test_sliver
1058 def actual_run (self):
1059 return self.test_sliver.check_initscript_stamp (self.stamp)
1061 return "initscript checker for %s"%self.test_sliver.name()
1062 def failure_message (self):
1063 return "initscript stamp %s not found in sliver %s"%(self.stamp,self.test_sliver.name())
1066 for slice_spec in self.plc_spec['slices']:
1067 if not slice_spec.has_key('initscriptstamp'):
1069 stamp=slice_spec['initscriptstamp']
1070 slicename=slice_spec['slice_fields']['name']
1071 for nodename in slice_spec['nodenames']:
1072 print 'nodename',nodename,'slicename',slicename,'stamp',stamp
1073 (site,node) = self.locate_node (nodename)
1074 # xxx - passing the wrong site - probably harmless
1075 test_site = TestSite (self,site)
1076 test_slice = TestSlice (self,test_site,slice_spec)
1077 test_node = TestNode (self,test_site,node)
1078 test_sliver = TestSliver (self, test_node, test_slice)
1079 tasks.append ( CompleterTaskInitscript (test_sliver, stamp))
1080 return Completer (tasks).run (timedelta(minutes=5), timedelta(minutes=4), timedelta(seconds=10))
1082 def check_initscripts(self):
1083 "check that the initscripts have triggered"
1084 return self.do_check_initscripts()
1086 def initscripts (self):
1087 "create initscripts with PLCAPI"
1088 for initscript in self.plc_spec['initscripts']:
1089 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
1090 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
1093 def delete_initscripts (self):
1094 "delete initscripts with PLCAPI"
1095 for initscript in self.plc_spec['initscripts']:
1096 initscript_name = initscript['initscript_fields']['name']
1097 print('Attempting to delete %s in plc %s'%(initscript_name,self.plc_spec['name']))
1099 self.apiserver.DeleteInitScript(self.auth_root(),initscript_name)
1100 print initscript_name,'deleted'
1102 print 'deletion went wrong - probably did not exist'
1107 "create slices with PLCAPI"
1108 return self.do_slices(action="add")
1110 def delete_slices (self):
1111 "delete slices with PLCAPI"
1112 return self.do_slices(action="delete")
1114 def fill_slices (self):
1115 "add nodes in slices with PLCAPI"
1116 return self.do_slices(action="fill")
1118 def empty_slices (self):
1119 "remove nodes from slices with PLCAPI"
1120 return self.do_slices(action="empty")
1122 def do_slices (self, action="add"):
1123 for slice in self.plc_spec['slices']:
1124 site_spec = self.locate_site (slice['sitename'])
1125 test_site = TestSite(self,site_spec)
1126 test_slice=TestSlice(self,test_site,slice)
1127 if action == "delete":
1128 test_slice.delete_slice()
1129 elif action=="fill":
1130 test_slice.add_nodes()
1131 elif action=="empty":
1132 test_slice.delete_nodes()
1134 test_slice.create_slice()
1138 def ssh_slice(self): pass
1140 def ssh_slice_off (self): pass
1142 def ssh_slice_basics(self): pass
1145 def check_vsys_defaults(self): pass
1148 def keys_clear_known_hosts (self): pass
1150 def plcapi_urls (self):
1151 return PlcapiUrlScanner (self.auth_root(),ip=self.vserverip).scan()
1153 def speed_up_slices (self):
1154 "tweak nodemanager settings on all nodes using a conf file"
1155 # create the template on the server-side
1156 template="%s.nodemanager"%self.name()
1157 template_file = open (template,"w")
1158 template_file.write('OPTIONS="-p 30 -r 11 -d"\n')
1159 template_file.close()
1160 in_vm="/var/www/html/PlanetLabConf/nodemanager"
1161 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1162 self.test_ssh.copy_abs(template,remote)
1164 self.apiserver.AddConfFile (self.auth_root(),
1165 {'dest':'/etc/sysconfig/nodemanager',
1166 'source':'PlanetLabConf/nodemanager',
1167 'postinstall_cmd':'service nm restart',})
1170 def debug_nodemanager (self):
1171 "sets verbose mode for nodemanager, and speeds up cycle even more (needs speed_up_slices first)"
1172 template="%s.nodemanager"%self.name()
1173 template_file = open (template,"w")
1174 template_file.write('OPTIONS="-p 10 -r 6 -v -d"\n')
1175 template_file.close()
1176 in_vm="/var/www/html/PlanetLabConf/nodemanager"
1177 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1178 self.test_ssh.copy_abs(template,remote)
1182 def qemu_start (self) : pass
1185 def timestamp_qemu (self) : pass
1187 # when a spec refers to a node possibly on another plc
1188 def locate_sliver_obj_cross (self, nodename, slicename, other_plcs):
1189 for plc in [ self ] + other_plcs:
1191 return plc.locate_sliver_obj (nodename, slicename)
1194 raise Exception, "Cannot locate sliver %s@%s among all PLCs"%(nodename,slicename)
1196 # implement this one as a cross step so that we can take advantage of different nodes
1197 # in multi-plcs mode
1198 def cross_check_tcp (self, other_plcs):
1199 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
1200 if 'tcp_specs' not in self.plc_spec or not self.plc_spec['tcp_specs']:
1201 utils.header ("check_tcp: no/empty config found")
1203 specs = self.plc_spec['tcp_specs']
1208 s_test_sliver = self.locate_sliver_obj_cross (spec['server_node'],spec['server_slice'],other_plcs)
1209 if not s_test_sliver.run_tcp_server(port,timeout=20):
1213 # idem for the client side
1214 c_test_sliver = self.locate_sliver_obj_cross (spec['client_node'],spec['client_slice'],other_plcs)
1215 # use nodename from locatesd sliver, unless 'client_connect' is set
1216 if 'client_connect' in spec:
1217 destination = spec['client_connect']
1219 destination=s_test_sliver.test_node.name()
1220 if not c_test_sliver.run_tcp_client(destination,port):
1224 # painfully enough, we need to allow for some time as netflow might show up last
1225 def check_system_slice (self):
1226 "all nodes: check that a system slice is alive"
1227 # netflow currently not working in the lxc distro
1228 # drl not built at all in the wtx distro
1229 # if we find either of them we're happy
1230 return self.check_netflow() or self.check_drl()
1233 def check_netflow (self): return self._check_system_slice ('netflow')
1234 def check_drl (self): return self._check_system_slice ('drl')
1236 # we have the slices up already here, so it should not take too long
1237 def _check_system_slice (self, slicename, timeout_minutes=5, period_seconds=15):
1238 class CompleterTaskSystemSlice (CompleterTask):
1239 def __init__ (self, test_node, dry_run):
1240 self.test_node=test_node
1241 self.dry_run=dry_run
1242 def actual_run (self):
1243 return self.test_node._check_system_slice (slicename, dry_run=self.dry_run)
1245 return "System slice %s @ %s"%(slicename, self.test_node.name())
1246 def failure_message (self):
1247 return "COULD not find system slice %s @ %s"%(slicename, self.test_node.name())
1248 timeout = timedelta(minutes=timeout_minutes)
1249 silent = timedelta (0)
1250 period = timedelta (seconds=period_seconds)
1251 tasks = [ CompleterTaskSystemSlice (test_node, self.options.dry_run) \
1252 for test_node in self.all_nodes() ]
1253 return Completer (tasks) . run (timeout, silent, period)
1255 def plcsh_stress_test (self):
1256 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
1257 # install the stress-test in the plc image
1258 location = "/usr/share/plc_api/plcsh_stress_test.py"
1259 remote="%s/%s"%(self.vm_root_in_host(),location)
1260 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1262 command += " -- --check"
1263 if self.options.size == 1:
1264 command += " --tiny"
1265 return ( self.run_in_guest(command) == 0)
1267 # populate runs the same utility without slightly different options
1268 # in particular runs with --preserve (dont cleanup) and without --check
1269 # also it gets run twice, once with the --foreign option for creating fake foreign entries
1271 def sfa_install_all (self):
1272 "yum install sfa sfa-plc sfa-sfatables sfa-client"
1273 return self.yum_install ("sfa sfa-plc sfa-sfatables sfa-client")
1275 def sfa_install_core(self):
1277 return self.yum_install ("sfa")
1279 def sfa_install_plc(self):
1280 "yum install sfa-plc"
1281 return self.yum_install("sfa-plc")
1283 def sfa_install_sfatables(self):
1284 "yum install sfa-sfatables"
1285 return self.yum_install ("sfa-sfatables")
1287 # for some very odd reason, this sometimes fails with the following symptom
1288 # # yum install sfa-client
1289 # Setting up Install Process
1291 # Downloading Packages:
1292 # Running rpm_check_debug
1293 # Running Transaction Test
1294 # Transaction Test Succeeded
1295 # Running Transaction
1296 # Transaction couldn't start:
1297 # installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem
1298 # [('installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem', (9, '/', 69632L))]
1299 # even though in the same context I have
1300 # [2012.05.23--f14-32-sfastd1-1-vplc07] / # df -h
1301 # Filesystem Size Used Avail Use% Mounted on
1302 # /dev/hdv1 806G 264G 501G 35% /
1303 # none 16M 36K 16M 1% /tmp
1305 # so as a workaround, we first try yum install, and then invoke rpm on the cached rpm...
1306 def sfa_install_client(self):
1307 "yum install sfa-client"
1308 first_try=self.yum_install("sfa-client")
1309 if first_try: return True
1310 utils.header ("********** Regular yum failed - special workaround in place, 2nd chance")
1311 (code,cached_rpm_path)=utils.output_of(self.actual_command_in_guest('find /var/cache/yum -name sfa-client\*.rpm'))
1312 utils.header("rpm_path=<<%s>>"%rpm_path)
1314 self.run_in_guest("rpm -i %s"%cached_rpm_path)
1315 return self.yum_check_installed ("sfa-client")
1317 def sfa_dbclean(self):
1318 "thoroughly wipes off the SFA database"
1319 return self.run_in_guest("sfaadmin reg nuke")==0 or \
1320 self.run_in_guest("sfa-nuke.py")==0 or \
1321 self.run_in_guest("sfa-nuke-plc.py")==0
1323 def sfa_fsclean(self):
1324 "cleanup /etc/sfa/trusted_roots and /var/lib/sfa"
1325 self.run_in_guest("rm -rf /etc/sfa/trusted_roots /var/lib/sfa/authorities")
1328 def sfa_plcclean(self):
1329 "cleans the PLC entries that were created as a side effect of running the script"
1331 sfa_spec=self.plc_spec['sfa']
1333 for auth_sfa_spec in sfa_spec['auth_sfa_specs']:
1334 login_base=auth_sfa_spec['login_base']
1335 try: self.apiserver.DeleteSite (self.auth_root(),login_base)
1336 except: print "Site %s already absent from PLC db"%login_base
1338 for spec_name in ['pi_spec','user_spec']:
1339 user_spec=auth_sfa_spec[spec_name]
1340 username=user_spec['email']
1341 try: self.apiserver.DeletePerson(self.auth_root(),username)
1343 # this in fact is expected as sites delete their members
1344 #print "User %s already absent from PLC db"%username
1347 print "REMEMBER TO RUN sfa_import AGAIN"
1350 def sfa_uninstall(self):
1351 "uses rpm to uninstall sfa - ignore result"
1352 self.run_in_guest("rpm -e sfa sfa-sfatables sfa-client sfa-plc")
1353 self.run_in_guest("rm -rf /var/lib/sfa")
1354 self.run_in_guest("rm -rf /etc/sfa")
1355 self.run_in_guest("rm -rf /var/log/sfa_access.log /var/log/sfa_import_plc.log /var/log/sfa.daemon")
1357 self.run_in_guest("rpm -e --noscripts sfa-plc")
1360 ### run unit tests for SFA
1361 # NOTE: for some reason on f14/i386, yum install sfa-tests fails for no reason
1362 # Running Transaction
1363 # Transaction couldn't start:
1364 # installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem
1365 # [('installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem', (9, '/', 208896L))]
1366 # no matter how many Gbs are available on the testplc
1367 # could not figure out what's wrong, so...
1368 # if the yum install phase fails, consider the test is successful
1369 # other combinations will eventually run it hopefully
1370 def sfa_utest(self):
1371 "yum install sfa-tests and run SFA unittests"
1372 self.run_in_guest("yum -y install sfa-tests")
1373 # failed to install - forget it
1374 if self.run_in_guest("rpm -q sfa-tests")!=0:
1375 utils.header("WARNING: SFA unit tests failed to install, ignoring")
1377 return self.run_in_guest("/usr/share/sfa/tests/testAll.py")==0
1381 dirname="conf.%s"%self.plc_spec['name']
1382 if not os.path.isdir(dirname):
1383 utils.system("mkdir -p %s"%dirname)
1384 if not os.path.isdir(dirname):
1385 raise Exception,"Cannot create config dir for plc %s"%self.name()
1388 def conffile(self,filename):
1389 return "%s/%s"%(self.confdir(),filename)
1390 def confsubdir(self,dirname,clean,dry_run=False):
1391 subdirname="%s/%s"%(self.confdir(),dirname)
1393 utils.system("rm -rf %s"%subdirname)
1394 if not os.path.isdir(subdirname):
1395 utils.system("mkdir -p %s"%subdirname)
1396 if not dry_run and not os.path.isdir(subdirname):
1397 raise "Cannot create config subdir %s for plc %s"%(dirname,self.name())
1400 def conffile_clean (self,filename):
1401 filename=self.conffile(filename)
1402 return utils.system("rm -rf %s"%filename)==0
1405 def sfa_configure(self):
1406 "run sfa-config-tty"
1407 tmpname=self.conffile("sfa-config-tty")
1408 fileconf=open(tmpname,'w')
1409 for var in [ 'SFA_REGISTRY_ROOT_AUTH',
1410 'SFA_INTERFACE_HRN',
1411 'SFA_REGISTRY_LEVEL1_AUTH',
1412 'SFA_REGISTRY_HOST',
1413 'SFA_AGGREGATE_HOST',
1423 'SFA_GENERIC_FLAVOUR',
1424 'SFA_AGGREGATE_ENABLED',
1426 if self.plc_spec['sfa'].has_key(var):
1427 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec['sfa'][var]))
1428 # the way plc_config handles booleans just sucks..
1431 if self.plc_spec['sfa'][var]: val='true'
1432 fileconf.write ('e %s\n%s\n'%(var,val))
1433 fileconf.write('w\n')
1434 fileconf.write('R\n')
1435 fileconf.write('q\n')
1437 utils.system('cat %s'%tmpname)
1438 self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
1441 def aggregate_xml_line(self):
1442 port=self.plc_spec['sfa']['neighbours-port']
1443 return '<aggregate addr="%s" hrn="%s" port="%r"/>' % \
1444 (self.vserverip,self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH'],port)
1446 def registry_xml_line(self):
1447 return '<registry addr="%s" hrn="%s" port="12345"/>' % \
1448 (self.vserverip,self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH'])
1451 # a cross step that takes all other plcs in argument
1452 def cross_sfa_configure(self, other_plcs):
1453 "writes aggregates.xml and registries.xml that point to all other PLCs in the test"
1454 # of course with a single plc, other_plcs is an empty list
1457 agg_fname=self.conffile("agg.xml")
1458 file(agg_fname,"w").write("<aggregates>%s</aggregates>\n" % \
1459 " ".join([ plc.aggregate_xml_line() for plc in other_plcs ]))
1460 utils.header ("(Over)wrote %s"%agg_fname)
1461 reg_fname=self.conffile("reg.xml")
1462 file(reg_fname,"w").write("<registries>%s</registries>\n" % \
1463 " ".join([ plc.registry_xml_line() for plc in other_plcs ]))
1464 utils.header ("(Over)wrote %s"%reg_fname)
1465 return self.test_ssh.copy_abs(agg_fname,'/%s/etc/sfa/aggregates.xml'%self.vm_root_in_host())==0 \
1466 and self.test_ssh.copy_abs(reg_fname,'/%s/etc/sfa/registries.xml'%self.vm_root_in_host())==0
1468 def sfa_import(self):
1469 "use sfaadmin to import from plc"
1470 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1472 self.run_in_guest('sfaadmin reg import_registry')==0
1473 # not needed anymore
1474 # self.run_in_guest('cp /etc/sfa/authorities/%s/%s.pkey /etc/sfa/authorities/server.key'%(auth,auth))
1476 def sfa_start(self):
1478 return self.run_in_guest('service sfa start')==0
1480 def sfi_configure(self):
1481 "Create /root/sfi on the plc side for sfi client configuration"
1482 if self.options.dry_run:
1483 utils.header("DRY RUN - skipping step")
1485 sfa_spec=self.plc_spec['sfa']
1486 # cannot use auth_sfa_mapper to pass dir_name
1487 for slice_spec in self.plc_spec['sfa']['auth_sfa_specs']:
1488 test_slice=TestAuthSfa(self,slice_spec)
1489 dir_basename=os.path.basename(test_slice.sfi_path())
1490 dir_name=self.confsubdir("dot-sfi/%s"%dir_basename,clean=True,dry_run=self.options.dry_run)
1491 test_slice.sfi_configure(dir_name)
1492 # push into the remote /root/sfi area
1493 location = test_slice.sfi_path()
1494 remote="%s/%s"%(self.vm_root_in_host(),location)
1495 self.test_ssh.mkdir(remote,abs=True)
1496 # need to strip last level or remote otherwise we get an extra dir level
1497 self.test_ssh.copy_abs(dir_name, os.path.dirname(remote), recursive=True)
1501 def sfi_clean (self):
1502 "clean up /root/sfi on the plc side"
1503 self.run_in_guest("rm -rf /root/sfi")
1507 def sfa_add_site (self): pass
1509 def sfa_add_pi (self): pass
1511 def sfa_add_user(self): pass
1513 def sfa_update_user(self): pass
1515 def sfa_add_slice(self): pass
1517 def sfa_renew_slice(self): pass
1519 def sfa_discover(self): pass
1521 def sfa_create_slice(self): pass
1523 def sfa_check_slice_plc(self): pass
1525 def sfa_update_slice(self): pass
1527 def sfi_list(self): pass
1529 def sfi_show(self): pass
1531 def ssh_slice_sfa(self): pass
1533 def sfa_delete_user(self): pass
1535 def sfa_delete_slice(self): pass
1539 self.run_in_guest('service sfa stop')==0
1542 def populate (self):
1543 "creates random entries in the PLCAPI"
1544 # install the stress-test in the plc image
1545 location = "/usr/share/plc_api/plcsh_stress_test.py"
1546 remote="%s/%s"%(self.vm_root_in_host(),location)
1547 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1549 command += " -- --preserve --short-names"
1550 local = (self.run_in_guest(command) == 0);
1551 # second run with --foreign
1552 command += ' --foreign'
1553 remote = (self.run_in_guest(command) == 0);
1554 return ( local and remote)
1556 def gather_logs (self):
1557 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1558 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1559 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1560 # (1.c) get the plc's /root/sfi -> logs/sfi.<plcname>/
1561 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1562 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1563 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1565 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1566 self.gather_var_logs ()
1568 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1569 self.gather_pgsql_logs ()
1571 print "-------------------- TestPlc.gather_logs : PLC's /root/sfi/"
1572 self.gather_root_sfi ()
1574 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1575 for site_spec in self.plc_spec['sites']:
1576 test_site = TestSite (self,site_spec)
1577 for node_spec in site_spec['nodes']:
1578 test_node=TestNode(self,test_site,node_spec)
1579 test_node.gather_qemu_logs()
1581 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1582 self.gather_nodes_var_logs()
1584 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1585 self.gather_slivers_var_logs()
1588 def gather_slivers_var_logs(self):
1589 for test_sliver in self.all_sliver_objs():
1590 remote = test_sliver.tar_var_logs()
1591 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
1592 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
1593 utils.system(command)
1596 def gather_var_logs (self):
1597 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
1598 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1599 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
1600 utils.system(command)
1601 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd"%self.name()
1602 utils.system(command)
1604 def gather_pgsql_logs (self):
1605 utils.system("mkdir -p logs/myplc.pgsql-log.%s"%self.name())
1606 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1607 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -"%self.name()
1608 utils.system(command)
1610 def gather_root_sfi (self):
1611 utils.system("mkdir -p logs/sfi.%s"%self.name())
1612 to_plc = self.actual_command_in_guest("tar -C /root/sfi/ -cf - .")
1613 command = to_plc + "| tar -C logs/sfi.%s -xf -"%self.name()
1614 utils.system(command)
1616 def gather_nodes_var_logs (self):
1617 for site_spec in self.plc_spec['sites']:
1618 test_site = TestSite (self,site_spec)
1619 for node_spec in site_spec['nodes']:
1620 test_node=TestNode(self,test_site,node_spec)
1621 test_ssh = TestSsh (test_node.name(),key="keys/key_admin.rsa")
1622 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1623 command = command + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
1624 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
1625 utils.system(command)
1628 # returns the filename to use for sql dump/restore, using options.dbname if set
1629 def dbfile (self, database):
1630 # uses options.dbname if it is found
1632 name=self.options.dbname
1633 if not isinstance(name,StringTypes):
1639 return "/root/%s-%s.sql"%(database,name)
1641 def plc_db_dump(self):
1642 'dump the planetlab5 DB in /root in the PLC - filename has time'
1643 dump=self.dbfile("planetab5")
1644 self.run_in_guest('pg_dump -U pgsqluser planetlab5 -f '+ dump)
1645 utils.header('Dumped planetlab5 database in %s'%dump)
1648 def plc_db_restore(self):
1649 'restore the planetlab5 DB - looks broken, but run -n might help'
1650 dump=self.dbfile("planetab5")
1651 ##stop httpd service
1652 self.run_in_guest('service httpd stop')
1653 # xxx - need another wrapper
1654 self.run_in_guest_piped('echo drop database planetlab5','psql --user=pgsqluser template1')
1655 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab5')
1656 self.run_in_guest('psql -U pgsqluser planetlab5 -f '+dump)
1657 ##starting httpd service
1658 self.run_in_guest('service httpd start')
1660 utils.header('Database restored from ' + dump)
1662 def standby_1_through_20(self):
1663 """convenience function to wait for a specified number of minutes"""
1666 def standby_1(): pass
1668 def standby_2(): pass
1670 def standby_3(): pass
1672 def standby_4(): pass
1674 def standby_5(): pass
1676 def standby_6(): pass
1678 def standby_7(): pass
1680 def standby_8(): pass
1682 def standby_9(): pass
1684 def standby_10(): pass
1686 def standby_11(): pass
1688 def standby_12(): pass
1690 def standby_13(): pass
1692 def standby_14(): pass
1694 def standby_15(): pass
1696 def standby_16(): pass
1698 def standby_17(): pass
1700 def standby_18(): pass
1702 def standby_19(): pass
1704 def standby_20(): pass