1 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
2 # Copyright (C) 2010 INRIA
9 from datetime import datetime, timedelta
10 from types import StringTypes
13 from TestSite import TestSite
14 from TestNode import TestNode
15 from TestUser import TestUser
16 from TestKey import TestKey
17 from TestSlice import TestSlice
18 from TestSliver import TestSliver
19 from TestBoxQemu import TestBoxQemu
20 from TestSsh import TestSsh
21 from TestApiserver import TestApiserver
22 from TestAuthSfa import TestAuthSfa
23 from PlcapiUrlScanner import PlcapiUrlScanner
24 from Completer import Completer, CompleterTask
26 # step methods must take (self) and return a boolean (options is a member of the class)
28 def standby(minutes,dry_run):
29 utils.header('Entering StandBy for %d mn'%minutes)
33 time.sleep(60*minutes)
36 def standby_generic (func):
38 minutes=int(func.__name__.split("_")[1])
39 return standby(minutes,self.options.dry_run)
42 def node_mapper (method):
43 def actual(self,*args, **kwds):
45 node_method = TestNode.__dict__[method.__name__]
46 for test_node in self.all_nodes():
47 if not node_method(test_node, *args, **kwds): overall=False
49 # restore the doc text
50 actual.__doc__=TestNode.__dict__[method.__name__].__doc__
53 def slice_mapper (method):
56 slice_method = TestSlice.__dict__[method.__name__]
57 for slice_spec in self.plc_spec['slices']:
58 site_spec = self.locate_site (slice_spec['sitename'])
59 test_site = TestSite(self,site_spec)
60 test_slice=TestSlice(self,test_site,slice_spec)
61 if not slice_method(test_slice,self.options): overall=False
63 # restore the doc text
64 actual.__doc__=TestSlice.__dict__[method.__name__].__doc__
67 def auth_sfa_mapper (method):
70 auth_method = TestAuthSfa.__dict__[method.__name__]
71 for auth_spec in self.plc_spec['sfa']['auth_sfa_specs']:
72 test_auth=TestAuthSfa(self,auth_spec)
73 if not auth_method(test_auth,self.options): overall=False
75 # restore the doc text
76 actual.__doc__=TestAuthSfa.__dict__[method.__name__].__doc__
86 'vs_delete','timestamp_vs','vs_create', SEP,
87 'plc_install', 'plc_configure', 'plc_start', SEP,
88 'keys_fetch', 'keys_store', 'keys_clear_known_hosts', SEP,
89 'plcapi_urls','speed_up_slices', SEP,
90 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', 'leases', SEP,
91 # slices created under plcsh interactively seem to be fine but these ones don't have the tags
92 # keep this our of the way for now
93 # 'check_vsys_defaults', SEP,
94 'nodestate_reinstall', 'qemu_local_init','bootcd', 'qemu_local_config', SEP,
95 'qemu_export', 'qemu_kill_mine', 'qemu_start', 'timestamp_qemu', SEP,
96 'sfa_install_all', 'sfa_configure', 'cross_sfa_configure', 'sfa_start', 'sfa_import', SEPSFA,
97 'sfi_configure@1', 'sfa_add_site@1','sfa_add_pi@1', SEPSFA,
98 'sfa_add_user@1', 'sfa_update_user@1', 'sfa_add_slice@1', 'sfa_renew_slice@1', SEPSFA,
99 'sfa_discover@1', 'sfa_create_slice@1', 'sfa_check_slice_plc@1', 'sfa_update_slice@1', SEPSFA,
100 'sfi_list@1', 'sfi_show@1', 'sfi_slices@1', 'sfa_utest@1', SEPSFA,
101 # we used to run plcsh_stress_test, and then ssh_node_debug and ssh_node_boot
102 # but as the stress test might take a while, we sometimes missed the debug mode..
103 'ssh_node_debug@1', 'plcsh_stress_test@1', SEP,
104 'ssh_node_boot@1', 'node_bmlogs@1', 'ssh_slice', 'ssh_slice_basics', 'check_initscripts', SEP,
105 'ssh_slice_sfa@1', 'sfa_delete_slice@1', 'sfa_delete_user@1', SEPSFA,
106 'cross_check_tcp@1', 'check_system_slice', SEP,
107 'empty_slices', 'ssh_slice_off', 'fill_slices', SEP,
108 'force_gather_logs', SEP,
111 'export', 'show_boxes', SEP,
112 'check_hooks', 'plc_stop', 'vs_start', 'vs_stop', SEP,
113 'delete_initscripts', 'delete_nodegroups','delete_all_sites', SEP,
114 'delete_sites', 'delete_nodes', 'delete_slices', 'keys_clean', SEP,
115 'delete_leases', 'list_leases', SEP,
117 'nodestate_show','nodestate_safeboot','nodestate_boot', SEP,
118 'qemu_list_all', 'qemu_list_mine', 'qemu_kill_all', SEP,
119 'sfa_install_core', 'sfa_install_sfatables', 'sfa_install_plc', 'sfa_install_client', SEPSFA,
120 'sfa_plcclean', 'sfa_dbclean', 'sfa_stop','sfa_uninstall', 'sfi_clean', SEPSFA,
121 'plc_db_dump' , 'plc_db_restore', SEP,
122 'check_netflow','check_drl', SEP,
123 'debug_nodemanager', SEP,
124 'standby_1_through_20',SEP,
128 def printable_steps (list):
129 single_line=" ".join(list)+" "
130 return single_line.replace(" "+SEP+" "," \\\n").replace(" "+SEPSFA+" "," \\\n")
132 def valid_step (step):
133 return step != SEP and step != SEPSFA
135 # turn off the sfa-related steps when build has skipped SFA
136 # this was originally for centos5 but is still valid
137 # for up to f12 as recent SFAs with sqlalchemy won't build before f14
139 def check_whether_build_has_sfa (rpms_url):
140 utils.header ("Checking if build provides SFA package...")
141 # warning, we're now building 'sface' so let's be a bit more picky
142 retcod=os.system ("curl --silent %s/ | grep -q sfa-"%rpms_url)
143 # full builds are expected to return with 0 here
145 utils.header("build does provide SFA")
147 # move all steps containing 'sfa' from default_steps to other_steps
148 utils.header("SFA package not found - removing steps with sfa or sfi")
149 sfa_steps= [ step for step in TestPlc.default_steps if step.find('sfa')>=0 or step.find("sfi")>=0 ]
150 TestPlc.other_steps += sfa_steps
151 for step in sfa_steps: TestPlc.default_steps.remove(step)
153 def __init__ (self,plc_spec,options):
154 self.plc_spec=plc_spec
156 self.test_ssh=TestSsh(self.plc_spec['host_box'],self.options.buildname)
157 self.vserverip=plc_spec['vserverip']
158 self.vservername=plc_spec['vservername']
159 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
160 self.apiserver=TestApiserver(self.url,options.dry_run)
161 (self.ssh_node_boot_timeout,self.ssh_node_boot_silent)=plc_spec['ssh_node_boot_timers']
162 (self.ssh_node_debug_timeout,self.ssh_node_debug_silent)=plc_spec['ssh_node_debug_timers']
164 def has_addresses_api (self):
165 return self.apiserver.has_method('AddIpAddress')
168 name=self.plc_spec['name']
169 return "%s.%s"%(name,self.vservername)
172 return self.plc_spec['host_box']
175 return self.test_ssh.is_local()
177 # define the API methods on this object through xmlrpc
178 # would help, but not strictly necessary
182 def actual_command_in_guest (self,command):
183 return self.test_ssh.actual_command(self.host_to_guest(command))
185 def start_guest (self):
186 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host()))
188 def stop_guest (self):
189 return utils.system(self.test_ssh.actual_command(self.stop_guest_in_host()))
191 def run_in_guest (self,command):
192 return utils.system(self.actual_command_in_guest(command))
194 def run_in_host (self,command):
195 return self.test_ssh.run_in_buildname(command, dry_run=self.options.dry_run)
197 #command gets run in the plc's vm
198 def host_to_guest(self,command):
199 if self.options.plcs_use_lxc:
200 return "ssh -o StrictHostKeyChecking=no %s %s"%(self.vserverip,command)
202 return "vserver %s exec %s"%(self.vservername,command)
204 def vm_root_in_host(self):
205 if self.options.plcs_use_lxc:
206 return "/vservers/%s/rootfs/"%(self.vservername)
208 return "/vservers/%s"%(self.vservername)
210 def vm_timestamp_path (self):
211 if self.options.plcs_use_lxc:
212 return "/vservers/%s/%s.timestamp"%(self.vservername,self.vservername)
214 return "/vservers/%s.timestamp"%(self.vservername)
216 #start/stop the vserver
217 def start_guest_in_host(self):
218 if self.options.plcs_use_lxc:
219 return "virsh -c lxc:// start %s"%(self.vservername)
221 return "vserver %s start"%(self.vservername)
223 def stop_guest_in_host(self):
224 if self.options.plcs_use_lxc:
225 return "virsh -c lxc:// destroy %s"%(self.vservername)
227 return "vserver %s stop"%(self.vservername)
230 def run_in_guest_piped (self,local,remote):
231 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
233 def yum_check_installed (self, rpms):
234 if isinstance (rpms, list):
236 return self.run_in_guest("rpm -q %s"%rpms)==0
238 # does a yum install in the vs, ignore yum retcod, check with rpm
239 def yum_install (self, rpms):
240 if isinstance (rpms, list):
242 self.run_in_guest("yum -y install %s"%rpms)
243 # yum-complete-transaction comes with yum-utils, that is in vtest.pkgs
244 self.run_in_guest("yum-complete-transaction -y")
245 return self.yum_check_installed (rpms)
247 def auth_root (self):
248 return {'Username':self.plc_spec['PLC_ROOT_USER'],
249 'AuthMethod':'password',
250 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
251 'Role' : self.plc_spec['role']
253 def locate_site (self,sitename):
254 for site in self.plc_spec['sites']:
255 if site['site_fields']['name'] == sitename:
257 if site['site_fields']['login_base'] == sitename:
259 raise Exception,"Cannot locate site %s"%sitename
261 def locate_node (self,nodename):
262 for site in self.plc_spec['sites']:
263 for node in site['nodes']:
264 if node['name'] == nodename:
266 raise Exception,"Cannot locate node %s"%nodename
268 def locate_hostname (self,hostname):
269 for site in self.plc_spec['sites']:
270 for node in site['nodes']:
271 if node['node_fields']['hostname'] == hostname:
273 raise Exception,"Cannot locate hostname %s"%hostname
275 def locate_key (self,key_name):
276 for key in self.plc_spec['keys']:
277 if key['key_name'] == key_name:
279 raise Exception,"Cannot locate key %s"%key_name
281 def locate_private_key_from_key_names (self, key_names):
282 # locate the first avail. key
284 for key_name in key_names:
285 key_spec=self.locate_key(key_name)
286 test_key=TestKey(self,key_spec)
287 publickey=test_key.publicpath()
288 privatekey=test_key.privatepath()
289 if os.path.isfile(publickey) and os.path.isfile(privatekey):
291 if found: return privatekey
294 def locate_slice (self, slicename):
295 for slice in self.plc_spec['slices']:
296 if slice['slice_fields']['name'] == slicename:
298 raise Exception,"Cannot locate slice %s"%slicename
300 def all_sliver_objs (self):
302 for slice_spec in self.plc_spec['slices']:
303 slicename = slice_spec['slice_fields']['name']
304 for nodename in slice_spec['nodenames']:
305 result.append(self.locate_sliver_obj (nodename,slicename))
308 def locate_sliver_obj (self,nodename,slicename):
309 (site,node) = self.locate_node(nodename)
310 slice = self.locate_slice (slicename)
312 test_site = TestSite (self, site)
313 test_node = TestNode (self, test_site,node)
314 # xxx the slice site is assumed to be the node site - mhh - probably harmless
315 test_slice = TestSlice (self, test_site, slice)
316 return TestSliver (self, test_node, test_slice)
318 def locate_first_node(self):
319 nodename=self.plc_spec['slices'][0]['nodenames'][0]
320 (site,node) = self.locate_node(nodename)
321 test_site = TestSite (self, site)
322 test_node = TestNode (self, test_site,node)
325 def locate_first_sliver (self):
326 slice_spec=self.plc_spec['slices'][0]
327 slicename=slice_spec['slice_fields']['name']
328 nodename=slice_spec['nodenames'][0]
329 return self.locate_sliver_obj(nodename,slicename)
331 # all different hostboxes used in this plc
332 def gather_hostBoxes(self):
333 # maps on sites and nodes, return [ (host_box,test_node) ]
335 for site_spec in self.plc_spec['sites']:
336 test_site = TestSite (self,site_spec)
337 for node_spec in site_spec['nodes']:
338 test_node = TestNode (self, test_site, node_spec)
339 if not test_node.is_real():
340 tuples.append( (test_node.host_box(),test_node) )
341 # transform into a dict { 'host_box' -> [ test_node .. ] }
343 for (box,node) in tuples:
344 if not result.has_key(box):
347 result[box].append(node)
350 # a step for checking this stuff
351 def show_boxes (self):
352 'print summary of nodes location'
353 for (box,nodes) in self.gather_hostBoxes().iteritems():
354 print box,":"," + ".join( [ node.name() for node in nodes ] )
357 # make this a valid step
358 def qemu_kill_all(self):
359 'kill all qemu instances on the qemu boxes involved by this setup'
360 # this is the brute force version, kill all qemus on that host box
361 for (box,nodes) in self.gather_hostBoxes().iteritems():
362 # pass the first nodename, as we don't push template-qemu on testboxes
363 nodedir=nodes[0].nodedir()
364 TestBoxQemu(box,self.options.buildname).qemu_kill_all(nodedir)
367 # make this a valid step
368 def qemu_list_all(self):
369 'list all qemu instances on the qemu boxes involved by this setup'
370 for (box,nodes) in self.gather_hostBoxes().iteritems():
371 # this is the brute force version, kill all qemus on that host box
372 TestBoxQemu(box,self.options.buildname).qemu_list_all()
375 # kill only the right qemus
376 def qemu_list_mine(self):
377 'list qemu instances for our nodes'
378 for (box,nodes) in self.gather_hostBoxes().iteritems():
379 # the fine-grain version
384 # kill only the right qemus
385 def qemu_kill_mine(self):
386 'kill the qemu instances for our nodes'
387 for (box,nodes) in self.gather_hostBoxes().iteritems():
388 # the fine-grain version
393 #################### display config
395 "show test configuration after localization"
400 # uggly hack to make sure 'run export' only reports about the 1st plc
401 # to avoid confusion - also we use 'inri_slice1' in various aliases..
404 "print cut'n paste-able stuff to export env variables to your shell"
405 # guess local domain from hostname
406 if TestPlc.exported_id>1:
407 print "export GUESTHOSTNAME%d=%s"%(TestPlc.exported_id,self.plc_spec['vservername'])
409 TestPlc.exported_id+=1
410 domain=socket.gethostname().split('.',1)[1]
411 fqdn="%s.%s"%(self.plc_spec['host_box'],domain)
412 print "export BUILD=%s"%self.options.buildname
413 if self.options.plcs_use_lxc:
414 print "export PLCHOSTLXC=%s"%fqdn
416 print "export PLCHOSTVS=%s"%fqdn
417 print "export GUESTNAME=%s"%self.plc_spec['vservername']
418 vplcname=self.plc_spec['vservername'].split('-')[-1]
419 print "export GUESTHOSTNAME=%s.%s"%(vplcname,domain)
420 # find hostname of first node
421 (hostname,qemubox) = self.all_node_infos()[0]
422 print "export KVMHOST=%s.%s"%(qemubox,domain)
423 print "export NODE=%s"%(hostname)
427 always_display_keys=['PLC_WWW_HOST','nodes','sites',]
428 def show_pass (self,passno):
429 for (key,val) in self.plc_spec.iteritems():
430 if not self.options.verbose and key not in TestPlc.always_display_keys: continue
434 self.display_site_spec(site)
435 for node in site['nodes']:
436 self.display_node_spec(node)
437 elif key=='initscripts':
438 for initscript in val:
439 self.display_initscript_spec (initscript)
442 self.display_slice_spec (slice)
445 self.display_key_spec (key)
447 if key not in ['sites','initscripts','slices','keys', 'sfa']:
448 print '+ ',key,':',val
450 def display_site_spec (self,site):
451 print '+ ======== site',site['site_fields']['name']
452 for (k,v) in site.iteritems():
453 if not self.options.verbose and k not in TestPlc.always_display_keys: continue
456 print '+ ','nodes : ',
458 print node['node_fields']['hostname'],'',
464 print user['name'],'',
466 elif k == 'site_fields':
467 print '+ login_base',':',v['login_base']
468 elif k == 'address_fields':
474 def display_initscript_spec (self,initscript):
475 print '+ ======== initscript',initscript['initscript_fields']['name']
477 def display_key_spec (self,key):
478 print '+ ======== key',key['key_name']
480 def display_slice_spec (self,slice):
481 print '+ ======== slice',slice['slice_fields']['name']
482 for (k,v) in slice.iteritems():
495 elif k=='slice_fields':
496 print '+ fields',':',
497 print 'max_nodes=',v['max_nodes'],
502 def display_node_spec (self,node):
503 print "+ node=%s host_box=%s"%(node['name'],node['host_box']),
504 print "hostname=",node['node_fields']['hostname'],
505 print "ip=",node['interface_fields']['ip']
506 if self.options.verbose:
507 utils.pprint("node details",node,depth=3)
509 # another entry point for just showing the boxes involved
510 def display_mapping (self):
511 TestPlc.display_mapping_plc(self.plc_spec)
515 def display_mapping_plc (plc_spec):
516 print '+ MyPLC',plc_spec['name']
517 # WARNING this would not be right for lxc-based PLC's - should be harmless though
518 print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['host_box'],plc_spec['vservername'])
519 print '+\tIP = %s/%s'%(plc_spec['PLC_API_HOST'],plc_spec['vserverip'])
520 for site_spec in plc_spec['sites']:
521 for node_spec in site_spec['nodes']:
522 TestPlc.display_mapping_node(node_spec)
525 def display_mapping_node (node_spec):
526 print '+ NODE %s'%(node_spec['name'])
527 print '+\tqemu box %s'%node_spec['host_box']
528 print '+\thostname=%s'%node_spec['node_fields']['hostname']
530 # write a timestamp in /vservers/<>.timestamp
531 # cannot be inside the vserver, that causes vserver .. build to cough
532 def timestamp_vs (self):
533 "Create a timestamp to remember creation date for this plc"
535 # TODO-lxc check this one
536 # a first approx. is to store the timestamp close to the VM root like vs does
537 stamp_path=self.vm_timestamp_path ()
538 stamp_dir = os.path.dirname (stamp_path)
539 utils.system(self.test_ssh.actual_command("mkdir -p %s"%stamp_dir))
540 return utils.system(self.test_ssh.actual_command("echo %d > %s"%(now,stamp_path)))==0
542 # this is called inconditionnally at the beginning of the test sequence
543 # just in case this is a rerun, so if the vm is not running it's fine
545 "vserver delete the test myplc"
546 stamp_path=self.vm_timestamp_path()
547 self.run_in_host("rm -f %s"%stamp_path)
548 if self.options.plcs_use_lxc:
549 self.run_in_host("virsh -c lxc:// destroy %s"%self.vservername)
550 self.run_in_host("virsh -c lxc:// undefine %s"%self.vservername)
551 self.run_in_host("rm -fr /vservers/%s"%self.vservername)
554 self.run_in_host("vserver --silent %s delete"%self.vservername)
558 # historically the build was being fetched by the tests
559 # now the build pushes itself as a subdir of the tests workdir
560 # so that the tests do not have to worry about extracting the build (svn, git, or whatever)
561 def vs_create (self):
562 "vserver creation (no install done)"
563 # push the local build/ dir to the testplc box
565 # a full path for the local calls
566 build_dir=os.path.dirname(sys.argv[0])
567 # sometimes this is empty - set to "." in such a case
568 if not build_dir: build_dir="."
569 build_dir += "/build"
571 # use a standard name - will be relative to remote buildname
573 # remove for safety; do *not* mkdir first, otherwise we end up with build/build/
574 self.test_ssh.rmdir(build_dir)
575 self.test_ssh.copy(build_dir,recursive=True)
576 # the repo url is taken from arch-rpms-url
577 # with the last step (i386) removed
578 repo_url = self.options.arch_rpms_url
579 for level in [ 'arch' ]:
580 repo_url = os.path.dirname(repo_url)
581 # pass the vbuild-nightly options to vtest-init-vserver
583 test_env_options += " -p %s"%self.options.personality
584 test_env_options += " -d %s"%self.options.pldistro
585 test_env_options += " -f %s"%self.options.fcdistro
586 if self.options.plcs_use_lxc:
587 script="vtest-init-lxc.sh"
589 script="vtest-init-vserver.sh"
590 vserver_name = self.vservername
591 vserver_options="--netdev eth0 --interface %s"%self.vserverip
593 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
594 vserver_options += " --hostname %s"%vserver_hostname
596 print "Cannot reverse lookup %s"%self.vserverip
597 print "This is considered fatal, as this might pollute the test results"
599 create_vserver="%(build_dir)s/%(script)s %(test_env_options)s %(vserver_name)s %(repo_url)s -- %(vserver_options)s"%locals()
600 return self.run_in_host(create_vserver) == 0
603 def plc_install(self):
604 "yum install myplc, noderepo, and the plain bootstrapfs"
606 # workaround for getting pgsql8.2 on centos5
607 if self.options.fcdistro == "centos5":
608 self.run_in_guest("rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-3.noarch.rpm")
611 if self.options.personality == "linux32":
613 elif self.options.personality == "linux64":
616 raise Exception, "Unsupported personality %r"%self.options.personality
617 nodefamily="%s-%s-%s"%(self.options.pldistro,self.options.fcdistro,arch)
620 pkgs_list.append ("slicerepo-%s"%nodefamily)
621 pkgs_list.append ("myplc")
622 pkgs_list.append ("noderepo-%s"%nodefamily)
623 pkgs_list.append ("nodeimage-%s-plain"%nodefamily)
624 pkgs_string=" ".join(pkgs_list)
625 return self.yum_install (pkgs_list)
628 def plc_configure(self):
630 tmpname='%s.plc-config-tty'%(self.name())
631 fileconf=open(tmpname,'w')
632 for var in [ 'PLC_NAME',
637 'PLC_MAIL_SUPPORT_ADDRESS',
640 # Above line was added for integrating SFA Testing
646 'PLC_RESERVATION_GRANULARITY',
648 'PLC_OMF_XMPP_SERVER',
651 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
652 fileconf.write('w\n')
653 fileconf.write('q\n')
655 utils.system('cat %s'%tmpname)
656 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
657 utils.system('rm %s'%tmpname)
662 self.run_in_guest('service plc start')
667 self.run_in_guest('service plc stop')
671 "start the PLC vserver"
676 "stop the PLC vserver"
680 # stores the keys from the config for further use
681 def keys_store(self):
682 "stores test users ssh keys in keys/"
683 for key_spec in self.plc_spec['keys']:
684 TestKey(self,key_spec).store_key()
687 def keys_clean(self):
688 "removes keys cached in keys/"
689 utils.system("rm -rf ./keys")
692 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
693 # for later direct access to the nodes
694 def keys_fetch(self):
695 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
697 if not os.path.isdir(dir):
699 vservername=self.vservername
700 vm_root=self.vm_root_in_host()
702 prefix = 'debug_ssh_key'
703 for ext in [ 'pub', 'rsa' ] :
704 src="%(vm_root)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
705 dst="keys/%(vservername)s-debug.%(ext)s"%locals()
706 if self.test_ssh.fetch(src,dst) != 0: overall=False
710 "create sites with PLCAPI"
711 return self.do_sites()
713 def delete_sites (self):
714 "delete sites with PLCAPI"
715 return self.do_sites(action="delete")
717 def do_sites (self,action="add"):
718 for site_spec in self.plc_spec['sites']:
719 test_site = TestSite (self,site_spec)
720 if (action != "add"):
721 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
722 test_site.delete_site()
723 # deleted with the site
724 #test_site.delete_users()
727 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
728 test_site.create_site()
729 test_site.create_users()
732 def delete_all_sites (self):
733 "Delete all sites in PLC, and related objects"
734 print 'auth_root',self.auth_root()
735 sites = self.apiserver.GetSites(self.auth_root(), {}, ['site_id','login_base'])
737 # keep automatic site - otherwise we shoot in our own foot, root_auth is not valid anymore
738 if site['login_base']==self.plc_spec['PLC_SLICE_PREFIX']: continue
739 site_id=site['site_id']
740 print 'Deleting site_id',site_id
741 self.apiserver.DeleteSite(self.auth_root(),site_id)
745 "create nodes with PLCAPI"
746 return self.do_nodes()
747 def delete_nodes (self):
748 "delete nodes with PLCAPI"
749 return self.do_nodes(action="delete")
751 def do_nodes (self,action="add"):
752 for site_spec in self.plc_spec['sites']:
753 test_site = TestSite (self,site_spec)
755 utils.header("Deleting nodes in site %s"%test_site.name())
756 for node_spec in site_spec['nodes']:
757 test_node=TestNode(self,test_site,node_spec)
758 utils.header("Deleting %s"%test_node.name())
759 test_node.delete_node()
761 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
762 for node_spec in site_spec['nodes']:
763 utils.pprint('Creating node %s'%node_spec,node_spec)
764 test_node = TestNode (self,test_site,node_spec)
765 test_node.create_node ()
768 def nodegroups (self):
769 "create nodegroups with PLCAPI"
770 return self.do_nodegroups("add")
771 def delete_nodegroups (self):
772 "delete nodegroups with PLCAPI"
773 return self.do_nodegroups("delete")
777 def translate_timestamp (start,grain,timestamp):
778 if timestamp < TestPlc.YEAR: return start+timestamp*grain
779 else: return timestamp
782 def timestamp_printable (timestamp):
783 return time.strftime('%m-%d %H:%M:%S UTC',time.gmtime(timestamp))
786 "create leases (on reservable nodes only, use e.g. run -c default -c resa)"
788 grain=self.apiserver.GetLeaseGranularity(self.auth_root())
789 print 'API answered grain=',grain
790 start=(now/grain)*grain
792 # find out all nodes that are reservable
793 nodes=self.all_reservable_nodenames()
795 utils.header ("No reservable node found - proceeding without leases")
798 # attach them to the leases as specified in plc_specs
799 # this is where the 'leases' field gets interpreted as relative of absolute
800 for lease_spec in self.plc_spec['leases']:
801 # skip the ones that come with a null slice id
802 if not lease_spec['slice']: continue
803 lease_spec['t_from']=TestPlc.translate_timestamp(start,grain,lease_spec['t_from'])
804 lease_spec['t_until']=TestPlc.translate_timestamp(start,grain,lease_spec['t_until'])
805 lease_addition=self.apiserver.AddLeases(self.auth_root(),nodes,
806 lease_spec['slice'],lease_spec['t_from'],lease_spec['t_until'])
807 if lease_addition['errors']:
808 utils.header("Cannot create leases, %s"%lease_addition['errors'])
811 utils.header('Leases on nodes %r for %s from %d (%s) until %d (%s)'%\
812 (nodes,lease_spec['slice'],
813 lease_spec['t_from'],TestPlc.timestamp_printable(lease_spec['t_from']),
814 lease_spec['t_until'],TestPlc.timestamp_printable(lease_spec['t_until'])))
818 def delete_leases (self):
819 "remove all leases in the myplc side"
820 lease_ids= [ l['lease_id'] for l in self.apiserver.GetLeases(self.auth_root())]
821 utils.header("Cleaning leases %r"%lease_ids)
822 self.apiserver.DeleteLeases(self.auth_root(),lease_ids)
825 def list_leases (self):
826 "list all leases known to the myplc"
827 leases = self.apiserver.GetLeases(self.auth_root())
830 current=l['t_until']>=now
831 if self.options.verbose or current:
832 utils.header("%s %s from %s until %s"%(l['hostname'],l['name'],
833 TestPlc.timestamp_printable(l['t_from']),
834 TestPlc.timestamp_printable(l['t_until'])))
837 # create nodegroups if needed, and populate
838 def do_nodegroups (self, action="add"):
839 # 1st pass to scan contents
841 for site_spec in self.plc_spec['sites']:
842 test_site = TestSite (self,site_spec)
843 for node_spec in site_spec['nodes']:
844 test_node=TestNode (self,test_site,node_spec)
845 if node_spec.has_key('nodegroups'):
846 nodegroupnames=node_spec['nodegroups']
847 if isinstance(nodegroupnames,StringTypes):
848 nodegroupnames = [ nodegroupnames ]
849 for nodegroupname in nodegroupnames:
850 if not groups_dict.has_key(nodegroupname):
851 groups_dict[nodegroupname]=[]
852 groups_dict[nodegroupname].append(test_node.name())
853 auth=self.auth_root()
855 for (nodegroupname,group_nodes) in groups_dict.iteritems():
857 print 'nodegroups:','dealing with nodegroup',nodegroupname,'on nodes',group_nodes
858 # first, check if the nodetagtype is here
859 tag_types = self.apiserver.GetTagTypes(auth,{'tagname':nodegroupname})
861 tag_type_id = tag_types[0]['tag_type_id']
863 tag_type_id = self.apiserver.AddTagType(auth,
864 {'tagname':nodegroupname,
865 'description': 'for nodegroup %s'%nodegroupname,
867 print 'located tag (type)',nodegroupname,'as',tag_type_id
869 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
871 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
872 print 'created nodegroup',nodegroupname,'from tagname',nodegroupname,'and value','yes'
873 # set node tag on all nodes, value='yes'
874 for nodename in group_nodes:
876 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
878 traceback.print_exc()
879 print 'node',nodename,'seems to already have tag',nodegroupname
882 expect_yes = self.apiserver.GetNodeTags(auth,
883 {'hostname':nodename,
884 'tagname':nodegroupname},
885 ['value'])[0]['value']
886 if expect_yes != "yes":
887 print 'Mismatch node tag on node',nodename,'got',expect_yes
890 if not self.options.dry_run:
891 print 'Cannot find tag',nodegroupname,'on node',nodename
895 print 'cleaning nodegroup',nodegroupname
896 self.apiserver.DeleteNodeGroup(auth,nodegroupname)
898 traceback.print_exc()
902 # a list of TestNode objs
903 def all_nodes (self):
905 for site_spec in self.plc_spec['sites']:
906 test_site = TestSite (self,site_spec)
907 for node_spec in site_spec['nodes']:
908 nodes.append(TestNode (self,test_site,node_spec))
911 # return a list of tuples (nodename,qemuname)
912 def all_node_infos (self) :
914 for site_spec in self.plc_spec['sites']:
915 node_infos += [ (node_spec['node_fields']['hostname'],node_spec['host_box']) \
916 for node_spec in site_spec['nodes'] ]
919 def all_nodenames (self): return [ x[0] for x in self.all_node_infos() ]
920 def all_reservable_nodenames (self):
922 for site_spec in self.plc_spec['sites']:
923 for node_spec in site_spec['nodes']:
924 node_fields=node_spec['node_fields']
925 if 'node_type' in node_fields and node_fields['node_type']=='reservable':
926 res.append(node_fields['hostname'])
929 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
930 def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period_seconds=15):
931 if self.options.dry_run:
935 class CompleterTaskBootState (CompleterTask):
936 def __init__ (self, test_plc,hostname):
937 self.test_plc=test_plc
938 self.hostname=hostname
939 self.last_boot_state='undef'
940 def actual_run (self):
942 node = self.test_plc.apiserver.GetNodes(self.test_plc.auth_root(), [ self.hostname ],
944 self.last_boot_state = node['boot_state']
945 return self.last_boot_state == target_boot_state
949 return "CompleterTaskBootState with node %s"%self.hostname
950 def failure_message (self):
951 return "node %s in state %s - expected %s"%(self.hostname,self.last_boot_state,target_boot_state)
953 timeout = timedelta(minutes=timeout_minutes)
954 graceout = timedelta(minutes=silent_minutes)
955 period = timedelta(seconds=period_seconds)
956 # the nodes that haven't checked yet - start with a full list and shrink over time
957 utils.header("checking nodes boot state (expected %s)"%target_boot_state)
958 tasks = [ CompleterTaskBootState (self,hostname) \
959 for (hostname,_) in self.all_node_infos() ]
960 return Completer (tasks).run (timeout, graceout, period)
962 def nodes_booted(self):
963 return self.nodes_check_boot_state('boot',timeout_minutes=30,silent_minutes=28)
965 def check_nodes_ssh(self,debug,timeout_minutes,silent_minutes,period_seconds=15):
966 class CompleterTaskNodeSsh (CompleterTask):
967 def __init__ (self, hostname, qemuname, boot_state, local_key):
968 self.hostname=hostname
969 self.qemuname=qemuname
970 self.boot_state=boot_state
971 self.local_key=local_key
972 def run (self, silent):
973 command = TestSsh (self.hostname,key=self.local_key).actual_command("hostname;uname -a")
974 return utils.system (command, silent=silent)==0
975 def failure_message (self):
976 return "Cannot reach %s @ %s in %s mode"%(self.hostname, self.qemuname, self.boot_state)
979 timeout = timedelta(minutes=timeout_minutes)
980 graceout = timedelta(minutes=silent_minutes)
981 period = timedelta(seconds=period_seconds)
982 vservername=self.vservername
985 local_key = "keys/%(vservername)s-debug.rsa"%locals()
988 local_key = "keys/key_admin.rsa"
989 utils.header("checking ssh access to nodes (expected in %s mode)"%message)
990 node_infos = self.all_node_infos()
991 tasks = [ CompleterTaskNodeSsh (nodename, qemuname, message, local_key) \
992 for (nodename,qemuname) in node_infos ]
993 return Completer (tasks).run (timeout, graceout, period)
995 def ssh_node_debug(self):
996 "Tries to ssh into nodes in debug mode with the debug ssh key"
997 return self.check_nodes_ssh(debug=True,
998 timeout_minutes=self.ssh_node_debug_timeout,
999 silent_minutes=self.ssh_node_debug_silent)
1001 def ssh_node_boot(self):
1002 "Tries to ssh into nodes in production mode with the root ssh key"
1003 return self.check_nodes_ssh(debug=False,
1004 timeout_minutes=self.ssh_node_boot_timeout,
1005 silent_minutes=self.ssh_node_boot_silent)
1007 def node_bmlogs(self):
1008 "Checks that there's a non-empty dir. /var/log/bm/raw"
1009 return utils.system(self.actual_command_in_guest("ls /var/log/bm/raw"))==0
1012 def qemu_local_init (self): pass
1014 def bootcd (self): pass
1016 def qemu_local_config (self): pass
1018 def nodestate_reinstall (self): pass
1020 def nodestate_safeboot (self): pass
1022 def nodestate_boot (self): pass
1024 def nodestate_show (self): pass
1026 def qemu_export (self): pass
1028 ### check hooks : invoke scripts from hooks/{node,slice}
1029 def check_hooks_node (self):
1030 return self.locate_first_node().check_hooks()
1031 def check_hooks_sliver (self) :
1032 return self.locate_first_sliver().check_hooks()
1034 def check_hooks (self):
1035 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
1036 return self.check_hooks_node() and self.check_hooks_sliver()
1039 def do_check_initscripts(self):
1040 class CompleterTaskInitscript (CompleterTask):
1041 def __init__ (self, test_sliver, stamp):
1042 self.test_sliver=test_sliver
1044 def actual_run (self):
1045 return self.test_sliver.check_initscript_stamp (self.stamp)
1047 return "initscript checker for %s"%self.test_sliver.name()
1048 def failure_message (self):
1049 return "initscript stamp %s not found in sliver %s"%(self.stamp,self.test_sliver.name())
1052 for slice_spec in self.plc_spec['slices']:
1053 if not slice_spec.has_key('initscriptstamp'):
1055 stamp=slice_spec['initscriptstamp']
1056 slicename=slice_spec['slice_fields']['name']
1057 for nodename in slice_spec['nodenames']:
1058 print 'nodename',nodename,'slicename',slicename,'stamp',stamp
1059 (site,node) = self.locate_node (nodename)
1060 # xxx - passing the wrong site - probably harmless
1061 test_site = TestSite (self,site)
1062 test_slice = TestSlice (self,test_site,slice_spec)
1063 test_node = TestNode (self,test_site,node)
1064 test_sliver = TestSliver (self, test_node, test_slice)
1065 tasks.append ( CompleterTaskInitscript (test_sliver, stamp))
1066 return Completer (tasks).run (timedelta(minutes=5), timedelta(minutes=4), timedelta(seconds=10))
1068 def check_initscripts(self):
1069 "check that the initscripts have triggered"
1070 return self.do_check_initscripts()
1072 def initscripts (self):
1073 "create initscripts with PLCAPI"
1074 for initscript in self.plc_spec['initscripts']:
1075 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
1076 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
1079 def delete_initscripts (self):
1080 "delete initscripts with PLCAPI"
1081 for initscript in self.plc_spec['initscripts']:
1082 initscript_name = initscript['initscript_fields']['name']
1083 print('Attempting to delete %s in plc %s'%(initscript_name,self.plc_spec['name']))
1085 self.apiserver.DeleteInitScript(self.auth_root(),initscript_name)
1086 print initscript_name,'deleted'
1088 print 'deletion went wrong - probably did not exist'
1093 "create slices with PLCAPI"
1094 return self.do_slices(action="add")
1096 def delete_slices (self):
1097 "delete slices with PLCAPI"
1098 return self.do_slices(action="delete")
1100 def fill_slices (self):
1101 "add nodes in slices with PLCAPI"
1102 return self.do_slices(action="fill")
1104 def empty_slices (self):
1105 "remove nodes from slices with PLCAPI"
1106 return self.do_slices(action="empty")
1108 def do_slices (self, action="add"):
1109 for slice in self.plc_spec['slices']:
1110 site_spec = self.locate_site (slice['sitename'])
1111 test_site = TestSite(self,site_spec)
1112 test_slice=TestSlice(self,test_site,slice)
1113 if action == "delete":
1114 test_slice.delete_slice()
1115 elif action=="fill":
1116 test_slice.add_nodes()
1117 elif action=="empty":
1118 test_slice.delete_nodes()
1120 test_slice.create_slice()
1124 def ssh_slice(self): pass
1126 def ssh_slice_off (self): pass
1128 def ssh_slice_basics(self): pass
1131 def check_vsys_defaults(self): pass
1134 def keys_clear_known_hosts (self): pass
1136 def plcapi_urls (self):
1137 return PlcapiUrlScanner (self.auth_root(),ip=self.vserverip).scan()
1139 def speed_up_slices (self):
1140 "tweak nodemanager settings on all nodes using a conf file"
1141 # create the template on the server-side
1142 template="%s.nodemanager"%self.name()
1143 template_file = open (template,"w")
1144 template_file.write('OPTIONS="-p 30 -r 11 -d"\n')
1145 template_file.close()
1146 in_vm="/var/www/html/PlanetLabConf/nodemanager"
1147 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1148 self.test_ssh.copy_abs(template,remote)
1150 self.apiserver.AddConfFile (self.auth_root(),
1151 {'dest':'/etc/sysconfig/nodemanager',
1152 'source':'PlanetLabConf/nodemanager',
1153 'postinstall_cmd':'service nm restart',})
1156 def debug_nodemanager (self):
1157 "sets verbose mode for nodemanager, and speeds up cycle even more (needs speed_up_slices first)"
1158 template="%s.nodemanager"%self.name()
1159 template_file = open (template,"w")
1160 template_file.write('OPTIONS="-p 10 -r 6 -v -d"\n')
1161 template_file.close()
1162 in_vm="/var/www/html/PlanetLabConf/nodemanager"
1163 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1164 self.test_ssh.copy_abs(template,remote)
1168 def qemu_start (self) : pass
1171 def timestamp_qemu (self) : pass
1173 # when a spec refers to a node possibly on another plc
1174 def locate_sliver_obj_cross (self, nodename, slicename, other_plcs):
1175 for plc in [ self ] + other_plcs:
1177 return plc.locate_sliver_obj (nodename, slicename)
1180 raise Exception, "Cannot locate sliver %s@%s among all PLCs"%(nodename,slicename)
1182 # implement this one as a cross step so that we can take advantage of different nodes
1183 # in multi-plcs mode
1184 def cross_check_tcp (self, other_plcs):
1185 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
1186 if 'tcp_specs' not in self.plc_spec or not self.plc_spec['tcp_specs']:
1187 utils.header ("check_tcp: no/empty config found")
1189 specs = self.plc_spec['tcp_specs']
1194 s_test_sliver = self.locate_sliver_obj_cross (spec['server_node'],spec['server_slice'],other_plcs)
1195 if not s_test_sliver.run_tcp_server(port,timeout=20):
1199 # idem for the client side
1200 c_test_sliver = self.locate_sliver_obj_cross (spec['client_node'],spec['client_slice'],other_plcs)
1201 # use nodename from locatesd sliver, unless 'client_connect' is set
1202 if 'client_connect' in spec:
1203 destination = spec['client_connect']
1205 destination=s_test_sliver.test_node.name()
1206 if not c_test_sliver.run_tcp_client(destination,port):
1210 # painfully enough, we need to allow for some time as netflow might show up last
1211 def check_system_slice (self):
1212 "all nodes: check that a system slice is alive"
1213 # netflow currently not working in the lxc distro
1214 # drl not built at all in the wtx distro
1215 # if we find either of them we're happy
1216 return self.check_netflow() or self.check_drl()
1219 def check_netflow (self): return self._check_system_slice ('netflow')
1220 def check_drl (self): return self._check_system_slice ('drl')
1222 # we have the slices up already here, so it should not take too long
1223 def _check_system_slice (self, slicename, timeout_minutes=5, period_seconds=15):
1224 class CompleterTaskSystemSlice (CompleterTask):
1225 def __init__ (self, test_node, dry_run):
1226 self.test_node=test_node
1227 self.dry_run=dry_run
1228 def actual_run (self):
1229 return self.test_node._check_system_slice (slicename, dry_run=self.dry_run)
1231 return "System slice %s @ %s"%(slicename, self.test_node.name())
1232 def failure_message (self):
1233 return "COULD not find system slice %s @ %s"%(slicename, self.test_node.name())
1234 timeout = timedelta(minutes=timeout_minutes)
1235 silent = timedelta (0)
1236 period = timedelta (seconds=period_seconds)
1237 tasks = [ CompleterTaskSystemSlice (test_node, self.options.dry_run) \
1238 for test_node in self.all_nodes() ]
1239 return Completer (tasks) . run (timeout, silent, period)
1241 def plcsh_stress_test (self):
1242 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
1243 # install the stress-test in the plc image
1244 location = "/usr/share/plc_api/plcsh_stress_test.py"
1245 remote="%s/%s"%(self.vm_root_in_host(),location)
1246 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1248 command += " -- --check"
1249 if self.options.size == 1:
1250 command += " --tiny"
1251 return ( self.run_in_guest(command) == 0)
1253 # populate runs the same utility without slightly different options
1254 # in particular runs with --preserve (dont cleanup) and without --check
1255 # also it gets run twice, once with the --foreign option for creating fake foreign entries
1257 def sfa_install_all (self):
1258 "yum install sfa sfa-plc sfa-sfatables sfa-client"
1259 return self.yum_install ("sfa sfa-plc sfa-sfatables sfa-client")
1261 def sfa_install_core(self):
1263 return self.yum_install ("sfa")
1265 def sfa_install_plc(self):
1266 "yum install sfa-plc"
1267 return self.yum_install("sfa-plc")
1269 def sfa_install_sfatables(self):
1270 "yum install sfa-sfatables"
1271 return self.yum_install ("sfa-sfatables")
1273 # for some very odd reason, this sometimes fails with the following symptom
1274 # # yum install sfa-client
1275 # Setting up Install Process
1277 # Downloading Packages:
1278 # Running rpm_check_debug
1279 # Running Transaction Test
1280 # Transaction Test Succeeded
1281 # Running Transaction
1282 # Transaction couldn't start:
1283 # installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem
1284 # [('installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem', (9, '/', 69632L))]
1285 # even though in the same context I have
1286 # [2012.05.23--f14-32-sfastd1-1-vplc07] / # df -h
1287 # Filesystem Size Used Avail Use% Mounted on
1288 # /dev/hdv1 806G 264G 501G 35% /
1289 # none 16M 36K 16M 1% /tmp
1291 # so as a workaround, we first try yum install, and then invoke rpm on the cached rpm...
1292 def sfa_install_client(self):
1293 "yum install sfa-client"
1294 first_try=self.yum_install("sfa-client")
1295 if first_try: return True
1296 utils.header ("********** Regular yum failed - special workaround in place, 2nd chance")
1297 (code,cached_rpm_path)=utils.output_of(self.actual_command_in_guest('find /var/cache/yum -name sfa-client\*.rpm'))
1298 utils.header("rpm_path=<<%s>>"%rpm_path)
1300 self.run_in_guest("rpm -i %s"%cached_rpm_path)
1301 return self.yum_check_installed ("sfa-client")
1303 def sfa_dbclean(self):
1304 "thoroughly wipes off the SFA database"
1305 return self.run_in_guest("sfaadmin reg nuke")==0 or \
1306 self.run_in_guest("sfa-nuke.py")==0 or \
1307 self.run_in_guest("sfa-nuke-plc.py")==0
1309 def sfa_fsclean(self):
1310 "cleanup /etc/sfa/trusted_roots and /var/lib/sfa"
1311 self.run_in_guest("rm -rf /etc/sfa/trusted_roots /var/lib/sfa/authorities")
1314 def sfa_plcclean(self):
1315 "cleans the PLC entries that were created as a side effect of running the script"
1317 sfa_spec=self.plc_spec['sfa']
1319 for auth_sfa_spec in sfa_spec['auth_sfa_specs']:
1320 login_base=auth_sfa_spec['login_base']
1321 try: self.apiserver.DeleteSite (self.auth_root(),login_base)
1322 except: print "Site %s already absent from PLC db"%login_base
1324 for spec_name in ['pi_spec','user_spec']:
1325 user_spec=auth_sfa_spec[spec_name]
1326 username=user_spec['email']
1327 try: self.apiserver.DeletePerson(self.auth_root(),username)
1329 # this in fact is expected as sites delete their members
1330 #print "User %s already absent from PLC db"%username
1333 print "REMEMBER TO RUN sfa_import AGAIN"
1336 def sfa_uninstall(self):
1337 "uses rpm to uninstall sfa - ignore result"
1338 self.run_in_guest("rpm -e sfa sfa-sfatables sfa-client sfa-plc")
1339 self.run_in_guest("rm -rf /var/lib/sfa")
1340 self.run_in_guest("rm -rf /etc/sfa")
1341 self.run_in_guest("rm -rf /var/log/sfa_access.log /var/log/sfa_import_plc.log /var/log/sfa.daemon")
1343 self.run_in_guest("rpm -e --noscripts sfa-plc")
1346 ### run unit tests for SFA
1347 # NOTE: for some reason on f14/i386, yum install sfa-tests fails for no reason
1348 # Running Transaction
1349 # Transaction couldn't start:
1350 # installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem
1351 # [('installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem', (9, '/', 208896L))]
1352 # no matter how many Gbs are available on the testplc
1353 # could not figure out what's wrong, so...
1354 # if the yum install phase fails, consider the test is successful
1355 # other combinations will eventually run it hopefully
1356 def sfa_utest(self):
1357 "yum install sfa-tests and run SFA unittests"
1358 self.run_in_guest("yum -y install sfa-tests")
1359 # failed to install - forget it
1360 if self.run_in_guest("rpm -q sfa-tests")!=0:
1361 utils.header("WARNING: SFA unit tests failed to install, ignoring")
1363 return self.run_in_guest("/usr/share/sfa/tests/testAll.py")==0
1367 dirname="conf.%s"%self.plc_spec['name']
1368 if not os.path.isdir(dirname):
1369 utils.system("mkdir -p %s"%dirname)
1370 if not os.path.isdir(dirname):
1371 raise Exception,"Cannot create config dir for plc %s"%self.name()
1374 def conffile(self,filename):
1375 return "%s/%s"%(self.confdir(),filename)
1376 def confsubdir(self,dirname,clean,dry_run=False):
1377 subdirname="%s/%s"%(self.confdir(),dirname)
1379 utils.system("rm -rf %s"%subdirname)
1380 if not os.path.isdir(subdirname):
1381 utils.system("mkdir -p %s"%subdirname)
1382 if not dry_run and not os.path.isdir(subdirname):
1383 raise "Cannot create config subdir %s for plc %s"%(dirname,self.name())
1386 def conffile_clean (self,filename):
1387 filename=self.conffile(filename)
1388 return utils.system("rm -rf %s"%filename)==0
1391 def sfa_configure(self):
1392 "run sfa-config-tty"
1393 tmpname=self.conffile("sfa-config-tty")
1394 fileconf=open(tmpname,'w')
1395 for var in [ 'SFA_REGISTRY_ROOT_AUTH',
1396 'SFA_INTERFACE_HRN',
1397 'SFA_REGISTRY_LEVEL1_AUTH',
1398 'SFA_REGISTRY_HOST',
1399 'SFA_AGGREGATE_HOST',
1409 'SFA_GENERIC_FLAVOUR',
1410 'SFA_AGGREGATE_ENABLED',
1412 if self.plc_spec['sfa'].has_key(var):
1413 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec['sfa'][var]))
1414 # the way plc_config handles booleans just sucks..
1417 if self.plc_spec['sfa'][var]: val='true'
1418 fileconf.write ('e %s\n%s\n'%(var,val))
1419 fileconf.write('w\n')
1420 fileconf.write('R\n')
1421 fileconf.write('q\n')
1423 utils.system('cat %s'%tmpname)
1424 self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
1427 def aggregate_xml_line(self):
1428 port=self.plc_spec['sfa']['neighbours-port']
1429 return '<aggregate addr="%s" hrn="%s" port="%r"/>' % \
1430 (self.vserverip,self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH'],port)
1432 def registry_xml_line(self):
1433 return '<registry addr="%s" hrn="%s" port="12345"/>' % \
1434 (self.vserverip,self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH'])
1437 # a cross step that takes all other plcs in argument
1438 def cross_sfa_configure(self, other_plcs):
1439 "writes aggregates.xml and registries.xml that point to all other PLCs in the test"
1440 # of course with a single plc, other_plcs is an empty list
1443 agg_fname=self.conffile("agg.xml")
1444 file(agg_fname,"w").write("<aggregates>%s</aggregates>\n" % \
1445 " ".join([ plc.aggregate_xml_line() for plc in other_plcs ]))
1446 utils.header ("(Over)wrote %s"%agg_fname)
1447 reg_fname=self.conffile("reg.xml")
1448 file(reg_fname,"w").write("<registries>%s</registries>\n" % \
1449 " ".join([ plc.registry_xml_line() for plc in other_plcs ]))
1450 utils.header ("(Over)wrote %s"%reg_fname)
1451 return self.test_ssh.copy_abs(agg_fname,'/%s/etc/sfa/aggregates.xml'%self.vm_root_in_host())==0 \
1452 and self.test_ssh.copy_abs(reg_fname,'/%s/etc/sfa/registries.xml'%self.vm_root_in_host())==0
1454 def sfa_import(self):
1455 "use sfaadmin to import from plc"
1456 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1458 self.run_in_guest('sfaadmin reg import_registry')==0
1459 # not needed anymore
1460 # self.run_in_guest('cp /etc/sfa/authorities/%s/%s.pkey /etc/sfa/authorities/server.key'%(auth,auth))
1462 def sfa_start(self):
1464 return self.run_in_guest('service sfa start')==0
1466 def sfi_configure(self):
1467 "Create /root/sfi on the plc side for sfi client configuration"
1468 if self.options.dry_run:
1469 utils.header("DRY RUN - skipping step")
1471 sfa_spec=self.plc_spec['sfa']
1472 # cannot use auth_sfa_mapper to pass dir_name
1473 for slice_spec in self.plc_spec['sfa']['auth_sfa_specs']:
1474 test_slice=TestAuthSfa(self,slice_spec)
1475 dir_basename=os.path.basename(test_slice.sfi_path())
1476 dir_name=self.confsubdir("dot-sfi/%s"%dir_basename,clean=True,dry_run=self.options.dry_run)
1477 test_slice.sfi_configure(dir_name)
1478 # push into the remote /root/sfi area
1479 location = test_slice.sfi_path()
1480 remote="%s/%s"%(self.vm_root_in_host(),location)
1481 self.test_ssh.mkdir(remote,abs=True)
1482 # need to strip last level or remote otherwise we get an extra dir level
1483 self.test_ssh.copy_abs(dir_name, os.path.dirname(remote), recursive=True)
1487 def sfi_clean (self):
1488 "clean up /root/sfi on the plc side"
1489 self.run_in_guest("rm -rf /root/sfi")
1493 def sfa_add_site (self): pass
1495 def sfa_add_pi (self): pass
1497 def sfa_add_user(self): pass
1499 def sfa_update_user(self): pass
1501 def sfa_add_slice(self): pass
1503 def sfa_renew_slice(self): pass
1505 def sfa_discover(self): pass
1507 def sfa_create_slice(self): pass
1509 def sfa_check_slice_plc(self): pass
1511 def sfa_update_slice(self): pass
1513 def sfi_list(self): pass
1515 def sfi_show(self): pass
1517 def sfi_slices(self): pass
1519 def ssh_slice_sfa(self): pass
1521 def sfa_delete_user(self): pass
1523 def sfa_delete_slice(self): pass
1527 self.run_in_guest('service sfa stop')==0
1530 def populate (self):
1531 "creates random entries in the PLCAPI"
1532 # install the stress-test in the plc image
1533 location = "/usr/share/plc_api/plcsh_stress_test.py"
1534 remote="%s/%s"%(self.vm_root_in_host(),location)
1535 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1537 command += " -- --preserve --short-names"
1538 local = (self.run_in_guest(command) == 0);
1539 # second run with --foreign
1540 command += ' --foreign'
1541 remote = (self.run_in_guest(command) == 0);
1542 return ( local and remote)
1544 def gather_logs (self):
1545 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1546 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1547 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1548 # (1.c) get the plc's /root/sfi -> logs/sfi.<plcname>/
1549 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1550 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1551 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1553 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1554 self.gather_var_logs ()
1556 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1557 self.gather_pgsql_logs ()
1559 print "-------------------- TestPlc.gather_logs : PLC's /root/sfi/"
1560 self.gather_root_sfi ()
1562 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1563 for site_spec in self.plc_spec['sites']:
1564 test_site = TestSite (self,site_spec)
1565 for node_spec in site_spec['nodes']:
1566 test_node=TestNode(self,test_site,node_spec)
1567 test_node.gather_qemu_logs()
1569 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1570 self.gather_nodes_var_logs()
1572 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1573 self.gather_slivers_var_logs()
1576 def gather_slivers_var_logs(self):
1577 for test_sliver in self.all_sliver_objs():
1578 remote = test_sliver.tar_var_logs()
1579 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
1580 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
1581 utils.system(command)
1584 def gather_var_logs (self):
1585 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
1586 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1587 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
1588 utils.system(command)
1589 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd"%self.name()
1590 utils.system(command)
1592 def gather_pgsql_logs (self):
1593 utils.system("mkdir -p logs/myplc.pgsql-log.%s"%self.name())
1594 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1595 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -"%self.name()
1596 utils.system(command)
1598 def gather_root_sfi (self):
1599 utils.system("mkdir -p logs/sfi.%s"%self.name())
1600 to_plc = self.actual_command_in_guest("tar -C /root/sfi/ -cf - .")
1601 command = to_plc + "| tar -C logs/sfi.%s -xf -"%self.name()
1602 utils.system(command)
1604 def gather_nodes_var_logs (self):
1605 for site_spec in self.plc_spec['sites']:
1606 test_site = TestSite (self,site_spec)
1607 for node_spec in site_spec['nodes']:
1608 test_node=TestNode(self,test_site,node_spec)
1609 test_ssh = TestSsh (test_node.name(),key="keys/key_admin.rsa")
1610 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1611 command = command + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
1612 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
1613 utils.system(command)
1616 # returns the filename to use for sql dump/restore, using options.dbname if set
1617 def dbfile (self, database):
1618 # uses options.dbname if it is found
1620 name=self.options.dbname
1621 if not isinstance(name,StringTypes):
1627 return "/root/%s-%s.sql"%(database,name)
1629 def plc_db_dump(self):
1630 'dump the planetlab5 DB in /root in the PLC - filename has time'
1631 dump=self.dbfile("planetab5")
1632 self.run_in_guest('pg_dump -U pgsqluser planetlab5 -f '+ dump)
1633 utils.header('Dumped planetlab5 database in %s'%dump)
1636 def plc_db_restore(self):
1637 'restore the planetlab5 DB - looks broken, but run -n might help'
1638 dump=self.dbfile("planetab5")
1639 ##stop httpd service
1640 self.run_in_guest('service httpd stop')
1641 # xxx - need another wrapper
1642 self.run_in_guest_piped('echo drop database planetlab5','psql --user=pgsqluser template1')
1643 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab5')
1644 self.run_in_guest('psql -U pgsqluser planetlab5 -f '+dump)
1645 ##starting httpd service
1646 self.run_in_guest('service httpd start')
1648 utils.header('Database restored from ' + dump)
1650 def standby_1_through_20(self):
1651 """convenience function to wait for a specified number of minutes"""
1654 def standby_1(): pass
1656 def standby_2(): pass
1658 def standby_3(): pass
1660 def standby_4(): pass
1662 def standby_5(): pass
1664 def standby_6(): pass
1666 def standby_7(): pass
1668 def standby_8(): pass
1670 def standby_9(): pass
1672 def standby_10(): pass
1674 def standby_11(): pass
1676 def standby_12(): pass
1678 def standby_13(): pass
1680 def standby_14(): pass
1682 def standby_15(): pass
1684 def standby_16(): pass
1686 def standby_17(): pass
1688 def standby_18(): pass
1690 def standby_19(): pass
1692 def standby_20(): pass