1 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
2 # Copyright (C) 2010 INRIA
9 from datetime import datetime, timedelta
10 from types import StringTypes
13 from TestSite import TestSite
14 from TestNode import TestNode
15 from TestUser import TestUser
16 from TestKey import TestKey
17 from TestSlice import TestSlice
18 from TestSliver import TestSliver
19 from TestBoxQemu import TestBoxQemu
20 from TestSsh import TestSsh
21 from TestApiserver import TestApiserver
22 from TestAuthSfa import TestAuthSfa
23 from PlcapiUrlScanner import PlcapiUrlScanner
24 from Completer import Completer, CompleterTask
26 # step methods must take (self) and return a boolean (options is a member of the class)
28 def standby(minutes,dry_run):
29 utils.header('Entering StandBy for %d mn'%minutes)
33 time.sleep(60*minutes)
36 def standby_generic (func):
38 minutes=int(func.__name__.split("_")[1])
39 return standby(minutes,self.options.dry_run)
42 def node_mapper (method):
43 def actual(self,*args, **kwds):
45 node_method = TestNode.__dict__[method.__name__]
46 for test_node in self.all_nodes():
47 if not node_method(test_node, *args, **kwds): overall=False
49 # restore the doc text
50 actual.__doc__=TestNode.__dict__[method.__name__].__doc__
53 def slice_mapper (method):
56 slice_method = TestSlice.__dict__[method.__name__]
57 for slice_spec in self.plc_spec['slices']:
58 site_spec = self.locate_site (slice_spec['sitename'])
59 test_site = TestSite(self,site_spec)
60 test_slice=TestSlice(self,test_site,slice_spec)
61 if not slice_method(test_slice,self.options): overall=False
63 # restore the doc text
64 actual.__doc__=TestSlice.__dict__[method.__name__].__doc__
67 def auth_sfa_mapper (method):
70 auth_method = TestAuthSfa.__dict__[method.__name__]
71 for auth_spec in self.plc_spec['sfa']['auth_sfa_specs']:
72 test_auth=TestAuthSfa(self,auth_spec)
73 if not auth_method(test_auth,self.options): overall=False
75 # restore the doc text
76 actual.__doc__=TestAuthSfa.__dict__[method.__name__].__doc__
86 'vs_delete','timestamp_vs','vs_create', SEP,
87 # 'plc_install', 'mod_python', 'plc_configure', 'plc_start', SEP,
88 'plc_install', 'plc_configure', 'plc_start', SEP,
89 'keys_fetch', 'keys_store', 'keys_clear_known_hosts', SEP,
90 'plcapi_urls','speed_up_slices', SEP,
91 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', 'leases', SEP,
92 # slices created under plcsh interactively seem to be fine but these ones don't have the tags
93 # keep this our of the way for now
94 # 'check_vsys_defaults', SEP,
95 'nodestate_reinstall', 'qemu_local_init','bootcd', 'qemu_local_config', SEP,
96 'qemu_kill_mine','qemu_clean_mine', 'qemu_export', 'qemu_start', 'timestamp_qemu', SEP,
97 'sfa_install_all', 'sfa_configure', 'cross_sfa_configure', 'sfa_start', 'sfa_import', SEPSFA,
98 'sfi_configure@1', 'sfa_add_site@1','sfa_add_pi@1', SEPSFA,
99 'sfa_add_user@1', 'sfa_update_user@1', 'sfa_add_slice@1', 'sfa_renew_slice@1', SEPSFA,
100 'sfa_discover@1', 'sfa_create_slice@1', 'sfa_check_slice_plc@1', 'sfa_update_slice@1', SEPSFA,
101 'sfi_list@1', 'sfi_show@1', 'sfa_utest@1', SEPSFA,
102 # we used to run plcsh_stress_test, and then ssh_node_debug and ssh_node_boot
103 # but as the stress test might take a while, we sometimes missed the debug mode..
104 'ssh_node_debug@1', 'plcsh_stress_test@1', SEP,
105 'ssh_node_boot@1', 'node_bmlogs@1', 'ssh_slice', 'ssh_slice_basics', 'check_initscripts', SEP,
106 'ssh_slice_sfa@1', 'sfa_delete_slice@1', 'sfa_delete_user@1', SEPSFA,
107 'cross_check_tcp@1', 'check_system_slice', SEP,
108 'empty_slices', 'ssh_slice_off', 'fill_slices', SEP,
109 'force_gather_logs', SEP,
112 'export', 'show_boxes', SEP,
113 'check_hooks', 'plc_stop', 'vs_start', 'vs_stop', SEP,
114 'delete_initscripts', 'delete_nodegroups','delete_all_sites', SEP,
115 'delete_sites', 'delete_nodes', 'delete_slices', 'keys_clean', SEP,
116 'delete_leases', 'list_leases', SEP,
118 'nodestate_show','nodestate_safeboot','nodestate_boot', SEP,
119 'qemu_list_all', 'qemu_list_mine', 'qemu_kill_all', SEP,
120 'sfa_install_core', 'sfa_install_sfatables', 'sfa_install_plc', 'sfa_install_client', SEPSFA,
121 'sfa_plcclean', 'sfa_dbclean', 'sfa_stop','sfa_uninstall', 'sfi_clean', SEPSFA,
122 'plc_db_dump' , 'plc_db_restore', SEP,
123 'check_netflow','check_drl', SEP,
124 'debug_nodemanager', SEP,
125 'standby_1_through_20',SEP,
129 def printable_steps (list):
130 single_line=" ".join(list)+" "
131 return single_line.replace(" "+SEP+" "," \\\n").replace(" "+SEPSFA+" "," \\\n")
133 def valid_step (step):
134 return step != SEP and step != SEPSFA
136 # turn off the sfa-related steps when build has skipped SFA
137 # this was originally for centos5 but is still valid
138 # for up to f12 as recent SFAs with sqlalchemy won't build before f14
140 def check_whether_build_has_sfa (rpms_url):
141 utils.header ("Checking if build provides SFA package...")
142 # warning, we're now building 'sface' so let's be a bit more picky
143 retcod=os.system ("curl --silent %s/ | grep -q sfa-"%rpms_url)
144 # full builds are expected to return with 0 here
146 utils.header("build does provide SFA")
148 # move all steps containing 'sfa' from default_steps to other_steps
149 utils.header("SFA package not found - removing steps with sfa or sfi")
150 sfa_steps= [ step for step in TestPlc.default_steps if step.find('sfa')>=0 or step.find("sfi")>=0 ]
151 TestPlc.other_steps += sfa_steps
152 for step in sfa_steps: TestPlc.default_steps.remove(step)
154 def __init__ (self,plc_spec,options):
155 self.plc_spec=plc_spec
157 self.test_ssh=TestSsh(self.plc_spec['host_box'],self.options.buildname)
158 self.vserverip=plc_spec['vserverip']
159 self.vservername=plc_spec['vservername']
160 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
161 self.apiserver=TestApiserver(self.url,options.dry_run)
162 (self.ssh_node_boot_timeout,self.ssh_node_boot_silent)=plc_spec['ssh_node_boot_timers']
163 (self.ssh_node_debug_timeout,self.ssh_node_debug_silent)=plc_spec['ssh_node_debug_timers']
165 def has_addresses_api (self):
166 return self.apiserver.has_method('AddIpAddress')
169 name=self.plc_spec['name']
170 return "%s.%s"%(name,self.vservername)
173 return self.plc_spec['host_box']
176 return self.test_ssh.is_local()
178 # define the API methods on this object through xmlrpc
179 # would help, but not strictly necessary
183 def actual_command_in_guest (self,command):
184 return self.test_ssh.actual_command(self.host_to_guest(command),dry_run=self.options.dry_run)
186 def start_guest (self):
187 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host(),dry_run=self.options.dry_run))
189 def stop_guest (self):
190 return utils.system(self.test_ssh.actual_command(self.stop_guest_in_host(),dry_run=self.options.dry_run))
192 def run_in_guest (self,command):
193 return utils.system(self.actual_command_in_guest(command))
195 def run_in_host (self,command):
196 return self.test_ssh.run_in_buildname(command, dry_run=self.options.dry_run)
198 #command gets run in the plc's vm
199 def host_to_guest(self,command):
200 if self.options.plcs_use_lxc:
201 return "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s %s"%(self.vserverip,command)
203 return "vserver %s exec %s"%(self.vservername,command)
205 def vm_root_in_host(self):
206 if self.options.plcs_use_lxc:
207 return "/vservers/%s/rootfs/"%(self.vservername)
209 return "/vservers/%s"%(self.vservername)
211 def vm_timestamp_path (self):
212 if self.options.plcs_use_lxc:
213 return "/vservers/%s/%s.timestamp"%(self.vservername,self.vservername)
215 return "/vservers/%s.timestamp"%(self.vservername)
217 #start/stop the vserver
218 def start_guest_in_host(self):
219 if self.options.plcs_use_lxc:
220 return "virsh -c lxc:// start %s"%(self.vservername)
222 return "vserver %s start"%(self.vservername)
224 def stop_guest_in_host(self):
225 if self.options.plcs_use_lxc:
226 return "virsh -c lxc:// destroy %s"%(self.vservername)
228 return "vserver %s stop"%(self.vservername)
231 def run_in_guest_piped (self,local,remote):
232 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
234 def yum_check_installed (self, rpms):
235 if isinstance (rpms, list):
237 return self.run_in_guest("rpm -q %s"%rpms)==0
239 # does a yum install in the vs, ignore yum retcod, check with rpm
240 def yum_install (self, rpms):
241 if isinstance (rpms, list):
243 self.run_in_guest("yum -y install %s"%rpms)
244 # yum-complete-transaction comes with yum-utils, that is in vtest.pkgs
245 self.run_in_guest("yum-complete-transaction -y")
246 return self.yum_check_installed (rpms)
248 def auth_root (self):
249 return {'Username':self.plc_spec['PLC_ROOT_USER'],
250 'AuthMethod':'password',
251 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
252 'Role' : self.plc_spec['role']
254 def locate_site (self,sitename):
255 for site in self.plc_spec['sites']:
256 if site['site_fields']['name'] == sitename:
258 if site['site_fields']['login_base'] == sitename:
260 raise Exception,"Cannot locate site %s"%sitename
262 def locate_node (self,nodename):
263 for site in self.plc_spec['sites']:
264 for node in site['nodes']:
265 if node['name'] == nodename:
267 raise Exception,"Cannot locate node %s"%nodename
269 def locate_hostname (self,hostname):
270 for site in self.plc_spec['sites']:
271 for node in site['nodes']:
272 if node['node_fields']['hostname'] == hostname:
274 raise Exception,"Cannot locate hostname %s"%hostname
276 def locate_key (self,key_name):
277 for key in self.plc_spec['keys']:
278 if key['key_name'] == key_name:
280 raise Exception,"Cannot locate key %s"%key_name
282 def locate_private_key_from_key_names (self, key_names):
283 # locate the first avail. key
285 for key_name in key_names:
286 key_spec=self.locate_key(key_name)
287 test_key=TestKey(self,key_spec)
288 publickey=test_key.publicpath()
289 privatekey=test_key.privatepath()
290 if os.path.isfile(publickey) and os.path.isfile(privatekey):
292 if found: return privatekey
295 def locate_slice (self, slicename):
296 for slice in self.plc_spec['slices']:
297 if slice['slice_fields']['name'] == slicename:
299 raise Exception,"Cannot locate slice %s"%slicename
301 def all_sliver_objs (self):
303 for slice_spec in self.plc_spec['slices']:
304 slicename = slice_spec['slice_fields']['name']
305 for nodename in slice_spec['nodenames']:
306 result.append(self.locate_sliver_obj (nodename,slicename))
309 def locate_sliver_obj (self,nodename,slicename):
310 (site,node) = self.locate_node(nodename)
311 slice = self.locate_slice (slicename)
313 test_site = TestSite (self, site)
314 test_node = TestNode (self, test_site,node)
315 # xxx the slice site is assumed to be the node site - mhh - probably harmless
316 test_slice = TestSlice (self, test_site, slice)
317 return TestSliver (self, test_node, test_slice)
319 def locate_first_node(self):
320 nodename=self.plc_spec['slices'][0]['nodenames'][0]
321 (site,node) = self.locate_node(nodename)
322 test_site = TestSite (self, site)
323 test_node = TestNode (self, test_site,node)
326 def locate_first_sliver (self):
327 slice_spec=self.plc_spec['slices'][0]
328 slicename=slice_spec['slice_fields']['name']
329 nodename=slice_spec['nodenames'][0]
330 return self.locate_sliver_obj(nodename,slicename)
332 # all different hostboxes used in this plc
333 def get_BoxNodes(self):
334 # maps on sites and nodes, return [ (host_box,test_node) ]
336 for site_spec in self.plc_spec['sites']:
337 test_site = TestSite (self,site_spec)
338 for node_spec in site_spec['nodes']:
339 test_node = TestNode (self, test_site, node_spec)
340 if not test_node.is_real():
341 tuples.append( (test_node.host_box(),test_node) )
342 # transform into a dict { 'host_box' -> [ test_node .. ] }
344 for (box,node) in tuples:
345 if not result.has_key(box):
348 result[box].append(node)
351 # a step for checking this stuff
352 def show_boxes (self):
353 'print summary of nodes location'
354 for (box,nodes) in self.get_BoxNodes().iteritems():
355 print box,":"," + ".join( [ node.name() for node in nodes ] )
358 # make this a valid step
359 def qemu_kill_all(self):
360 'kill all qemu instances on the qemu boxes involved by this setup'
361 # this is the brute force version, kill all qemus on that host box
362 for (box,nodes) in self.get_BoxNodes().iteritems():
363 # pass the first nodename, as we don't push template-qemu on testboxes
364 nodedir=nodes[0].nodedir()
365 TestBoxQemu(box,self.options.buildname).qemu_kill_all(nodedir)
368 # make this a valid step
369 def qemu_list_all(self):
370 'list all qemu instances on the qemu boxes involved by this setup'
371 for (box,nodes) in self.get_BoxNodes().iteritems():
372 # this is the brute force version, kill all qemus on that host box
373 TestBoxQemu(box,self.options.buildname).qemu_list_all()
376 # kill only the qemus related to this test
377 def qemu_list_mine(self):
378 'list qemu instances for our nodes'
379 for (box,nodes) in self.get_BoxNodes().iteritems():
380 # the fine-grain version
385 # kill only the qemus related to this test
386 def qemu_clean_mine(self):
387 'cleanup (rm -rf) qemu instances for our nodes'
388 for (box,nodes) in self.get_BoxNodes().iteritems():
389 # the fine-grain version
394 # kill only the right qemus
395 def qemu_kill_mine(self):
396 'kill the qemu instances for our nodes'
397 for (box,nodes) in self.get_BoxNodes().iteritems():
398 # the fine-grain version
403 #################### display config
405 "show test configuration after localization"
410 # uggly hack to make sure 'run export' only reports about the 1st plc
411 # to avoid confusion - also we use 'inri_slice1' in various aliases..
414 "print cut'n paste-able stuff to export env variables to your shell"
415 # guess local domain from hostname
416 if TestPlc.exported_id>1:
417 print "export GUESTHOSTNAME%d=%s"%(TestPlc.exported_id,self.plc_spec['vservername'])
419 TestPlc.exported_id+=1
420 domain=socket.gethostname().split('.',1)[1]
421 fqdn="%s.%s"%(self.plc_spec['host_box'],domain)
422 print "export BUILD=%s"%self.options.buildname
423 if self.options.plcs_use_lxc:
424 print "export PLCHOSTLXC=%s"%fqdn
426 print "export PLCHOSTVS=%s"%fqdn
427 print "export GUESTNAME=%s"%self.plc_spec['vservername']
428 vplcname=self.plc_spec['vservername'].split('-')[-1]
429 print "export GUESTHOSTNAME=%s.%s"%(vplcname,domain)
430 # find hostname of first node
431 (hostname,qemubox) = self.all_node_infos()[0]
432 print "export KVMHOST=%s.%s"%(qemubox,domain)
433 print "export NODE=%s"%(hostname)
437 always_display_keys=['PLC_WWW_HOST','nodes','sites',]
438 def show_pass (self,passno):
439 for (key,val) in self.plc_spec.iteritems():
440 if not self.options.verbose and key not in TestPlc.always_display_keys: continue
444 self.display_site_spec(site)
445 for node in site['nodes']:
446 self.display_node_spec(node)
447 elif key=='initscripts':
448 for initscript in val:
449 self.display_initscript_spec (initscript)
452 self.display_slice_spec (slice)
455 self.display_key_spec (key)
457 if key not in ['sites','initscripts','slices','keys', 'sfa']:
458 print '+ ',key,':',val
460 def display_site_spec (self,site):
461 print '+ ======== site',site['site_fields']['name']
462 for (k,v) in site.iteritems():
463 if not self.options.verbose and k not in TestPlc.always_display_keys: continue
466 print '+ ','nodes : ',
468 print node['node_fields']['hostname'],'',
474 print user['name'],'',
476 elif k == 'site_fields':
477 print '+ login_base',':',v['login_base']
478 elif k == 'address_fields':
484 def display_initscript_spec (self,initscript):
485 print '+ ======== initscript',initscript['initscript_fields']['name']
487 def display_key_spec (self,key):
488 print '+ ======== key',key['key_name']
490 def display_slice_spec (self,slice):
491 print '+ ======== slice',slice['slice_fields']['name']
492 for (k,v) in slice.iteritems():
505 elif k=='slice_fields':
506 print '+ fields',':',
507 print 'max_nodes=',v['max_nodes'],
512 def display_node_spec (self,node):
513 print "+ node=%s host_box=%s"%(node['name'],node['host_box']),
514 print "hostname=",node['node_fields']['hostname'],
515 print "ip=",node['interface_fields']['ip']
516 if self.options.verbose:
517 utils.pprint("node details",node,depth=3)
519 # another entry point for just showing the boxes involved
520 def display_mapping (self):
521 TestPlc.display_mapping_plc(self.plc_spec)
525 def display_mapping_plc (plc_spec):
526 print '+ MyPLC',plc_spec['name']
527 # WARNING this would not be right for lxc-based PLC's - should be harmless though
528 print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['host_box'],plc_spec['vservername'])
529 print '+\tIP = %s/%s'%(plc_spec['PLC_API_HOST'],plc_spec['vserverip'])
530 for site_spec in plc_spec['sites']:
531 for node_spec in site_spec['nodes']:
532 TestPlc.display_mapping_node(node_spec)
535 def display_mapping_node (node_spec):
536 print '+ NODE %s'%(node_spec['name'])
537 print '+\tqemu box %s'%node_spec['host_box']
538 print '+\thostname=%s'%node_spec['node_fields']['hostname']
540 # write a timestamp in /vservers/<>.timestamp
541 # cannot be inside the vserver, that causes vserver .. build to cough
542 def timestamp_vs (self):
543 "Create a timestamp to remember creation date for this plc"
545 # TODO-lxc check this one
546 # a first approx. is to store the timestamp close to the VM root like vs does
547 stamp_path=self.vm_timestamp_path ()
548 stamp_dir = os.path.dirname (stamp_path)
549 utils.system(self.test_ssh.actual_command("mkdir -p %s"%stamp_dir))
550 return utils.system(self.test_ssh.actual_command("echo %d > %s"%(now,stamp_path)))==0
552 # this is called inconditionnally at the beginning of the test sequence
553 # just in case this is a rerun, so if the vm is not running it's fine
555 "vserver delete the test myplc"
556 stamp_path=self.vm_timestamp_path()
557 self.run_in_host("rm -f %s"%stamp_path)
558 if self.options.plcs_use_lxc:
559 self.run_in_host("virsh -c lxc:// destroy %s"%self.vservername)
560 self.run_in_host("virsh -c lxc:// undefine %s"%self.vservername)
561 self.run_in_host("rm -fr /vservers/%s"%self.vservername)
564 self.run_in_host("vserver --silent %s delete"%self.vservername)
568 # historically the build was being fetched by the tests
569 # now the build pushes itself as a subdir of the tests workdir
570 # so that the tests do not have to worry about extracting the build (svn, git, or whatever)
571 def vs_create (self):
572 "vserver creation (no install done)"
573 # push the local build/ dir to the testplc box
575 # a full path for the local calls
576 build_dir=os.path.dirname(sys.argv[0])
577 # sometimes this is empty - set to "." in such a case
578 if not build_dir: build_dir="."
579 build_dir += "/build"
581 # use a standard name - will be relative to remote buildname
583 # remove for safety; do *not* mkdir first, otherwise we end up with build/build/
584 self.test_ssh.rmdir(build_dir)
585 self.test_ssh.copy(build_dir,recursive=True)
586 # the repo url is taken from arch-rpms-url
587 # with the last step (i386) removed
588 repo_url = self.options.arch_rpms_url
589 for level in [ 'arch' ]:
590 repo_url = os.path.dirname(repo_url)
591 # pass the vbuild-nightly options to vtest-init-vserver
593 test_env_options += " -p %s"%self.options.personality
594 test_env_options += " -d %s"%self.options.pldistro
595 test_env_options += " -f %s"%self.options.fcdistro
596 if self.options.plcs_use_lxc:
597 script="vtest-init-lxc.sh"
599 script="vtest-init-vserver.sh"
600 vserver_name = self.vservername
601 vserver_options="--netdev eth0 --interface %s"%self.vserverip
603 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
604 vserver_options += " --hostname %s"%vserver_hostname
606 print "Cannot reverse lookup %s"%self.vserverip
607 print "This is considered fatal, as this might pollute the test results"
609 create_vserver="%(build_dir)s/%(script)s %(test_env_options)s %(vserver_name)s %(repo_url)s -- %(vserver_options)s"%locals()
610 return self.run_in_host(create_vserver) == 0
613 def plc_install(self):
614 "yum install myplc, noderepo, and the plain bootstrapfs"
616 # workaround for getting pgsql8.2 on centos5
617 if self.options.fcdistro == "centos5":
618 self.run_in_guest("rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-3.noarch.rpm")
621 if self.options.personality == "linux32":
623 elif self.options.personality == "linux64":
626 raise Exception, "Unsupported personality %r"%self.options.personality
627 nodefamily="%s-%s-%s"%(self.options.pldistro,self.options.fcdistro,arch)
630 pkgs_list.append ("slicerepo-%s"%nodefamily)
631 pkgs_list.append ("myplc")
632 pkgs_list.append ("noderepo-%s"%nodefamily)
633 pkgs_list.append ("nodeimage-%s-plain"%nodefamily)
634 pkgs_string=" ".join(pkgs_list)
635 return self.yum_install (pkgs_list)
638 def mod_python(self):
639 """yum install mod_python, useful on f18 and above so as to avoid broken wsgi"""
640 return self.yum_install ( [ 'mod_python' ] )
643 def plc_configure(self):
645 tmpname='%s.plc-config-tty'%(self.name())
646 fileconf=open(tmpname,'w')
647 for var in [ 'PLC_NAME',
652 'PLC_MAIL_SUPPORT_ADDRESS',
655 # Above line was added for integrating SFA Testing
661 'PLC_RESERVATION_GRANULARITY',
663 'PLC_OMF_XMPP_SERVER',
666 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
667 fileconf.write('w\n')
668 fileconf.write('q\n')
670 utils.system('cat %s'%tmpname)
671 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
672 utils.system('rm %s'%tmpname)
677 self.run_in_guest('service plc start')
682 self.run_in_guest('service plc stop')
686 "start the PLC vserver"
691 "stop the PLC vserver"
695 # stores the keys from the config for further use
696 def keys_store(self):
697 "stores test users ssh keys in keys/"
698 for key_spec in self.plc_spec['keys']:
699 TestKey(self,key_spec).store_key()
702 def keys_clean(self):
703 "removes keys cached in keys/"
704 utils.system("rm -rf ./keys")
707 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
708 # for later direct access to the nodes
709 def keys_fetch(self):
710 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
712 if not os.path.isdir(dir):
714 vservername=self.vservername
715 vm_root=self.vm_root_in_host()
717 prefix = 'debug_ssh_key'
718 for ext in [ 'pub', 'rsa' ] :
719 src="%(vm_root)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
720 dst="keys/%(vservername)s-debug.%(ext)s"%locals()
721 if self.test_ssh.fetch(src,dst) != 0: overall=False
725 "create sites with PLCAPI"
726 return self.do_sites()
728 def delete_sites (self):
729 "delete sites with PLCAPI"
730 return self.do_sites(action="delete")
732 def do_sites (self,action="add"):
733 for site_spec in self.plc_spec['sites']:
734 test_site = TestSite (self,site_spec)
735 if (action != "add"):
736 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
737 test_site.delete_site()
738 # deleted with the site
739 #test_site.delete_users()
742 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
743 test_site.create_site()
744 test_site.create_users()
747 def delete_all_sites (self):
748 "Delete all sites in PLC, and related objects"
749 print 'auth_root',self.auth_root()
750 sites = self.apiserver.GetSites(self.auth_root(), {}, ['site_id','login_base'])
752 # keep automatic site - otherwise we shoot in our own foot, root_auth is not valid anymore
753 if site['login_base']==self.plc_spec['PLC_SLICE_PREFIX']: continue
754 site_id=site['site_id']
755 print 'Deleting site_id',site_id
756 self.apiserver.DeleteSite(self.auth_root(),site_id)
760 "create nodes with PLCAPI"
761 return self.do_nodes()
762 def delete_nodes (self):
763 "delete nodes with PLCAPI"
764 return self.do_nodes(action="delete")
766 def do_nodes (self,action="add"):
767 for site_spec in self.plc_spec['sites']:
768 test_site = TestSite (self,site_spec)
770 utils.header("Deleting nodes in site %s"%test_site.name())
771 for node_spec in site_spec['nodes']:
772 test_node=TestNode(self,test_site,node_spec)
773 utils.header("Deleting %s"%test_node.name())
774 test_node.delete_node()
776 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
777 for node_spec in site_spec['nodes']:
778 utils.pprint('Creating node %s'%node_spec,node_spec)
779 test_node = TestNode (self,test_site,node_spec)
780 test_node.create_node ()
783 def nodegroups (self):
784 "create nodegroups with PLCAPI"
785 return self.do_nodegroups("add")
786 def delete_nodegroups (self):
787 "delete nodegroups with PLCAPI"
788 return self.do_nodegroups("delete")
792 def translate_timestamp (start,grain,timestamp):
793 if timestamp < TestPlc.YEAR: return start+timestamp*grain
794 else: return timestamp
797 def timestamp_printable (timestamp):
798 return time.strftime('%m-%d %H:%M:%S UTC',time.gmtime(timestamp))
801 "create leases (on reservable nodes only, use e.g. run -c default -c resa)"
803 grain=self.apiserver.GetLeaseGranularity(self.auth_root())
804 print 'API answered grain=',grain
805 start=(now/grain)*grain
807 # find out all nodes that are reservable
808 nodes=self.all_reservable_nodenames()
810 utils.header ("No reservable node found - proceeding without leases")
813 # attach them to the leases as specified in plc_specs
814 # this is where the 'leases' field gets interpreted as relative of absolute
815 for lease_spec in self.plc_spec['leases']:
816 # skip the ones that come with a null slice id
817 if not lease_spec['slice']: continue
818 lease_spec['t_from']=TestPlc.translate_timestamp(start,grain,lease_spec['t_from'])
819 lease_spec['t_until']=TestPlc.translate_timestamp(start,grain,lease_spec['t_until'])
820 lease_addition=self.apiserver.AddLeases(self.auth_root(),nodes,
821 lease_spec['slice'],lease_spec['t_from'],lease_spec['t_until'])
822 if lease_addition['errors']:
823 utils.header("Cannot create leases, %s"%lease_addition['errors'])
826 utils.header('Leases on nodes %r for %s from %d (%s) until %d (%s)'%\
827 (nodes,lease_spec['slice'],
828 lease_spec['t_from'],TestPlc.timestamp_printable(lease_spec['t_from']),
829 lease_spec['t_until'],TestPlc.timestamp_printable(lease_spec['t_until'])))
833 def delete_leases (self):
834 "remove all leases in the myplc side"
835 lease_ids= [ l['lease_id'] for l in self.apiserver.GetLeases(self.auth_root())]
836 utils.header("Cleaning leases %r"%lease_ids)
837 self.apiserver.DeleteLeases(self.auth_root(),lease_ids)
840 def list_leases (self):
841 "list all leases known to the myplc"
842 leases = self.apiserver.GetLeases(self.auth_root())
845 current=l['t_until']>=now
846 if self.options.verbose or current:
847 utils.header("%s %s from %s until %s"%(l['hostname'],l['name'],
848 TestPlc.timestamp_printable(l['t_from']),
849 TestPlc.timestamp_printable(l['t_until'])))
852 # create nodegroups if needed, and populate
853 def do_nodegroups (self, action="add"):
854 # 1st pass to scan contents
856 for site_spec in self.plc_spec['sites']:
857 test_site = TestSite (self,site_spec)
858 for node_spec in site_spec['nodes']:
859 test_node=TestNode (self,test_site,node_spec)
860 if node_spec.has_key('nodegroups'):
861 nodegroupnames=node_spec['nodegroups']
862 if isinstance(nodegroupnames,StringTypes):
863 nodegroupnames = [ nodegroupnames ]
864 for nodegroupname in nodegroupnames:
865 if not groups_dict.has_key(nodegroupname):
866 groups_dict[nodegroupname]=[]
867 groups_dict[nodegroupname].append(test_node.name())
868 auth=self.auth_root()
870 for (nodegroupname,group_nodes) in groups_dict.iteritems():
872 print 'nodegroups:','dealing with nodegroup',nodegroupname,'on nodes',group_nodes
873 # first, check if the nodetagtype is here
874 tag_types = self.apiserver.GetTagTypes(auth,{'tagname':nodegroupname})
876 tag_type_id = tag_types[0]['tag_type_id']
878 tag_type_id = self.apiserver.AddTagType(auth,
879 {'tagname':nodegroupname,
880 'description': 'for nodegroup %s'%nodegroupname,
882 print 'located tag (type)',nodegroupname,'as',tag_type_id
884 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
886 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
887 print 'created nodegroup',nodegroupname,'from tagname',nodegroupname,'and value','yes'
888 # set node tag on all nodes, value='yes'
889 for nodename in group_nodes:
891 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
893 traceback.print_exc()
894 print 'node',nodename,'seems to already have tag',nodegroupname
897 expect_yes = self.apiserver.GetNodeTags(auth,
898 {'hostname':nodename,
899 'tagname':nodegroupname},
900 ['value'])[0]['value']
901 if expect_yes != "yes":
902 print 'Mismatch node tag on node',nodename,'got',expect_yes
905 if not self.options.dry_run:
906 print 'Cannot find tag',nodegroupname,'on node',nodename
910 print 'cleaning nodegroup',nodegroupname
911 self.apiserver.DeleteNodeGroup(auth,nodegroupname)
913 traceback.print_exc()
917 # a list of TestNode objs
918 def all_nodes (self):
920 for site_spec in self.plc_spec['sites']:
921 test_site = TestSite (self,site_spec)
922 for node_spec in site_spec['nodes']:
923 nodes.append(TestNode (self,test_site,node_spec))
926 # return a list of tuples (nodename,qemuname)
927 def all_node_infos (self) :
929 for site_spec in self.plc_spec['sites']:
930 node_infos += [ (node_spec['node_fields']['hostname'],node_spec['host_box']) \
931 for node_spec in site_spec['nodes'] ]
934 def all_nodenames (self): return [ x[0] for x in self.all_node_infos() ]
935 def all_reservable_nodenames (self):
937 for site_spec in self.plc_spec['sites']:
938 for node_spec in site_spec['nodes']:
939 node_fields=node_spec['node_fields']
940 if 'node_type' in node_fields and node_fields['node_type']=='reservable':
941 res.append(node_fields['hostname'])
944 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
945 def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period_seconds=15):
946 if self.options.dry_run:
950 class CompleterTaskBootState (CompleterTask):
951 def __init__ (self, test_plc,hostname):
952 self.test_plc=test_plc
953 self.hostname=hostname
954 self.last_boot_state='undef'
955 def actual_run (self):
957 node = self.test_plc.apiserver.GetNodes(self.test_plc.auth_root(), [ self.hostname ],
959 self.last_boot_state = node['boot_state']
960 return self.last_boot_state == target_boot_state
964 return "CompleterTaskBootState with node %s"%self.hostname
965 def failure_message (self):
966 return "node %s in state %s - expected %s"%(self.hostname,self.last_boot_state,target_boot_state)
968 timeout = timedelta(minutes=timeout_minutes)
969 graceout = timedelta(minutes=silent_minutes)
970 period = timedelta(seconds=period_seconds)
971 # the nodes that haven't checked yet - start with a full list and shrink over time
972 utils.header("checking nodes boot state (expected %s)"%target_boot_state)
973 tasks = [ CompleterTaskBootState (self,hostname) \
974 for (hostname,_) in self.all_node_infos() ]
975 return Completer (tasks).run (timeout, graceout, period)
977 def nodes_booted(self):
978 return self.nodes_check_boot_state('boot',timeout_minutes=30,silent_minutes=28)
980 def check_nodes_ssh(self,debug,timeout_minutes,silent_minutes,period_seconds=15):
981 class CompleterTaskNodeSsh (CompleterTask):
982 def __init__ (self, hostname, qemuname, boot_state, local_key):
983 self.hostname=hostname
984 self.qemuname=qemuname
985 self.boot_state=boot_state
986 self.local_key=local_key
987 def run (self, silent):
988 command = TestSsh (self.hostname,key=self.local_key).actual_command("hostname;uname -a")
989 return utils.system (command, silent=silent)==0
990 def failure_message (self):
991 return "Cannot reach %s @ %s in %s mode"%(self.hostname, self.qemuname, self.boot_state)
994 timeout = timedelta(minutes=timeout_minutes)
995 graceout = timedelta(minutes=silent_minutes)
996 period = timedelta(seconds=period_seconds)
997 vservername=self.vservername
1000 local_key = "keys/%(vservername)s-debug.rsa"%locals()
1003 local_key = "keys/key_admin.rsa"
1004 utils.header("checking ssh access to nodes (expected in %s mode)"%message)
1005 node_infos = self.all_node_infos()
1006 tasks = [ CompleterTaskNodeSsh (nodename, qemuname, message, local_key) \
1007 for (nodename,qemuname) in node_infos ]
1008 return Completer (tasks).run (timeout, graceout, period)
1010 def ssh_node_debug(self):
1011 "Tries to ssh into nodes in debug mode with the debug ssh key"
1012 return self.check_nodes_ssh(debug=True,
1013 timeout_minutes=self.ssh_node_debug_timeout,
1014 silent_minutes=self.ssh_node_debug_silent)
1016 def ssh_node_boot(self):
1017 "Tries to ssh into nodes in production mode with the root ssh key"
1018 return self.check_nodes_ssh(debug=False,
1019 timeout_minutes=self.ssh_node_boot_timeout,
1020 silent_minutes=self.ssh_node_boot_silent)
1022 def node_bmlogs(self):
1023 "Checks that there's a non-empty dir. /var/log/bm/raw"
1024 return utils.system(self.actual_command_in_guest("ls /var/log/bm/raw"))==0
1027 def qemu_local_init (self): pass
1029 def bootcd (self): pass
1031 def qemu_local_config (self): pass
1033 def nodestate_reinstall (self): pass
1035 def nodestate_safeboot (self): pass
1037 def nodestate_boot (self): pass
1039 def nodestate_show (self): pass
1041 def qemu_export (self): pass
1043 ### check hooks : invoke scripts from hooks/{node,slice}
1044 def check_hooks_node (self):
1045 return self.locate_first_node().check_hooks()
1046 def check_hooks_sliver (self) :
1047 return self.locate_first_sliver().check_hooks()
1049 def check_hooks (self):
1050 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
1051 return self.check_hooks_node() and self.check_hooks_sliver()
1054 def do_check_initscripts(self):
1055 class CompleterTaskInitscript (CompleterTask):
1056 def __init__ (self, test_sliver, stamp):
1057 self.test_sliver=test_sliver
1059 def actual_run (self):
1060 return self.test_sliver.check_initscript_stamp (self.stamp)
1062 return "initscript checker for %s"%self.test_sliver.name()
1063 def failure_message (self):
1064 return "initscript stamp %s not found in sliver %s"%(self.stamp,self.test_sliver.name())
1067 for slice_spec in self.plc_spec['slices']:
1068 if not slice_spec.has_key('initscriptstamp'):
1070 stamp=slice_spec['initscriptstamp']
1071 slicename=slice_spec['slice_fields']['name']
1072 for nodename in slice_spec['nodenames']:
1073 print 'nodename',nodename,'slicename',slicename,'stamp',stamp
1074 (site,node) = self.locate_node (nodename)
1075 # xxx - passing the wrong site - probably harmless
1076 test_site = TestSite (self,site)
1077 test_slice = TestSlice (self,test_site,slice_spec)
1078 test_node = TestNode (self,test_site,node)
1079 test_sliver = TestSliver (self, test_node, test_slice)
1080 tasks.append ( CompleterTaskInitscript (test_sliver, stamp))
1081 return Completer (tasks).run (timedelta(minutes=5), timedelta(minutes=4), timedelta(seconds=10))
1083 def check_initscripts(self):
1084 "check that the initscripts have triggered"
1085 return self.do_check_initscripts()
1087 def initscripts (self):
1088 "create initscripts with PLCAPI"
1089 for initscript in self.plc_spec['initscripts']:
1090 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
1091 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
1094 def delete_initscripts (self):
1095 "delete initscripts with PLCAPI"
1096 for initscript in self.plc_spec['initscripts']:
1097 initscript_name = initscript['initscript_fields']['name']
1098 print('Attempting to delete %s in plc %s'%(initscript_name,self.plc_spec['name']))
1100 self.apiserver.DeleteInitScript(self.auth_root(),initscript_name)
1101 print initscript_name,'deleted'
1103 print 'deletion went wrong - probably did not exist'
1108 "create slices with PLCAPI"
1109 return self.do_slices(action="add")
1111 def delete_slices (self):
1112 "delete slices with PLCAPI"
1113 return self.do_slices(action="delete")
1115 def fill_slices (self):
1116 "add nodes in slices with PLCAPI"
1117 return self.do_slices(action="fill")
1119 def empty_slices (self):
1120 "remove nodes from slices with PLCAPI"
1121 return self.do_slices(action="empty")
1123 def do_slices (self, action="add"):
1124 for slice in self.plc_spec['slices']:
1125 site_spec = self.locate_site (slice['sitename'])
1126 test_site = TestSite(self,site_spec)
1127 test_slice=TestSlice(self,test_site,slice)
1128 if action == "delete":
1129 test_slice.delete_slice()
1130 elif action=="fill":
1131 test_slice.add_nodes()
1132 elif action=="empty":
1133 test_slice.delete_nodes()
1135 test_slice.create_slice()
1139 def ssh_slice(self): pass
1141 def ssh_slice_off (self): pass
1143 def ssh_slice_basics(self): pass
1146 def check_vsys_defaults(self): pass
1149 def keys_clear_known_hosts (self): pass
1151 def plcapi_urls (self):
1152 return PlcapiUrlScanner (self.auth_root(),ip=self.vserverip).scan()
1154 def speed_up_slices (self):
1155 "tweak nodemanager settings on all nodes using a conf file"
1156 # create the template on the server-side
1157 template="%s.nodemanager"%self.name()
1158 template_file = open (template,"w")
1159 template_file.write('OPTIONS="-p 30 -r 11 -d"\n')
1160 template_file.close()
1161 in_vm="/var/www/html/PlanetLabConf/nodemanager"
1162 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1163 self.test_ssh.copy_abs(template,remote)
1165 self.apiserver.AddConfFile (self.auth_root(),
1166 {'dest':'/etc/sysconfig/nodemanager',
1167 'source':'PlanetLabConf/nodemanager',
1168 'postinstall_cmd':'service nm restart',})
1171 def debug_nodemanager (self):
1172 "sets verbose mode for nodemanager, and speeds up cycle even more (needs speed_up_slices first)"
1173 template="%s.nodemanager"%self.name()
1174 template_file = open (template,"w")
1175 template_file.write('OPTIONS="-p 10 -r 6 -v -d"\n')
1176 template_file.close()
1177 in_vm="/var/www/html/PlanetLabConf/nodemanager"
1178 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1179 self.test_ssh.copy_abs(template,remote)
1183 def qemu_start (self) : pass
1186 def timestamp_qemu (self) : pass
1188 # when a spec refers to a node possibly on another plc
1189 def locate_sliver_obj_cross (self, nodename, slicename, other_plcs):
1190 for plc in [ self ] + other_plcs:
1192 return plc.locate_sliver_obj (nodename, slicename)
1195 raise Exception, "Cannot locate sliver %s@%s among all PLCs"%(nodename,slicename)
1197 # implement this one as a cross step so that we can take advantage of different nodes
1198 # in multi-plcs mode
1199 def cross_check_tcp (self, other_plcs):
1200 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
1201 if 'tcp_specs' not in self.plc_spec or not self.plc_spec['tcp_specs']:
1202 utils.header ("check_tcp: no/empty config found")
1204 specs = self.plc_spec['tcp_specs']
1209 s_test_sliver = self.locate_sliver_obj_cross (spec['server_node'],spec['server_slice'],other_plcs)
1210 if not s_test_sliver.run_tcp_server(port,timeout=20):
1214 # idem for the client side
1215 c_test_sliver = self.locate_sliver_obj_cross (spec['client_node'],spec['client_slice'],other_plcs)
1216 # use nodename from locatesd sliver, unless 'client_connect' is set
1217 if 'client_connect' in spec:
1218 destination = spec['client_connect']
1220 destination=s_test_sliver.test_node.name()
1221 if not c_test_sliver.run_tcp_client(destination,port):
1225 # painfully enough, we need to allow for some time as netflow might show up last
1226 def check_system_slice (self):
1227 "all nodes: check that a system slice is alive"
1228 # netflow currently not working in the lxc distro
1229 # drl not built at all in the wtx distro
1230 # if we find either of them we're happy
1231 return self.check_netflow() or self.check_drl()
1234 def check_netflow (self): return self._check_system_slice ('netflow')
1235 def check_drl (self): return self._check_system_slice ('drl')
1237 # we have the slices up already here, so it should not take too long
1238 def _check_system_slice (self, slicename, timeout_minutes=5, period_seconds=15):
1239 class CompleterTaskSystemSlice (CompleterTask):
1240 def __init__ (self, test_node, dry_run):
1241 self.test_node=test_node
1242 self.dry_run=dry_run
1243 def actual_run (self):
1244 return self.test_node._check_system_slice (slicename, dry_run=self.dry_run)
1246 return "System slice %s @ %s"%(slicename, self.test_node.name())
1247 def failure_message (self):
1248 return "COULD not find system slice %s @ %s"%(slicename, self.test_node.name())
1249 timeout = timedelta(minutes=timeout_minutes)
1250 silent = timedelta (0)
1251 period = timedelta (seconds=period_seconds)
1252 tasks = [ CompleterTaskSystemSlice (test_node, self.options.dry_run) \
1253 for test_node in self.all_nodes() ]
1254 return Completer (tasks) . run (timeout, silent, period)
1256 def plcsh_stress_test (self):
1257 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
1258 # install the stress-test in the plc image
1259 location = "/usr/share/plc_api/plcsh_stress_test.py"
1260 remote="%s/%s"%(self.vm_root_in_host(),location)
1261 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1263 command += " -- --check"
1264 if self.options.size == 1:
1265 command += " --tiny"
1266 return ( self.run_in_guest(command) == 0)
1268 # populate runs the same utility without slightly different options
1269 # in particular runs with --preserve (dont cleanup) and without --check
1270 # also it gets run twice, once with the --foreign option for creating fake foreign entries
1272 def sfa_install_all (self):
1273 "yum install sfa sfa-plc sfa-sfatables sfa-client"
1274 return self.yum_install ("sfa sfa-plc sfa-sfatables sfa-client")
1276 def sfa_install_core(self):
1278 return self.yum_install ("sfa")
1280 def sfa_install_plc(self):
1281 "yum install sfa-plc"
1282 return self.yum_install("sfa-plc")
1284 def sfa_install_sfatables(self):
1285 "yum install sfa-sfatables"
1286 return self.yum_install ("sfa-sfatables")
1288 # for some very odd reason, this sometimes fails with the following symptom
1289 # # yum install sfa-client
1290 # Setting up Install Process
1292 # Downloading Packages:
1293 # Running rpm_check_debug
1294 # Running Transaction Test
1295 # Transaction Test Succeeded
1296 # Running Transaction
1297 # Transaction couldn't start:
1298 # installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem
1299 # [('installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem', (9, '/', 69632L))]
1300 # even though in the same context I have
1301 # [2012.05.23--f14-32-sfastd1-1-vplc07] / # df -h
1302 # Filesystem Size Used Avail Use% Mounted on
1303 # /dev/hdv1 806G 264G 501G 35% /
1304 # none 16M 36K 16M 1% /tmp
1306 # so as a workaround, we first try yum install, and then invoke rpm on the cached rpm...
1307 def sfa_install_client(self):
1308 "yum install sfa-client"
1309 first_try=self.yum_install("sfa-client")
1310 if first_try: return True
1311 utils.header ("********** Regular yum failed - special workaround in place, 2nd chance")
1312 (code,cached_rpm_path)=utils.output_of(self.actual_command_in_guest('find /var/cache/yum -name sfa-client\*.rpm'))
1313 utils.header("rpm_path=<<%s>>"%rpm_path)
1315 self.run_in_guest("rpm -i %s"%cached_rpm_path)
1316 return self.yum_check_installed ("sfa-client")
1318 def sfa_dbclean(self):
1319 "thoroughly wipes off the SFA database"
1320 return self.run_in_guest("sfaadmin reg nuke")==0 or \
1321 self.run_in_guest("sfa-nuke.py")==0 or \
1322 self.run_in_guest("sfa-nuke-plc.py")==0
1324 def sfa_fsclean(self):
1325 "cleanup /etc/sfa/trusted_roots and /var/lib/sfa"
1326 self.run_in_guest("rm -rf /etc/sfa/trusted_roots /var/lib/sfa/authorities")
1329 def sfa_plcclean(self):
1330 "cleans the PLC entries that were created as a side effect of running the script"
1332 sfa_spec=self.plc_spec['sfa']
1334 for auth_sfa_spec in sfa_spec['auth_sfa_specs']:
1335 login_base=auth_sfa_spec['login_base']
1336 try: self.apiserver.DeleteSite (self.auth_root(),login_base)
1337 except: print "Site %s already absent from PLC db"%login_base
1339 for spec_name in ['pi_spec','user_spec']:
1340 user_spec=auth_sfa_spec[spec_name]
1341 username=user_spec['email']
1342 try: self.apiserver.DeletePerson(self.auth_root(),username)
1344 # this in fact is expected as sites delete their members
1345 #print "User %s already absent from PLC db"%username
1348 print "REMEMBER TO RUN sfa_import AGAIN"
1351 def sfa_uninstall(self):
1352 "uses rpm to uninstall sfa - ignore result"
1353 self.run_in_guest("rpm -e sfa sfa-sfatables sfa-client sfa-plc")
1354 self.run_in_guest("rm -rf /var/lib/sfa")
1355 self.run_in_guest("rm -rf /etc/sfa")
1356 self.run_in_guest("rm -rf /var/log/sfa_access.log /var/log/sfa_import_plc.log /var/log/sfa.daemon")
1358 self.run_in_guest("rpm -e --noscripts sfa-plc")
1361 ### run unit tests for SFA
1362 # NOTE: for some reason on f14/i386, yum install sfa-tests fails for no reason
1363 # Running Transaction
1364 # Transaction couldn't start:
1365 # installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem
1366 # [('installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem', (9, '/', 208896L))]
1367 # no matter how many Gbs are available on the testplc
1368 # could not figure out what's wrong, so...
1369 # if the yum install phase fails, consider the test is successful
1370 # other combinations will eventually run it hopefully
1371 def sfa_utest(self):
1372 "yum install sfa-tests and run SFA unittests"
1373 self.run_in_guest("yum -y install sfa-tests")
1374 # failed to install - forget it
1375 if self.run_in_guest("rpm -q sfa-tests")!=0:
1376 utils.header("WARNING: SFA unit tests failed to install, ignoring")
1378 return self.run_in_guest("/usr/share/sfa/tests/testAll.py")==0
1382 dirname="conf.%s"%self.plc_spec['name']
1383 if not os.path.isdir(dirname):
1384 utils.system("mkdir -p %s"%dirname)
1385 if not os.path.isdir(dirname):
1386 raise Exception,"Cannot create config dir for plc %s"%self.name()
1389 def conffile(self,filename):
1390 return "%s/%s"%(self.confdir(),filename)
1391 def confsubdir(self,dirname,clean,dry_run=False):
1392 subdirname="%s/%s"%(self.confdir(),dirname)
1394 utils.system("rm -rf %s"%subdirname)
1395 if not os.path.isdir(subdirname):
1396 utils.system("mkdir -p %s"%subdirname)
1397 if not dry_run and not os.path.isdir(subdirname):
1398 raise "Cannot create config subdir %s for plc %s"%(dirname,self.name())
1401 def conffile_clean (self,filename):
1402 filename=self.conffile(filename)
1403 return utils.system("rm -rf %s"%filename)==0
1406 def sfa_configure(self):
1407 "run sfa-config-tty"
1408 tmpname=self.conffile("sfa-config-tty")
1409 fileconf=open(tmpname,'w')
1410 for var in [ 'SFA_REGISTRY_ROOT_AUTH',
1411 'SFA_INTERFACE_HRN',
1412 'SFA_REGISTRY_LEVEL1_AUTH',
1413 'SFA_REGISTRY_HOST',
1414 'SFA_AGGREGATE_HOST',
1424 'SFA_GENERIC_FLAVOUR',
1425 'SFA_AGGREGATE_ENABLED',
1427 if self.plc_spec['sfa'].has_key(var):
1428 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec['sfa'][var]))
1429 # the way plc_config handles booleans just sucks..
1432 if self.plc_spec['sfa'][var]: val='true'
1433 fileconf.write ('e %s\n%s\n'%(var,val))
1434 fileconf.write('w\n')
1435 fileconf.write('R\n')
1436 fileconf.write('q\n')
1438 utils.system('cat %s'%tmpname)
1439 self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
1442 def aggregate_xml_line(self):
1443 port=self.plc_spec['sfa']['neighbours-port']
1444 return '<aggregate addr="%s" hrn="%s" port="%r"/>' % \
1445 (self.vserverip,self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH'],port)
1447 def registry_xml_line(self):
1448 return '<registry addr="%s" hrn="%s" port="12345"/>' % \
1449 (self.vserverip,self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH'])
1452 # a cross step that takes all other plcs in argument
1453 def cross_sfa_configure(self, other_plcs):
1454 "writes aggregates.xml and registries.xml that point to all other PLCs in the test"
1455 # of course with a single plc, other_plcs is an empty list
1458 agg_fname=self.conffile("agg.xml")
1459 file(agg_fname,"w").write("<aggregates>%s</aggregates>\n" % \
1460 " ".join([ plc.aggregate_xml_line() for plc in other_plcs ]))
1461 utils.header ("(Over)wrote %s"%agg_fname)
1462 reg_fname=self.conffile("reg.xml")
1463 file(reg_fname,"w").write("<registries>%s</registries>\n" % \
1464 " ".join([ plc.registry_xml_line() for plc in other_plcs ]))
1465 utils.header ("(Over)wrote %s"%reg_fname)
1466 return self.test_ssh.copy_abs(agg_fname,'/%s/etc/sfa/aggregates.xml'%self.vm_root_in_host())==0 \
1467 and self.test_ssh.copy_abs(reg_fname,'/%s/etc/sfa/registries.xml'%self.vm_root_in_host())==0
1469 def sfa_import(self):
1470 "use sfaadmin to import from plc"
1471 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1473 self.run_in_guest('sfaadmin reg import_registry')==0
1474 # not needed anymore
1475 # self.run_in_guest('cp /etc/sfa/authorities/%s/%s.pkey /etc/sfa/authorities/server.key'%(auth,auth))
1477 def sfa_start(self):
1479 return self.run_in_guest('service sfa start')==0
1481 def sfi_configure(self):
1482 "Create /root/sfi on the plc side for sfi client configuration"
1483 if self.options.dry_run:
1484 utils.header("DRY RUN - skipping step")
1486 sfa_spec=self.plc_spec['sfa']
1487 # cannot use auth_sfa_mapper to pass dir_name
1488 for slice_spec in self.plc_spec['sfa']['auth_sfa_specs']:
1489 test_slice=TestAuthSfa(self,slice_spec)
1490 dir_basename=os.path.basename(test_slice.sfi_path())
1491 dir_name=self.confsubdir("dot-sfi/%s"%dir_basename,clean=True,dry_run=self.options.dry_run)
1492 test_slice.sfi_configure(dir_name)
1493 # push into the remote /root/sfi area
1494 location = test_slice.sfi_path()
1495 remote="%s/%s"%(self.vm_root_in_host(),location)
1496 self.test_ssh.mkdir(remote,abs=True)
1497 # need to strip last level or remote otherwise we get an extra dir level
1498 self.test_ssh.copy_abs(dir_name, os.path.dirname(remote), recursive=True)
1502 def sfi_clean (self):
1503 "clean up /root/sfi on the plc side"
1504 self.run_in_guest("rm -rf /root/sfi")
1508 def sfa_add_site (self): pass
1510 def sfa_add_pi (self): pass
1512 def sfa_add_user(self): pass
1514 def sfa_update_user(self): pass
1516 def sfa_add_slice(self): pass
1518 def sfa_renew_slice(self): pass
1520 def sfa_discover(self): pass
1522 def sfa_create_slice(self): pass
1524 def sfa_check_slice_plc(self): pass
1526 def sfa_update_slice(self): pass
1528 def sfi_list(self): pass
1530 def sfi_show(self): pass
1532 def ssh_slice_sfa(self): pass
1534 def sfa_delete_user(self): pass
1536 def sfa_delete_slice(self): pass
1540 self.run_in_guest('service sfa stop')==0
1543 def populate (self):
1544 "creates random entries in the PLCAPI"
1545 # install the stress-test in the plc image
1546 location = "/usr/share/plc_api/plcsh_stress_test.py"
1547 remote="%s/%s"%(self.vm_root_in_host(),location)
1548 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1550 command += " -- --preserve --short-names"
1551 local = (self.run_in_guest(command) == 0);
1552 # second run with --foreign
1553 command += ' --foreign'
1554 remote = (self.run_in_guest(command) == 0);
1555 return ( local and remote)
1557 def gather_logs (self):
1558 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1559 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1560 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1561 # (1.c) get the plc's /root/sfi -> logs/sfi.<plcname>/
1562 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1563 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1564 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1566 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1567 self.gather_var_logs ()
1569 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1570 self.gather_pgsql_logs ()
1572 print "-------------------- TestPlc.gather_logs : PLC's /root/sfi/"
1573 self.gather_root_sfi ()
1575 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1576 for site_spec in self.plc_spec['sites']:
1577 test_site = TestSite (self,site_spec)
1578 for node_spec in site_spec['nodes']:
1579 test_node=TestNode(self,test_site,node_spec)
1580 test_node.gather_qemu_logs()
1582 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1583 self.gather_nodes_var_logs()
1585 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1586 self.gather_slivers_var_logs()
1589 def gather_slivers_var_logs(self):
1590 for test_sliver in self.all_sliver_objs():
1591 remote = test_sliver.tar_var_logs()
1592 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
1593 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
1594 utils.system(command)
1597 def gather_var_logs (self):
1598 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
1599 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1600 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
1601 utils.system(command)
1602 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd"%self.name()
1603 utils.system(command)
1605 def gather_pgsql_logs (self):
1606 utils.system("mkdir -p logs/myplc.pgsql-log.%s"%self.name())
1607 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1608 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -"%self.name()
1609 utils.system(command)
1611 def gather_root_sfi (self):
1612 utils.system("mkdir -p logs/sfi.%s"%self.name())
1613 to_plc = self.actual_command_in_guest("tar -C /root/sfi/ -cf - .")
1614 command = to_plc + "| tar -C logs/sfi.%s -xf -"%self.name()
1615 utils.system(command)
1617 def gather_nodes_var_logs (self):
1618 for site_spec in self.plc_spec['sites']:
1619 test_site = TestSite (self,site_spec)
1620 for node_spec in site_spec['nodes']:
1621 test_node=TestNode(self,test_site,node_spec)
1622 test_ssh = TestSsh (test_node.name(),key="keys/key_admin.rsa")
1623 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1624 command = command + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
1625 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
1626 utils.system(command)
1629 # returns the filename to use for sql dump/restore, using options.dbname if set
1630 def dbfile (self, database):
1631 # uses options.dbname if it is found
1633 name=self.options.dbname
1634 if not isinstance(name,StringTypes):
1640 return "/root/%s-%s.sql"%(database,name)
1642 def plc_db_dump(self):
1643 'dump the planetlab5 DB in /root in the PLC - filename has time'
1644 dump=self.dbfile("planetab5")
1645 self.run_in_guest('pg_dump -U pgsqluser planetlab5 -f '+ dump)
1646 utils.header('Dumped planetlab5 database in %s'%dump)
1649 def plc_db_restore(self):
1650 'restore the planetlab5 DB - looks broken, but run -n might help'
1651 dump=self.dbfile("planetab5")
1652 ##stop httpd service
1653 self.run_in_guest('service httpd stop')
1654 # xxx - need another wrapper
1655 self.run_in_guest_piped('echo drop database planetlab5','psql --user=pgsqluser template1')
1656 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab5')
1657 self.run_in_guest('psql -U pgsqluser planetlab5 -f '+dump)
1658 ##starting httpd service
1659 self.run_in_guest('service httpd start')
1661 utils.header('Database restored from ' + dump)
1663 def standby_1_through_20(self):
1664 """convenience function to wait for a specified number of minutes"""
1667 def standby_1(): pass
1669 def standby_2(): pass
1671 def standby_3(): pass
1673 def standby_4(): pass
1675 def standby_5(): pass
1677 def standby_6(): pass
1679 def standby_7(): pass
1681 def standby_8(): pass
1683 def standby_9(): pass
1685 def standby_10(): pass
1687 def standby_11(): pass
1689 def standby_12(): pass
1691 def standby_13(): pass
1693 def standby_14(): pass
1695 def standby_15(): pass
1697 def standby_16(): pass
1699 def standby_17(): pass
1701 def standby_18(): pass
1703 def standby_19(): pass
1705 def standby_20(): pass