1 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
2 # Copyright (C) 2010 INRIA
9 from types import StringTypes
13 from TestSite import TestSite
14 from TestNode import TestNode
15 from TestUser import TestUser
16 from TestKey import TestKey
17 from TestSlice import TestSlice
18 from TestSliver import TestSliver
19 from TestBoxQemu import TestBoxQemu
20 from TestSsh import TestSsh
21 from TestApiserver import TestApiserver
22 from TestSliceSfa import TestSliceSfa
24 # step methods must take (self) and return a boolean (options is a member of the class)
26 def standby(minutes,dry_run):
27 utils.header('Entering StandBy for %d mn'%minutes)
31 time.sleep(60*minutes)
34 def standby_generic (func):
36 minutes=int(func.__name__.split("_")[1])
37 return standby(minutes,self.options.dry_run)
40 def node_mapper (method):
41 def actual(self,*args, **kwds):
43 node_method = TestNode.__dict__[method.__name__]
44 for test_node in self.all_nodes():
45 if not node_method(test_node, *args, **kwds): overall=False
47 # restore the doc text
48 actual.__doc__=TestNode.__dict__[method.__name__].__doc__
51 def slice_mapper (method):
54 slice_method = TestSlice.__dict__[method.__name__]
55 for slice_spec in self.plc_spec['slices']:
56 site_spec = self.locate_site (slice_spec['sitename'])
57 test_site = TestSite(self,site_spec)
58 test_slice=TestSlice(self,test_site,slice_spec)
59 if not slice_method(test_slice,self.options): overall=False
61 # restore the doc text
62 actual.__doc__=TestSlice.__dict__[method.__name__].__doc__
65 def slice_sfa_mapper (method):
68 slice_method = TestSliceSfa.__dict__[method.__name__]
69 for slice_spec in self.plc_spec['sfa']['sfa_slice_specs']:
70 test_slice=TestSliceSfa(self,slice_spec)
71 if not slice_method(test_slice,self.options): overall=False
73 # restore the doc text
74 actual.__doc__=TestSliceSfa.__dict__[method.__name__].__doc__
84 'vs_delete','timestamp_vs','vs_create', SEP,
85 'plc_install', 'plc_configure', 'plc_start', SEP,
86 'keys_fetch', 'keys_store', 'keys_clear_known_hosts', 'speed_up_slices', SEP,
87 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', 'leases', SEP,
88 'nodestate_reinstall', 'qemu_local_init','bootcd', 'qemu_local_config', SEP,
89 'qemu_export', 'qemu_kill_mine', 'qemu_start', 'timestamp_qemu', SEP,
90 'sfa_install_all', 'sfa_configure', 'cross_sfa_configure', 'sfa_start', 'sfa_import', SEPSFA,
91 'sfi_configure@1', 'sfa_add_site@1','sfa_add_pi@1', SEPSFA,
92 'sfa_add_user@1', 'sfa_add_slice@1', 'sfa_discover@1', SEPSFA,
93 'sfa_create_slice@1', 'sfa_check_slice_plc@1', SEPSFA,
94 'sfa_update_user@1', 'sfa_update_slice@1', SEPSFA,
95 'sfi_list@1', 'sfi_show@1', 'sfi_slices@1', 'sfa_utest@1', SEPSFA,
96 # we used to run plcsh_stress_test, and then ssh_node_debug and ssh_node_boot
97 # but as the stress test might take a while, we sometimes missed the debug mode..
98 'ssh_node_debug@1', 'plcsh_stress_test@1', SEP,
99 'ssh_node_boot@1', 'ssh_slice', 'check_initscripts', SEP,
100 'ssh_slice_sfa@1', 'sfa_delete_slice@1', 'sfa_delete_user@1', SEPSFA,
101 'check_tcp', 'check_sys_slice', SEP,
102 'empty_slices', 'ssh_slice_off', 'fill_slices', SEP,
103 'force_gather_logs', SEP,
106 'export', 'show_boxes', SEP,
107 'check_hooks', 'plc_stop', 'vs_start', 'vs_stop', SEP,
108 'delete_initscripts', 'delete_nodegroups','delete_all_sites', SEP,
109 'delete_sites', 'delete_nodes', 'delete_slices', 'keys_clean', SEP,
110 'delete_leases', 'list_leases', SEP,
112 'nodestate_show','nodestate_safeboot','nodestate_boot', SEP,
113 'qemu_list_all', 'qemu_list_mine', 'qemu_kill_all', SEP,
114 'sfa_install_core', 'sfa_install_sfatables', 'sfa_install_plc', 'sfa_install_client', SEPSFA,
115 'sfa_plcclean', 'sfa_dbclean', 'sfa_stop','sfa_uninstall', 'sfi_clean', SEPSFA,
116 'plc_db_dump' , 'plc_db_restore', SEP,
117 'standby_1_through_20',SEP,
121 def printable_steps (list):
122 single_line=" ".join(list)+" "
123 return single_line.replace(" "+SEP+" "," \\\n").replace(" "+SEPSFA+" "," \\\n")
125 def valid_step (step):
126 return step != SEP and step != SEPSFA
128 # turn off the sfa-related steps when build has skipped SFA
129 # this is originally for centos5 as recent SFAs won't build on this platform
131 def check_whether_build_has_sfa (rpms_url):
132 # warning, we're now building 'sface' so let's be a bit more picky
133 retcod=os.system ("curl --silent %s/ | grep -q sfa-"%rpms_url)
134 # full builds are expected to return with 0 here
136 # move all steps containing 'sfa' from default_steps to other_steps
137 sfa_steps= [ step for step in TestPlc.default_steps if step.find('sfa')>=0 ]
138 TestPlc.other_steps += sfa_steps
139 for step in sfa_steps: TestPlc.default_steps.remove(step)
141 def __init__ (self,plc_spec,options):
142 self.plc_spec=plc_spec
144 self.test_ssh=TestSsh(self.plc_spec['host_box'],self.options.buildname)
145 self.vserverip=plc_spec['vserverip']
146 self.vservername=plc_spec['vservername']
147 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
148 self.apiserver=TestApiserver(self.url,options.dry_run)
150 def has_addresses_api (self):
151 return self.apiserver.has_method('AddIpAddress')
154 name=self.plc_spec['name']
155 return "%s.%s"%(name,self.vservername)
158 return self.plc_spec['host_box']
161 return self.test_ssh.is_local()
163 # define the API methods on this object through xmlrpc
164 # would help, but not strictly necessary
168 def actual_command_in_guest (self,command):
169 return self.test_ssh.actual_command(self.host_to_guest(command))
171 def start_guest (self):
172 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host()))
174 def stop_guest (self):
175 return utils.system(self.test_ssh.actual_command(self.stop_guest_in_host()))
177 def run_in_guest (self,command):
178 return utils.system(self.actual_command_in_guest(command))
180 def run_in_host (self,command):
181 return self.test_ssh.run_in_buildname(command)
183 #command gets run in the plc's vm
184 def host_to_guest(self,command):
185 if self.options.plcs_use_lxc:
186 return "ssh -o StrictHostKeyChecking=no %s %s"%(self.vserverip,command)
188 return "vserver %s exec %s"%(self.vservername,command)
190 def vm_root_in_host(self):
191 if self.options.plcs_use_lxc:
192 return "/var/lib/lxc/%s/rootfs/"%(self.vservername)
194 return "/vservers/%s"%(self.vservername)
196 def vm_timestamp_path (self):
197 if self.options.plcs_use_lxc:
198 return "/var/lib/lxc/%s/%s.timestamp"%(self.vservername,self.vservername)
200 return "/vservers/%s.timestamp"%(self.vservername)
202 #start/stop the vserver
203 def start_guest_in_host(self):
204 if self.options.plcs_use_lxc:
205 return "lxc-start --daemon --name=%s"%(self.vservername)
207 return "vserver %s start"%(self.vservername)
209 def stop_guest_in_host(self):
210 if self.options.plcs_use_lxc:
211 return "lxc-stop --name=%s"%(self.vservername)
213 return "vserver %s stop"%(self.vservername)
216 def run_in_guest_piped (self,local,remote):
217 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
219 def yum_check_installed (self, rpms):
220 if isinstance (rpms, list):
222 return self.run_in_guest("rpm -q %s"%rpms)==0
224 # does a yum install in the vs, ignore yum retcod, check with rpm
225 def yum_install (self, rpms):
226 if isinstance (rpms, list):
228 self.run_in_guest("yum -y install %s"%rpms)
229 # yum-complete-transaction comes with yum-utils, that is in vtest.pkgs
230 self.run_in_guest("yum-complete-transaction -y")
231 return self.yum_check_installed (rpms)
233 def auth_root (self):
234 return {'Username':self.plc_spec['PLC_ROOT_USER'],
235 'AuthMethod':'password',
236 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
237 'Role' : self.plc_spec['role']
239 def locate_site (self,sitename):
240 for site in self.plc_spec['sites']:
241 if site['site_fields']['name'] == sitename:
243 if site['site_fields']['login_base'] == sitename:
245 raise Exception,"Cannot locate site %s"%sitename
247 def locate_node (self,nodename):
248 for site in self.plc_spec['sites']:
249 for node in site['nodes']:
250 if node['name'] == nodename:
252 raise Exception,"Cannot locate node %s"%nodename
254 def locate_hostname (self,hostname):
255 for site in self.plc_spec['sites']:
256 for node in site['nodes']:
257 if node['node_fields']['hostname'] == hostname:
259 raise Exception,"Cannot locate hostname %s"%hostname
261 def locate_key (self,keyname):
262 for key in self.plc_spec['keys']:
263 if key['name'] == keyname:
265 raise Exception,"Cannot locate key %s"%keyname
267 def locate_slice (self, slicename):
268 for slice in self.plc_spec['slices']:
269 if slice['slice_fields']['name'] == slicename:
271 raise Exception,"Cannot locate slice %s"%slicename
273 def all_sliver_objs (self):
275 for slice_spec in self.plc_spec['slices']:
276 slicename = slice_spec['slice_fields']['name']
277 for nodename in slice_spec['nodenames']:
278 result.append(self.locate_sliver_obj (nodename,slicename))
281 def locate_sliver_obj (self,nodename,slicename):
282 (site,node) = self.locate_node(nodename)
283 slice = self.locate_slice (slicename)
285 test_site = TestSite (self, site)
286 test_node = TestNode (self, test_site,node)
287 # xxx the slice site is assumed to be the node site - mhh - probably harmless
288 test_slice = TestSlice (self, test_site, slice)
289 return TestSliver (self, test_node, test_slice)
291 def locate_first_node(self):
292 nodename=self.plc_spec['slices'][0]['nodenames'][0]
293 (site,node) = self.locate_node(nodename)
294 test_site = TestSite (self, site)
295 test_node = TestNode (self, test_site,node)
298 def locate_first_sliver (self):
299 slice_spec=self.plc_spec['slices'][0]
300 slicename=slice_spec['slice_fields']['name']
301 nodename=slice_spec['nodenames'][0]
302 return self.locate_sliver_obj(nodename,slicename)
304 # all different hostboxes used in this plc
305 def gather_hostBoxes(self):
306 # maps on sites and nodes, return [ (host_box,test_node) ]
308 for site_spec in self.plc_spec['sites']:
309 test_site = TestSite (self,site_spec)
310 for node_spec in site_spec['nodes']:
311 test_node = TestNode (self, test_site, node_spec)
312 if not test_node.is_real():
313 tuples.append( (test_node.host_box(),test_node) )
314 # transform into a dict { 'host_box' -> [ test_node .. ] }
316 for (box,node) in tuples:
317 if not result.has_key(box):
320 result[box].append(node)
323 # a step for checking this stuff
324 def show_boxes (self):
325 'print summary of nodes location'
326 for (box,nodes) in self.gather_hostBoxes().iteritems():
327 print box,":"," + ".join( [ node.name() for node in nodes ] )
330 # make this a valid step
331 def qemu_kill_all(self):
332 'kill all qemu instances on the qemu boxes involved by this setup'
333 # this is the brute force version, kill all qemus on that host box
334 for (box,nodes) in self.gather_hostBoxes().iteritems():
335 # pass the first nodename, as we don't push template-qemu on testboxes
336 nodedir=nodes[0].nodedir()
337 TestBoxQemu(box,self.options.buildname).qemu_kill_all(nodedir)
340 # make this a valid step
341 def qemu_list_all(self):
342 'list all qemu instances on the qemu boxes involved by this setup'
343 for (box,nodes) in self.gather_hostBoxes().iteritems():
344 # this is the brute force version, kill all qemus on that host box
345 TestBoxQemu(box,self.options.buildname).qemu_list_all()
348 # kill only the right qemus
349 def qemu_list_mine(self):
350 'list qemu instances for our nodes'
351 for (box,nodes) in self.gather_hostBoxes().iteritems():
352 # the fine-grain version
357 # kill only the right qemus
358 def qemu_kill_mine(self):
359 'kill the qemu instances for our nodes'
360 for (box,nodes) in self.gather_hostBoxes().iteritems():
361 # the fine-grain version
366 #################### display config
368 "show test configuration after localization"
374 "print cut'n paste-able stuff to export env variables to your shell"
375 # guess local domain from hostname
376 domain=socket.gethostname().split('.',1)[1]
377 fqdn="%s.%s"%(self.plc_spec['host_box'],domain)
378 print "export BUILD=%s"%self.options.buildname
379 if self.options.plcs_use_lxc:
380 print "export PLCHOSTLXC=%s"%fqdn
382 print "export PLCHOSTVS=%s"%fqdn
383 print "export GUESTNAME=%s"%self.plc_spec['vservername']
384 vplcname=self.plc_spec['vservername'].split('-')[-1]
385 print "export GUESTHOSTNAME=%s.%s"%(vplcname,domain)
386 # find hostname of first node
387 (hostname,qemubox) = self.all_node_infos()[0]
388 print "export KVMHOST=%s.%s"%(qemubox,domain)
389 print "export NODE=%s"%(hostname)
393 always_display_keys=['PLC_WWW_HOST','nodes','sites',]
394 def show_pass (self,passno):
395 for (key,val) in self.plc_spec.iteritems():
396 if not self.options.verbose and key not in TestPlc.always_display_keys: continue
400 self.display_site_spec(site)
401 for node in site['nodes']:
402 self.display_node_spec(node)
403 elif key=='initscripts':
404 for initscript in val:
405 self.display_initscript_spec (initscript)
408 self.display_slice_spec (slice)
411 self.display_key_spec (key)
413 if key not in ['sites','initscripts','slices','keys', 'sfa']:
414 print '+ ',key,':',val
416 def display_site_spec (self,site):
417 print '+ ======== site',site['site_fields']['name']
418 for (k,v) in site.iteritems():
419 if not self.options.verbose and k not in TestPlc.always_display_keys: continue
422 print '+ ','nodes : ',
424 print node['node_fields']['hostname'],'',
430 print user['name'],'',
432 elif k == 'site_fields':
433 print '+ login_base',':',v['login_base']
434 elif k == 'address_fields':
440 def display_initscript_spec (self,initscript):
441 print '+ ======== initscript',initscript['initscript_fields']['name']
443 def display_key_spec (self,key):
444 print '+ ======== key',key['name']
446 def display_slice_spec (self,slice):
447 print '+ ======== slice',slice['slice_fields']['name']
448 for (k,v) in slice.iteritems():
461 elif k=='slice_fields':
462 print '+ fields',':',
463 print 'max_nodes=',v['max_nodes'],
468 def display_node_spec (self,node):
469 print "+ node=%s host_box=%s"%(node['name'],node['host_box']),
470 print "hostname=",node['node_fields']['hostname'],
471 print "ip=",node['interface_fields']['ip']
472 if self.options.verbose:
473 utils.pprint("node details",node,depth=3)
475 # another entry point for just showing the boxes involved
476 def display_mapping (self):
477 TestPlc.display_mapping_plc(self.plc_spec)
481 def display_mapping_plc (plc_spec):
482 print '+ MyPLC',plc_spec['name']
483 # WARNING this would not be right for lxc-based PLC's - should be harmless though
484 print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['host_box'],plc_spec['vservername'])
485 print '+\tIP = %s/%s'%(plc_spec['PLC_API_HOST'],plc_spec['vserverip'])
486 for site_spec in plc_spec['sites']:
487 for node_spec in site_spec['nodes']:
488 TestPlc.display_mapping_node(node_spec)
491 def display_mapping_node (node_spec):
492 print '+ NODE %s'%(node_spec['name'])
493 print '+\tqemu box %s'%node_spec['host_box']
494 print '+\thostname=%s'%node_spec['node_fields']['hostname']
496 # write a timestamp in /vservers/<>.timestamp
497 # cannot be inside the vserver, that causes vserver .. build to cough
498 def timestamp_vs (self):
499 "Create a timestamp to remember creation date for this plc"
501 # TODO-lxc check this one
502 # a first approx. is to store the timestamp close to the VM root like vs does
503 stamp_path=self.vm_timestamp_path ()
504 stamp_dir = os.path.dirname (stamp_path)
505 utils.system(self.test_ssh.actual_command("mkdir -p %s"%stamp_dir))
506 return utils.system(self.test_ssh.actual_command("echo %d > %s"%(now,stamp_path)))==0
508 # this is called inconditionnally at the beginning of the test sequence
509 # just in case this is a rerun, so if the vm is not running it's fine
511 "vserver delete the test myplc"
512 stamp_path=self.vm_timestamp_path()
513 self.run_in_host("rm -f %s"%stamp_path)
514 if self.options.plcs_use_lxc:
515 self.run_in_host("lxc-stop --name %s"%self.vservername)
516 self.run_in_host("lxc-destroy --name %s"%self.vservername)
519 self.run_in_host("vserver --silent %s delete"%self.vservername)
523 # historically the build was being fetched by the tests
524 # now the build pushes itself as a subdir of the tests workdir
525 # so that the tests do not have to worry about extracting the build (svn, git, or whatever)
526 def vs_create (self):
527 "vserver creation (no install done)"
528 # push the local build/ dir to the testplc box
530 # a full path for the local calls
531 build_dir=os.path.dirname(sys.argv[0])
532 # sometimes this is empty - set to "." in such a case
533 if not build_dir: build_dir="."
534 build_dir += "/build"
536 # use a standard name - will be relative to remote buildname
538 # remove for safety; do *not* mkdir first, otherwise we end up with build/build/
539 self.test_ssh.rmdir(build_dir)
540 self.test_ssh.copy(build_dir,recursive=True)
541 # the repo url is taken from arch-rpms-url
542 # with the last step (i386) removed
543 repo_url = self.options.arch_rpms_url
544 for level in [ 'arch' ]:
545 repo_url = os.path.dirname(repo_url)
546 # pass the vbuild-nightly options to vtest-init-vserver
548 test_env_options += " -p %s"%self.options.personality
549 test_env_options += " -d %s"%self.options.pldistro
550 test_env_options += " -f %s"%self.options.fcdistro
551 if self.options.plcs_use_lxc:
552 script="vtest-init-lxc.sh"
554 script="vtest-init-vserver.sh"
555 vserver_name = self.vservername
556 vserver_options="--netdev eth0 --interface %s"%self.vserverip
558 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
559 vserver_options += " --hostname %s"%vserver_hostname
561 print "Cannot reverse lookup %s"%self.vserverip
562 print "This is considered fatal, as this might pollute the test results"
564 create_vserver="%(build_dir)s/%(script)s %(test_env_options)s %(vserver_name)s %(repo_url)s -- %(vserver_options)s"%locals()
565 return self.run_in_host(create_vserver) == 0
568 def plc_install(self):
569 "yum install myplc, noderepo, and the plain bootstrapfs"
571 # workaround for getting pgsql8.2 on centos5
572 if self.options.fcdistro == "centos5":
573 self.run_in_guest("rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-3.noarch.rpm")
576 if self.options.personality == "linux32":
578 elif self.options.personality == "linux64":
581 raise Exception, "Unsupported personality %r"%self.options.personality
582 nodefamily="%s-%s-%s"%(self.options.pldistro,self.options.fcdistro,arch)
585 pkgs_list.append ("slicerepo-%s"%nodefamily)
586 pkgs_list.append ("myplc")
587 pkgs_list.append ("noderepo-%s"%nodefamily)
588 pkgs_list.append ("nodeimage-%s-plain"%nodefamily)
589 pkgs_string=" ".join(pkgs_list)
590 return self.yum_install (pkgs_list)
593 def plc_configure(self):
595 tmpname='%s.plc-config-tty'%(self.name())
596 fileconf=open(tmpname,'w')
597 for var in [ 'PLC_NAME',
602 'PLC_MAIL_SUPPORT_ADDRESS',
605 # Above line was added for integrating SFA Testing
611 'PLC_RESERVATION_GRANULARITY',
613 'PLC_OMF_XMPP_SERVER',
615 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
616 fileconf.write('w\n')
617 fileconf.write('q\n')
619 utils.system('cat %s'%tmpname)
620 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
621 utils.system('rm %s'%tmpname)
626 self.run_in_guest('service plc start')
631 self.run_in_guest('service plc stop')
635 "start the PLC vserver"
640 "stop the PLC vserver"
644 # stores the keys from the config for further use
645 def keys_store(self):
646 "stores test users ssh keys in keys/"
647 for key_spec in self.plc_spec['keys']:
648 TestKey(self,key_spec).store_key()
651 def keys_clean(self):
652 "removes keys cached in keys/"
653 utils.system("rm -rf ./keys")
656 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
657 # for later direct access to the nodes
658 def keys_fetch(self):
659 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
661 if not os.path.isdir(dir):
663 vservername=self.vservername
664 vm_root=self.vm_root_in_host()
666 prefix = 'debug_ssh_key'
667 for ext in [ 'pub', 'rsa' ] :
668 src="%(vm_root)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
669 dst="keys/%(vservername)s-debug.%(ext)s"%locals()
670 if self.test_ssh.fetch(src,dst) != 0: overall=False
674 "create sites with PLCAPI"
675 return self.do_sites()
677 def delete_sites (self):
678 "delete sites with PLCAPI"
679 return self.do_sites(action="delete")
681 def do_sites (self,action="add"):
682 for site_spec in self.plc_spec['sites']:
683 test_site = TestSite (self,site_spec)
684 if (action != "add"):
685 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
686 test_site.delete_site()
687 # deleted with the site
688 #test_site.delete_users()
691 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
692 test_site.create_site()
693 test_site.create_users()
696 def delete_all_sites (self):
697 "Delete all sites in PLC, and related objects"
698 print 'auth_root',self.auth_root()
699 sites = self.apiserver.GetSites(self.auth_root(), {}, ['site_id'])
701 # keep automatic site - otherwise we shoot in our own foot, root_auth is not valid anymore
702 if site['login_base']==self.plc_spec['PLC_SLICE_PREFIX']: continue
703 site_id=site['site_id']
704 print 'Deleting site_id',site_id
705 self.apiserver.DeleteSite(self.auth_root(),site_id)
709 "create nodes with PLCAPI"
710 return self.do_nodes()
711 def delete_nodes (self):
712 "delete nodes with PLCAPI"
713 return self.do_nodes(action="delete")
715 def do_nodes (self,action="add"):
716 for site_spec in self.plc_spec['sites']:
717 test_site = TestSite (self,site_spec)
719 utils.header("Deleting nodes in site %s"%test_site.name())
720 for node_spec in site_spec['nodes']:
721 test_node=TestNode(self,test_site,node_spec)
722 utils.header("Deleting %s"%test_node.name())
723 test_node.delete_node()
725 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
726 for node_spec in site_spec['nodes']:
727 utils.pprint('Creating node %s'%node_spec,node_spec)
728 test_node = TestNode (self,test_site,node_spec)
729 test_node.create_node ()
732 def nodegroups (self):
733 "create nodegroups with PLCAPI"
734 return self.do_nodegroups("add")
735 def delete_nodegroups (self):
736 "delete nodegroups with PLCAPI"
737 return self.do_nodegroups("delete")
741 def translate_timestamp (start,grain,timestamp):
742 if timestamp < TestPlc.YEAR: return start+timestamp*grain
743 else: return timestamp
746 def timestamp_printable (timestamp):
747 return time.strftime('%m-%d %H:%M:%S UTC',time.gmtime(timestamp))
750 "create leases (on reservable nodes only, use e.g. run -c default -c resa)"
752 grain=self.apiserver.GetLeaseGranularity(self.auth_root())
753 print 'API answered grain=',grain
754 start=(now/grain)*grain
756 # find out all nodes that are reservable
757 nodes=self.all_reservable_nodenames()
759 utils.header ("No reservable node found - proceeding without leases")
762 # attach them to the leases as specified in plc_specs
763 # this is where the 'leases' field gets interpreted as relative of absolute
764 for lease_spec in self.plc_spec['leases']:
765 # skip the ones that come with a null slice id
766 if not lease_spec['slice']: continue
767 lease_spec['t_from']=TestPlc.translate_timestamp(start,grain,lease_spec['t_from'])
768 lease_spec['t_until']=TestPlc.translate_timestamp(start,grain,lease_spec['t_until'])
769 lease_addition=self.apiserver.AddLeases(self.auth_root(),nodes,
770 lease_spec['slice'],lease_spec['t_from'],lease_spec['t_until'])
771 if lease_addition['errors']:
772 utils.header("Cannot create leases, %s"%lease_addition['errors'])
775 utils.header('Leases on nodes %r for %s from %d (%s) until %d (%s)'%\
776 (nodes,lease_spec['slice'],
777 lease_spec['t_from'],TestPlc.timestamp_printable(lease_spec['t_from']),
778 lease_spec['t_until'],TestPlc.timestamp_printable(lease_spec['t_until'])))
782 def delete_leases (self):
783 "remove all leases in the myplc side"
784 lease_ids= [ l['lease_id'] for l in self.apiserver.GetLeases(self.auth_root())]
785 utils.header("Cleaning leases %r"%lease_ids)
786 self.apiserver.DeleteLeases(self.auth_root(),lease_ids)
789 def list_leases (self):
790 "list all leases known to the myplc"
791 leases = self.apiserver.GetLeases(self.auth_root())
794 current=l['t_until']>=now
795 if self.options.verbose or current:
796 utils.header("%s %s from %s until %s"%(l['hostname'],l['name'],
797 TestPlc.timestamp_printable(l['t_from']),
798 TestPlc.timestamp_printable(l['t_until'])))
801 # create nodegroups if needed, and populate
802 def do_nodegroups (self, action="add"):
803 # 1st pass to scan contents
805 for site_spec in self.plc_spec['sites']:
806 test_site = TestSite (self,site_spec)
807 for node_spec in site_spec['nodes']:
808 test_node=TestNode (self,test_site,node_spec)
809 if node_spec.has_key('nodegroups'):
810 nodegroupnames=node_spec['nodegroups']
811 if isinstance(nodegroupnames,StringTypes):
812 nodegroupnames = [ nodegroupnames ]
813 for nodegroupname in nodegroupnames:
814 if not groups_dict.has_key(nodegroupname):
815 groups_dict[nodegroupname]=[]
816 groups_dict[nodegroupname].append(test_node.name())
817 auth=self.auth_root()
819 for (nodegroupname,group_nodes) in groups_dict.iteritems():
821 print 'nodegroups:','dealing with nodegroup',nodegroupname,'on nodes',group_nodes
822 # first, check if the nodetagtype is here
823 tag_types = self.apiserver.GetTagTypes(auth,{'tagname':nodegroupname})
825 tag_type_id = tag_types[0]['tag_type_id']
827 tag_type_id = self.apiserver.AddTagType(auth,
828 {'tagname':nodegroupname,
829 'description': 'for nodegroup %s'%nodegroupname,
831 print 'located tag (type)',nodegroupname,'as',tag_type_id
833 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
835 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
836 print 'created nodegroup',nodegroupname,'from tagname',nodegroupname,'and value','yes'
837 # set node tag on all nodes, value='yes'
838 for nodename in group_nodes:
840 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
842 traceback.print_exc()
843 print 'node',nodename,'seems to already have tag',nodegroupname
846 expect_yes = self.apiserver.GetNodeTags(auth,
847 {'hostname':nodename,
848 'tagname':nodegroupname},
849 ['value'])[0]['value']
850 if expect_yes != "yes":
851 print 'Mismatch node tag on node',nodename,'got',expect_yes
854 if not self.options.dry_run:
855 print 'Cannot find tag',nodegroupname,'on node',nodename
859 print 'cleaning nodegroup',nodegroupname
860 self.apiserver.DeleteNodeGroup(auth,nodegroupname)
862 traceback.print_exc()
866 # a list of TestNode objs
867 def all_nodes (self):
869 for site_spec in self.plc_spec['sites']:
870 test_site = TestSite (self,site_spec)
871 for node_spec in site_spec['nodes']:
872 nodes.append(TestNode (self,test_site,node_spec))
875 # return a list of tuples (nodename,qemuname)
876 def all_node_infos (self) :
878 for site_spec in self.plc_spec['sites']:
879 node_infos += [ (node_spec['node_fields']['hostname'],node_spec['host_box']) \
880 for node_spec in site_spec['nodes'] ]
883 def all_nodenames (self): return [ x[0] for x in self.all_node_infos() ]
884 def all_reservable_nodenames (self):
886 for site_spec in self.plc_spec['sites']:
887 for node_spec in site_spec['nodes']:
888 node_fields=node_spec['node_fields']
889 if 'node_type' in node_fields and node_fields['node_type']=='reservable':
890 res.append(node_fields['hostname'])
893 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
894 def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period=15):
895 if self.options.dry_run:
899 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
900 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
901 # the nodes that haven't checked yet - start with a full list and shrink over time
902 tocheck = self.all_hostnames()
903 utils.header("checking nodes %r"%tocheck)
904 # create a dict hostname -> status
905 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
908 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
910 for array in tocheck_status:
911 hostname=array['hostname']
912 boot_state=array['boot_state']
913 if boot_state == target_boot_state:
914 utils.header ("%s has reached the %s state"%(hostname,target_boot_state))
916 # if it's a real node, never mind
917 (site_spec,node_spec)=self.locate_hostname(hostname)
918 if TestNode.is_real_model(node_spec['node_fields']['model']):
919 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
921 boot_state = target_boot_state
922 elif datetime.datetime.now() > graceout:
923 utils.header ("%s still in '%s' state"%(hostname,boot_state))
924 graceout=datetime.datetime.now()+datetime.timedelta(1)
925 status[hostname] = boot_state
927 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != target_boot_state ]
930 if datetime.datetime.now() > timeout:
931 for hostname in tocheck:
932 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
934 # otherwise, sleep for a while
936 # only useful in empty plcs
939 def nodes_booted(self):
940 return self.nodes_check_boot_state('boot',timeout_minutes=30,silent_minutes=28)
942 def check_nodes_ssh(self,debug,timeout_minutes,silent_minutes,period=15):
944 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
945 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
946 vservername=self.vservername
949 local_key = "keys/%(vservername)s-debug.rsa"%locals()
952 local_key = "keys/key1.rsa"
953 node_infos = self.all_node_infos()
954 utils.header("checking ssh access (expected in %s mode) to nodes:"%message)
955 for (nodename,qemuname) in node_infos:
956 utils.header("hostname=%s -- qemubox=%s"%(nodename,qemuname))
957 utils.header("max timeout is %d minutes, silent for %d minutes (period is %s)"%\
958 (timeout_minutes,silent_minutes,period))
960 for node_info in node_infos:
961 (hostname,qemuname) = node_info
962 # try to run 'hostname' in the node
963 command = TestSsh (hostname,key=local_key).actual_command("hostname;uname -a")
964 # don't spam logs - show the command only after the grace period
965 success = utils.system ( command, silent=datetime.datetime.now() < graceout)
967 utils.header('Successfully entered root@%s (%s)'%(hostname,message))
969 node_infos.remove(node_info)
971 # we will have tried real nodes once, in case they're up - but if not, just skip
972 (site_spec,node_spec)=self.locate_hostname(hostname)
973 if TestNode.is_real_model(node_spec['node_fields']['model']):
974 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
975 node_infos.remove(node_info)
978 if datetime.datetime.now() > timeout:
979 for (hostname,qemuname) in node_infos:
980 utils.header("FAILURE to ssh into %s (on %s)"%(hostname,qemuname))
982 # otherwise, sleep for a while
984 # only useful in empty plcs
987 def ssh_node_debug(self):
988 "Tries to ssh into nodes in debug mode with the debug ssh key"
989 return self.check_nodes_ssh(debug=True,timeout_minutes=10,silent_minutes=8)
991 def ssh_node_boot(self):
992 "Tries to ssh into nodes in production mode with the root ssh key"
993 return self.check_nodes_ssh(debug=False,timeout_minutes=40,silent_minutes=38)
996 def qemu_local_init (self): pass
998 def bootcd (self): pass
1000 def qemu_local_config (self): pass
1002 def nodestate_reinstall (self): pass
1004 def nodestate_safeboot (self): pass
1006 def nodestate_boot (self): pass
1008 def nodestate_show (self): pass
1010 def qemu_export (self): pass
1012 ### check hooks : invoke scripts from hooks/{node,slice}
1013 def check_hooks_node (self):
1014 return self.locate_first_node().check_hooks()
1015 def check_hooks_sliver (self) :
1016 return self.locate_first_sliver().check_hooks()
1018 def check_hooks (self):
1019 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
1020 return self.check_hooks_node() and self.check_hooks_sliver()
1023 def do_check_initscripts(self):
1025 for slice_spec in self.plc_spec['slices']:
1026 if not slice_spec.has_key('initscriptstamp'):
1028 stamp=slice_spec['initscriptstamp']
1029 for nodename in slice_spec['nodenames']:
1030 (site,node) = self.locate_node (nodename)
1031 # xxx - passing the wrong site - probably harmless
1032 test_site = TestSite (self,site)
1033 test_slice = TestSlice (self,test_site,slice_spec)
1034 test_node = TestNode (self,test_site,node)
1035 test_sliver = TestSliver (self, test_node, test_slice)
1036 if not test_sliver.check_initscript_stamp(stamp):
1040 def check_initscripts(self):
1041 "check that the initscripts have triggered"
1042 return self.do_check_initscripts()
1044 def initscripts (self):
1045 "create initscripts with PLCAPI"
1046 for initscript in self.plc_spec['initscripts']:
1047 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
1048 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
1051 def delete_initscripts (self):
1052 "delete initscripts with PLCAPI"
1053 for initscript in self.plc_spec['initscripts']:
1054 initscript_name = initscript['initscript_fields']['name']
1055 print('Attempting to delete %s in plc %s'%(initscript_name,self.plc_spec['name']))
1057 self.apiserver.DeleteInitScript(self.auth_root(),initscript_name)
1058 print initscript_name,'deleted'
1060 print 'deletion went wrong - probably did not exist'
1065 "create slices with PLCAPI"
1066 return self.do_slices(action="add")
1068 def delete_slices (self):
1069 "delete slices with PLCAPI"
1070 return self.do_slices(action="delete")
1072 def fill_slices (self):
1073 "add nodes in slices with PLCAPI"
1074 return self.do_slices(action="fill")
1076 def empty_slices (self):
1077 "remove nodes from slices with PLCAPI"
1078 return self.do_slices(action="empty")
1080 def do_slices (self, action="add"):
1081 for slice in self.plc_spec['slices']:
1082 site_spec = self.locate_site (slice['sitename'])
1083 test_site = TestSite(self,site_spec)
1084 test_slice=TestSlice(self,test_site,slice)
1085 if action == "delete":
1086 test_slice.delete_slice()
1087 elif action=="fill":
1088 test_slice.add_nodes()
1089 elif action=="empty":
1090 test_slice.delete_nodes()
1092 test_slice.create_slice()
1096 def ssh_slice(self): pass
1098 def ssh_slice_off (self): pass
1101 def keys_clear_known_hosts (self): pass
1103 def speed_up_slices (self):
1104 "tweak nodemanager settings on all nodes using a conf file"
1105 # create the template on the server-side
1106 template="%s.nodemanager"%self.name()
1107 template_file = open (template,"w")
1108 template_file.write('OPTIONS="-p 30 -r 11 -d"\n')
1109 template_file.close()
1110 in_vm="/var/www/html/PlanetLabConf/nodemanager"
1111 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1112 self.test_ssh.copy_abs(template,remote)
1114 self.apiserver.AddConfFile (self.auth_root(),
1115 {'dest':'/etc/sysconfig/nodemanager',
1116 'source':'PlanetLabConf/nodemanager',
1117 'postinstall_cmd':'service nm restart',})
1121 def qemu_start (self) : pass
1124 def timestamp_qemu (self) : pass
1126 def check_tcp (self):
1127 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
1128 specs = self.plc_spec['tcp_test']
1133 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
1134 if not s_test_sliver.run_tcp_server(port,timeout=10):
1138 # idem for the client side
1139 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
1140 if not c_test_sliver.run_tcp_client(s_test_sliver.test_node.name(),port):
1144 # painfully enough, we need to allow for some time as netflow might show up last
1145 def check_sys_slice (self):
1146 "all nodes: check that a system slice is alive"
1147 # would probably make more sense to check for netflow,
1148 # but that one is currently not working in the lxc distro
1149 # return self.check_systemslice ('netflow')
1150 return self.check_systemslice ('drl')
1152 # we have the slices up already here, so it should not take too long
1153 def check_systemslice (self, slicename, timeout_minutes=5, period=15):
1154 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
1155 test_nodes=self.all_nodes()
1157 for test_node in test_nodes:
1158 if test_node.check_systemslice (slicename,dry_run=self.options.dry_run):
1160 test_nodes.remove(test_node)
1165 if datetime.datetime.now () > timeout:
1166 for test_node in test_nodes:
1167 utils.header ("can't find system slice %s in %s"%(slicename,test_node.name()))
1172 def plcsh_stress_test (self):
1173 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
1174 # install the stress-test in the plc image
1175 location = "/usr/share/plc_api/plcsh_stress_test.py"
1176 remote="%s/%s"%(self.vm_root_in_host(),location)
1177 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1179 command += " -- --check"
1180 if self.options.size == 1:
1181 command += " --tiny"
1182 return ( self.run_in_guest(command) == 0)
1184 # populate runs the same utility without slightly different options
1185 # in particular runs with --preserve (dont cleanup) and without --check
1186 # also it gets run twice, once with the --foreign option for creating fake foreign entries
1188 def sfa_install_all (self):
1189 "yum install sfa sfa-plc sfa-sfatables sfa-client"
1190 return self.yum_install ("sfa sfa-plc sfa-sfatables sfa-client")
1192 def sfa_install_core(self):
1194 return self.yum_install ("sfa")
1196 def sfa_install_plc(self):
1197 "yum install sfa-plc"
1198 return self.yum_install("sfa-plc")
1200 def sfa_install_sfatables(self):
1201 "yum install sfa-sfatables"
1202 return self.yum_install ("sfa-sfatables")
1204 # for some very odd reason, this sometimes fails with the following symptom
1205 # # yum install sfa-client
1206 # Setting up Install Process
1208 # Downloading Packages:
1209 # Running rpm_check_debug
1210 # Running Transaction Test
1211 # Transaction Test Succeeded
1212 # Running Transaction
1213 # Transaction couldn't start:
1214 # installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem
1215 # [('installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem', (9, '/', 69632L))]
1216 # even though in the same context I have
1217 # [2012.05.23--f14-32-sfastd1-1-vplc07] / # df -h
1218 # Filesystem Size Used Avail Use% Mounted on
1219 # /dev/hdv1 806G 264G 501G 35% /
1220 # none 16M 36K 16M 1% /tmp
1222 # so as a workaround, we first try yum install, and then invoke rpm on the cached rpm...
1223 def sfa_install_client(self):
1224 "yum install sfa-client"
1225 first_try=self.yum_install("sfa-client")
1226 if first_try: return True
1227 utils.header ("********** Regular yum failed - special workaround in place, 2nd chance")
1228 (code,cached_rpm_path)=utils.output_of(self.actual_command_in_guest('find /var/cache/yum -name sfa-client\*.rpm'))
1229 utils.header("rpm_path=<<%s>>"%rpm_path)
1231 self.run_in_guest("rpm -i %s"%cached_rpm_path)
1232 return self.yum_check_installed ("sfa-client")
1234 def sfa_dbclean(self):
1235 "thoroughly wipes off the SFA database"
1236 return self.run_in_guest("sfaadmin.py registry nuke")==0 or \
1237 self.run_in_guest("sfa-nuke.py")==0 or \
1238 self.run_in_guest("sfa-nuke-plc.py")==0
1240 def sfa_plcclean(self):
1241 "cleans the PLC entries that were created as a side effect of running the script"
1243 sfa_spec=self.plc_spec['sfa']
1245 for sfa_slice_spec in sfa_spec['sfa_slice_specs']:
1246 login_base=sfa_slice_spec['login_base']
1247 try: self.apiserver.DeleteSite (self.auth_root(),login_base)
1248 except: print "Site %s already absent from PLC db"%login_base
1250 for key in ['piuser','regularuser']:
1251 username="%s@%s"%(sfa_slice_spec[key],sfa_slice_spec['domain'])
1252 try: self.apiserver.DeletePerson(self.auth_root(),username)
1253 except: print "User %s already absent from PLC db"%username
1255 print "REMEMBER TO RUN sfa_import AGAIN"
1258 def sfa_uninstall(self):
1259 "uses rpm to uninstall sfa - ignore result"
1260 self.run_in_guest("rpm -e sfa sfa-sfatables sfa-client sfa-plc")
1261 self.run_in_guest("rm -rf /var/lib/sfa")
1262 self.run_in_guest("rm -rf /etc/sfa")
1263 self.run_in_guest("rm -rf /var/log/sfa_access.log /var/log/sfa_import_plc.log /var/log/sfa.daemon")
1265 self.run_in_guest("rpm -e --noscripts sfa-plc")
1268 ### run unit tests for SFA
1269 # NOTE: for some reason on f14/i386, yum install sfa-tests fails for no reason
1270 # Running Transaction
1271 # Transaction couldn't start:
1272 # installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem
1273 # [('installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem', (9, '/', 208896L))]
1274 # no matter how many Gbs are available on the testplc
1275 # could not figure out what's wrong, so...
1276 # if the yum install phase fails, consider the test is successful
1277 # other combinations will eventually run it hopefully
1278 def sfa_utest(self):
1279 "yum install sfa-tests and run SFA unittests"
1280 self.run_in_guest("yum -y install sfa-tests")
1281 # failed to install - forget it
1282 if self.run_in_guest("rpm -q sfa-tests")!=0:
1283 utils.header("WARNING: SFA unit tests failed to install, ignoring")
1285 return self.run_in_guest("/usr/share/sfa/tests/testAll.py")==0
1289 dirname="conf.%s"%self.plc_spec['name']
1290 if not os.path.isdir(dirname):
1291 utils.system("mkdir -p %s"%dirname)
1292 if not os.path.isdir(dirname):
1293 raise "Cannot create config dir for plc %s"%self.name()
1296 def conffile(self,filename):
1297 return "%s/%s"%(self.confdir(),filename)
1298 def confsubdir(self,dirname,clean,dry_run=False):
1299 subdirname="%s/%s"%(self.confdir(),dirname)
1301 utils.system("rm -rf %s"%subdirname)
1302 if not os.path.isdir(subdirname):
1303 utils.system("mkdir -p %s"%subdirname)
1304 if not dry_run and not os.path.isdir(subdirname):
1305 raise "Cannot create config subdir %s for plc %s"%(dirname,self.name())
1308 def conffile_clean (self,filename):
1309 filename=self.conffile(filename)
1310 return utils.system("rm -rf %s"%filename)==0
1313 def sfa_configure(self):
1314 "run sfa-config-tty"
1315 tmpname=self.conffile("sfa-config-tty")
1316 fileconf=open(tmpname,'w')
1317 for var in [ 'SFA_REGISTRY_ROOT_AUTH',
1318 'SFA_INTERFACE_HRN',
1319 'SFA_REGISTRY_LEVEL1_AUTH',
1320 'SFA_REGISTRY_HOST',
1321 'SFA_AGGREGATE_HOST',
1331 'SFA_GENERIC_FLAVOUR',
1332 'SFA_AGGREGATE_ENABLED',
1334 if self.plc_spec['sfa'].has_key(var):
1335 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec['sfa'][var]))
1336 # the way plc_config handles booleans just sucks..
1339 if self.plc_spec['sfa'][var]: val='true'
1340 fileconf.write ('e %s\n%s\n'%(var,val))
1341 fileconf.write('w\n')
1342 fileconf.write('R\n')
1343 fileconf.write('q\n')
1345 utils.system('cat %s'%tmpname)
1346 self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
1349 def aggregate_xml_line(self):
1350 port=self.plc_spec['sfa']['neighbours-port']
1351 return '<aggregate addr="%s" hrn="%s" port="%r"/>' % \
1352 (self.vserverip,self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH'],port)
1354 def registry_xml_line(self):
1355 return '<registry addr="%s" hrn="%s" port="12345"/>' % \
1356 (self.vserverip,self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH'])
1359 # a cross step that takes all other plcs in argument
1360 def cross_sfa_configure(self, other_plcs):
1361 "writes aggregates.xml and registries.xml that point to all other PLCs in the test"
1362 # of course with a single plc, other_plcs is an empty list
1365 agg_fname=self.conffile("agg.xml")
1366 file(agg_fname,"w").write("<aggregates>%s</aggregates>\n" % \
1367 " ".join([ plc.aggregate_xml_line() for plc in other_plcs ]))
1368 utils.header ("(Over)wrote %s"%agg_fname)
1369 reg_fname=self.conffile("reg.xml")
1370 file(reg_fname,"w").write("<registries>%s</registries>\n" % \
1371 " ".join([ plc.registry_xml_line() for plc in other_plcs ]))
1372 utils.header ("(Over)wrote %s"%reg_fname)
1373 return self.test_ssh.copy_abs(agg_fname,'/%s/etc/sfa/aggregates.xml'%self.vm_root_in_host())==0 \
1374 and self.test_ssh.copy_abs(reg_fname,'/%s/etc/sfa/registries.xml'%self.vm_root_in_host())==0
1376 def sfa_import(self):
1378 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1379 return self.run_in_guest('sfa-import.py')==0 or \
1380 self.run_in_guest('sfa-import-plc.py')==0 or \
1381 self.run_in_guest('sfaadmin.py registry import_registry')==0
1382 # not needed anymore
1383 # self.run_in_guest('cp /etc/sfa/authorities/%s/%s.pkey /etc/sfa/authorities/server.key'%(auth,auth))
1385 def sfa_start(self):
1387 return self.run_in_guest('service sfa start')==0
1389 def sfi_configure(self):
1390 "Create /root/sfi on the plc side for sfi client configuration"
1391 if self.options.dry_run:
1392 utils.header("DRY RUN - skipping step")
1394 sfa_spec=self.plc_spec['sfa']
1395 # cannot use slice_sfa_mapper to pass dir_name
1396 for slice_spec in self.plc_spec['sfa']['sfa_slice_specs']:
1397 test_slice=TestSliceSfa(self,slice_spec)
1398 dir_basename=os.path.basename(test_slice.sfi_path())
1399 dir_name=self.confsubdir("dot-sfi/%s"%dir_basename,clean=True,dry_run=self.options.dry_run)
1400 test_slice.sfi_config(dir_name)
1401 # push into the remote /root/sfi area
1402 location = test_slice.sfi_path()
1403 remote="%s/%s"%(self.vm_root_in_host(),location)
1404 self.test_ssh.mkdir(remote,abs=True)
1405 # need to strip last level or remote otherwise we get an extra dir level
1406 self.test_ssh.copy_abs(dir_name, os.path.dirname(remote), recursive=True)
1410 def sfi_clean (self):
1411 "clean up /root/sfi on the plc side"
1412 self.run_in_guest("rm -rf /root/sfi")
1416 def sfa_add_site (self): pass
1418 def sfa_add_pi (self): pass
1420 def sfa_add_user(self): pass
1422 def sfa_update_user(self): pass
1424 def sfa_add_slice(self): pass
1426 def sfa_discover(self): pass
1428 def sfa_create_slice(self): pass
1430 def sfa_check_slice_plc(self): pass
1432 def sfa_update_slice(self): pass
1434 def sfi_list(self): pass
1436 def sfi_show(self): pass
1438 def sfi_slices(self): pass
1440 def ssh_slice_sfa(self): pass
1442 def sfa_delete_user(self): pass
1444 def sfa_delete_slice(self): pass
1448 self.run_in_guest('service sfa stop')==0
1451 def populate (self):
1452 "creates random entries in the PLCAPI"
1453 # install the stress-test in the plc image
1454 location = "/usr/share/plc_api/plcsh_stress_test.py"
1455 remote="%s/%s"%(self.vm_root_in_host(),location)
1456 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1458 command += " -- --preserve --short-names"
1459 local = (self.run_in_guest(command) == 0);
1460 # second run with --foreign
1461 command += ' --foreign'
1462 remote = (self.run_in_guest(command) == 0);
1463 return ( local and remote)
1465 def gather_logs (self):
1466 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1467 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1468 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1469 # (1.c) get the plc's /root/sfi -> logs/sfi.<plcname>/
1470 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1471 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1472 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1474 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1475 self.gather_var_logs ()
1477 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1478 self.gather_pgsql_logs ()
1480 print "-------------------- TestPlc.gather_logs : PLC's /root/sfi/"
1481 self.gather_root_sfi ()
1483 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1484 for site_spec in self.plc_spec['sites']:
1485 test_site = TestSite (self,site_spec)
1486 for node_spec in site_spec['nodes']:
1487 test_node=TestNode(self,test_site,node_spec)
1488 test_node.gather_qemu_logs()
1490 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1491 self.gather_nodes_var_logs()
1493 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1494 self.gather_slivers_var_logs()
1497 def gather_slivers_var_logs(self):
1498 for test_sliver in self.all_sliver_objs():
1499 remote = test_sliver.tar_var_logs()
1500 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
1501 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
1502 utils.system(command)
1505 def gather_var_logs (self):
1506 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
1507 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1508 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
1509 utils.system(command)
1510 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd"%self.name()
1511 utils.system(command)
1513 def gather_pgsql_logs (self):
1514 utils.system("mkdir -p logs/myplc.pgsql-log.%s"%self.name())
1515 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1516 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -"%self.name()
1517 utils.system(command)
1519 def gather_root_sfi (self):
1520 utils.system("mkdir -p logs/sfi.%s"%self.name())
1521 to_plc = self.actual_command_in_guest("tar -C /root/sfi/ -cf - .")
1522 command = to_plc + "| tar -C logs/sfi.%s -xf -"%self.name()
1523 utils.system(command)
1525 def gather_nodes_var_logs (self):
1526 for site_spec in self.plc_spec['sites']:
1527 test_site = TestSite (self,site_spec)
1528 for node_spec in site_spec['nodes']:
1529 test_node=TestNode(self,test_site,node_spec)
1530 test_ssh = TestSsh (test_node.name(),key="keys/key1.rsa")
1531 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1532 command = command + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
1533 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
1534 utils.system(command)
1537 # returns the filename to use for sql dump/restore, using options.dbname if set
1538 def dbfile (self, database):
1539 # uses options.dbname if it is found
1541 name=self.options.dbname
1542 if not isinstance(name,StringTypes):
1545 t=datetime.datetime.now()
1548 return "/root/%s-%s.sql"%(database,name)
1550 def plc_db_dump(self):
1551 'dump the planetlab5 DB in /root in the PLC - filename has time'
1552 dump=self.dbfile("planetab5")
1553 self.run_in_guest('pg_dump -U pgsqluser planetlab5 -f '+ dump)
1554 utils.header('Dumped planetlab5 database in %s'%dump)
1557 def plc_db_restore(self):
1558 'restore the planetlab5 DB - looks broken, but run -n might help'
1559 dump=self.dbfile("planetab5")
1560 ##stop httpd service
1561 self.run_in_guest('service httpd stop')
1562 # xxx - need another wrapper
1563 self.run_in_guest_piped('echo drop database planetlab5','psql --user=pgsqluser template1')
1564 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab5')
1565 self.run_in_guest('psql -U pgsqluser planetlab5 -f '+dump)
1566 ##starting httpd service
1567 self.run_in_guest('service httpd start')
1569 utils.header('Database restored from ' + dump)
1571 def standby_1_through_20(self):
1572 """convenience function to wait for a specified number of minutes"""
1575 def standby_1(): pass
1577 def standby_2(): pass
1579 def standby_3(): pass
1581 def standby_4(): pass
1583 def standby_5(): pass
1585 def standby_6(): pass
1587 def standby_7(): pass
1589 def standby_8(): pass
1591 def standby_9(): pass
1593 def standby_10(): pass
1595 def standby_11(): pass
1597 def standby_12(): pass
1599 def standby_13(): pass
1601 def standby_14(): pass
1603 def standby_15(): pass
1605 def standby_16(): pass
1607 def standby_17(): pass
1609 def standby_18(): pass
1611 def standby_19(): pass
1613 def standby_20(): pass