1 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
2 # Copyright (C) 2010 INRIA
9 from types import StringTypes
13 from TestSite import TestSite
14 from TestNode import TestNode
15 from TestUser import TestUser
16 from TestKey import TestKey
17 from TestSlice import TestSlice
18 from TestSliver import TestSliver
19 from TestBoxQemu import TestBoxQemu
20 from TestSsh import TestSsh
21 from TestApiserver import TestApiserver
22 from TestSliceSfa import TestSliceSfa
24 # step methods must take (self) and return a boolean (options is a member of the class)
26 def standby(minutes,dry_run):
27 utils.header('Entering StandBy for %d mn'%minutes)
31 time.sleep(60*minutes)
34 def standby_generic (func):
36 minutes=int(func.__name__.split("_")[1])
37 return standby(minutes,self.options.dry_run)
40 def node_mapper (method):
41 def actual(self,*args, **kwds):
43 node_method = TestNode.__dict__[method.__name__]
44 for test_node in self.all_nodes():
45 if not node_method(test_node, *args, **kwds): overall=False
47 # restore the doc text
48 actual.__doc__=method.__doc__
51 def slice_mapper (method):
54 slice_method = TestSlice.__dict__[method.__name__]
55 for slice_spec in self.plc_spec['slices']:
56 site_spec = self.locate_site (slice_spec['sitename'])
57 test_site = TestSite(self,site_spec)
58 test_slice=TestSlice(self,test_site,slice_spec)
59 if not slice_method(test_slice,self.options): overall=False
61 # restore the doc text
62 actual.__doc__=method.__doc__
65 def slice_sfa_mapper (method):
68 slice_method = TestSliceSfa.__dict__[method.__name__]
69 for slice_spec in self.plc_spec['sfa']['sfa_slice_specs']:
70 test_slice=TestSliceSfa(self,slice_spec)
71 if not slice_method(test_slice,self.options): overall=False
73 # restore the doc text
74 actual.__doc__=method.__doc__
84 'vs_delete','timestamp_vs','vs_create', SEP,
85 'plc_install', 'plc_configure', 'plc_start', SEP,
86 'keys_fetch', 'keys_store', 'keys_clear_known_hosts', 'speed_up_slices', SEP,
87 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', 'leases', SEP,
88 'nodestate_reinstall', 'qemu_local_init','bootcd', 'qemu_local_config', SEP,
89 'qemu_export', 'qemu_kill_mine', 'qemu_start', 'timestamp_qemu', SEP,
90 'sfa_install_all', 'sfa_configure', 'cross_sfa_configure', 'sfa_start', 'sfa_import', SEPSFA,
91 'sfi_configure@1', 'sfa_add_site@1','sfa_add_pi@1', SEPSFA,
92 'sfa_add_user@1', 'sfa_add_slice@1', 'sfa_discover@1', SEPSFA,
93 'sfa_create_slice@1', 'sfa_check_slice_plc@1', SEPSFA,
94 'sfa_update_user@1', 'sfa_update_slice@1', 'sfa_view@1', 'sfa_utest@1',SEPSFA,
95 # we used to run plcsh_stress_test, and then ssh_node_debug and ssh_node_boot
96 # but as the stress test might take a while, we sometimes missed the debug mode..
97 'ssh_node_debug@1', 'plcsh_stress_test@1', SEP,
98 'ssh_node_boot@1', 'ssh_slice', 'check_initscripts', SEP,
99 'ssh_slice_sfa@1', 'sfa_delete_slice@1', 'sfa_delete_user@1', SEPSFA,
100 'check_tcp', 'check_sys_slice', SEP,
101 'empty_slices', 'ssh_slice_off', 'fill_slices', SEP,
102 'force_gather_logs', SEP,
105 'export', 'show_boxes', SEP,
106 'check_hooks', 'plc_stop', 'vs_start', 'vs_stop', SEP,
107 'delete_initscripts', 'delete_nodegroups','delete_all_sites', SEP,
108 'delete_sites', 'delete_nodes', 'delete_slices', 'keys_clean', SEP,
109 'delete_leases', 'list_leases', SEP,
111 'nodestate_show','nodestate_safeboot','nodestate_boot', SEP,
112 'qemu_list_all', 'qemu_list_mine', 'qemu_kill_all', SEP,
113 'sfa_install_core', 'sfa_install_sfatables', 'sfa_install_plc', 'sfa_install_client', SEPSFA,
114 'sfa_plcclean', 'sfa_dbclean', 'sfa_stop','sfa_uninstall', 'sfi_clean', SEPSFA,
115 'plc_db_dump' , 'plc_db_restore', SEP,
116 'standby_1_through_20',SEP,
120 def printable_steps (list):
121 single_line=" ".join(list)+" "
122 return single_line.replace(" "+SEP+" "," \\\n").replace(" "+SEPSFA+" "," \\\n")
124 def valid_step (step):
125 return step != SEP and step != SEPSFA
127 # turn off the sfa-related steps when build has skipped SFA
128 # this is originally for centos5 as recent SFAs won't build on this platform
130 def check_whether_build_has_sfa (rpms_url):
131 # warning, we're now building 'sface' so let's be a bit more picky
132 retcod=os.system ("curl --silent %s/ | grep -q sfa-"%rpms_url)
133 # full builds are expected to return with 0 here
135 # move all steps containing 'sfa' from default_steps to other_steps
136 sfa_steps= [ step for step in TestPlc.default_steps if step.find('sfa')>=0 ]
137 TestPlc.other_steps += sfa_steps
138 for step in sfa_steps: TestPlc.default_steps.remove(step)
140 def __init__ (self,plc_spec,options):
141 self.plc_spec=plc_spec
143 self.test_ssh=TestSsh(self.plc_spec['host_box'],self.options.buildname)
144 self.vserverip=plc_spec['vserverip']
145 self.vservername=plc_spec['vservername']
146 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
147 self.apiserver=TestApiserver(self.url,options.dry_run)
149 def has_addresses_api (self):
150 return self.apiserver.has_method('AddIpAddress')
153 name=self.plc_spec['name']
154 return "%s.%s"%(name,self.vservername)
157 return self.plc_spec['host_box']
160 return self.test_ssh.is_local()
162 # define the API methods on this object through xmlrpc
163 # would help, but not strictly necessary
167 def actual_command_in_guest (self,command):
168 return self.test_ssh.actual_command(self.host_to_guest(command))
170 def start_guest (self):
171 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host()))
173 def stop_guest (self):
174 return utils.system(self.test_ssh.actual_command(self.stop_guest_in_host()))
176 def run_in_guest (self,command):
177 return utils.system(self.actual_command_in_guest(command))
179 def run_in_host (self,command):
180 return self.test_ssh.run_in_buildname(command)
182 #command gets run in the plc's vm
183 def host_to_guest(self,command):
184 if self.options.plcs_use_lxc:
185 return "ssh -o StrictHostKeyChecking=no %s %s"%(self.vserverip,command)
187 return "vserver %s exec %s"%(self.vservername,command)
189 def vm_root_in_host(self):
190 if self.options.plcs_use_lxc:
191 return "/var/lib/lxc/%s/rootfs/"%(self.vservername)
193 return "/vservers/%s"%(self.vservername)
195 def vm_timestamp_path (self):
196 if self.options.plcs_use_lxc:
197 return "/var/lib/lxc/%s/%s.timestamp"%(self.vservername,self.vservername)
199 return "/vservers/%s.timestamp"%(self.vservername)
201 #start/stop the vserver
202 def start_guest_in_host(self):
203 if self.options.plcs_use_lxc:
204 return "lxc-start --daemon --name=%s"%(self.vservername)
206 return "vserver %s start"%(self.vservername)
208 def stop_guest_in_host(self):
209 if self.options.plcs_use_lxc:
210 return "lxc-stop --name=%s"%(self.vservername)
212 return "vserver %s stop"%(self.vservername)
215 def run_in_guest_piped (self,local,remote):
216 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
218 def yum_check_installed (self, rpms):
219 if isinstance (rpms, list):
221 return self.run_in_guest("rpm -q %s"%rpms)==0
223 # does a yum install in the vs, ignore yum retcod, check with rpm
224 def yum_install (self, rpms):
225 if isinstance (rpms, list):
227 self.run_in_guest("yum -y install %s"%rpms)
228 # yum-complete-transaction comes with yum-utils, that is in vtest.pkgs
229 self.run_in_guest("yum-complete-transaction -y")
230 return self.yum_check_installed (rpms)
232 def auth_root (self):
233 return {'Username':self.plc_spec['PLC_ROOT_USER'],
234 'AuthMethod':'password',
235 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
236 'Role' : self.plc_spec['role']
238 def locate_site (self,sitename):
239 for site in self.plc_spec['sites']:
240 if site['site_fields']['name'] == sitename:
242 if site['site_fields']['login_base'] == sitename:
244 raise Exception,"Cannot locate site %s"%sitename
246 def locate_node (self,nodename):
247 for site in self.plc_spec['sites']:
248 for node in site['nodes']:
249 if node['name'] == nodename:
251 raise Exception,"Cannot locate node %s"%nodename
253 def locate_hostname (self,hostname):
254 for site in self.plc_spec['sites']:
255 for node in site['nodes']:
256 if node['node_fields']['hostname'] == hostname:
258 raise Exception,"Cannot locate hostname %s"%hostname
260 def locate_key (self,keyname):
261 for key in self.plc_spec['keys']:
262 if key['name'] == keyname:
264 raise Exception,"Cannot locate key %s"%keyname
266 def locate_slice (self, slicename):
267 for slice in self.plc_spec['slices']:
268 if slice['slice_fields']['name'] == slicename:
270 raise Exception,"Cannot locate slice %s"%slicename
272 def all_sliver_objs (self):
274 for slice_spec in self.plc_spec['slices']:
275 slicename = slice_spec['slice_fields']['name']
276 for nodename in slice_spec['nodenames']:
277 result.append(self.locate_sliver_obj (nodename,slicename))
280 def locate_sliver_obj (self,nodename,slicename):
281 (site,node) = self.locate_node(nodename)
282 slice = self.locate_slice (slicename)
284 test_site = TestSite (self, site)
285 test_node = TestNode (self, test_site,node)
286 # xxx the slice site is assumed to be the node site - mhh - probably harmless
287 test_slice = TestSlice (self, test_site, slice)
288 return TestSliver (self, test_node, test_slice)
290 def locate_first_node(self):
291 nodename=self.plc_spec['slices'][0]['nodenames'][0]
292 (site,node) = self.locate_node(nodename)
293 test_site = TestSite (self, site)
294 test_node = TestNode (self, test_site,node)
297 def locate_first_sliver (self):
298 slice_spec=self.plc_spec['slices'][0]
299 slicename=slice_spec['slice_fields']['name']
300 nodename=slice_spec['nodenames'][0]
301 return self.locate_sliver_obj(nodename,slicename)
303 # all different hostboxes used in this plc
304 def gather_hostBoxes(self):
305 # maps on sites and nodes, return [ (host_box,test_node) ]
307 for site_spec in self.plc_spec['sites']:
308 test_site = TestSite (self,site_spec)
309 for node_spec in site_spec['nodes']:
310 test_node = TestNode (self, test_site, node_spec)
311 if not test_node.is_real():
312 tuples.append( (test_node.host_box(),test_node) )
313 # transform into a dict { 'host_box' -> [ test_node .. ] }
315 for (box,node) in tuples:
316 if not result.has_key(box):
319 result[box].append(node)
322 # a step for checking this stuff
323 def show_boxes (self):
324 'print summary of nodes location'
325 for (box,nodes) in self.gather_hostBoxes().iteritems():
326 print box,":"," + ".join( [ node.name() for node in nodes ] )
329 # make this a valid step
330 def qemu_kill_all(self):
331 'kill all qemu instances on the qemu boxes involved by this setup'
332 # this is the brute force version, kill all qemus on that host box
333 for (box,nodes) in self.gather_hostBoxes().iteritems():
334 # pass the first nodename, as we don't push template-qemu on testboxes
335 nodedir=nodes[0].nodedir()
336 TestBoxQemu(box,self.options.buildname).qemu_kill_all(nodedir)
339 # make this a valid step
340 def qemu_list_all(self):
341 'list all qemu instances on the qemu boxes involved by this setup'
342 for (box,nodes) in self.gather_hostBoxes().iteritems():
343 # this is the brute force version, kill all qemus on that host box
344 TestBoxQemu(box,self.options.buildname).qemu_list_all()
347 # kill only the right qemus
348 def qemu_list_mine(self):
349 'list qemu instances for our nodes'
350 for (box,nodes) in self.gather_hostBoxes().iteritems():
351 # the fine-grain version
356 # kill only the right qemus
357 def qemu_kill_mine(self):
358 'kill the qemu instances for our nodes'
359 for (box,nodes) in self.gather_hostBoxes().iteritems():
360 # the fine-grain version
365 #################### display config
367 "show test configuration after localization"
373 "print cut'n paste-able stuff to export env variables to your shell"
374 # guess local domain from hostname
375 domain=socket.gethostname().split('.',1)[1]
376 fqdn="%s.%s"%(self.plc_spec['host_box'],domain)
377 print "export BUILD=%s"%self.options.buildname
378 if self.options.plcs_use_lxc:
379 print "export PLCHOSTLXC=%s"%fqdn
381 print "export PLCHOSTVS=%s"%fqdn
382 print "export GUESTNAME=%s"%self.plc_spec['vservername']
383 vplcname=self.plc_spec['vservername'].split('-')[-1]
384 print "export GUESTHOSTNAME=%s.%s"%(vplcname,domain)
385 # find hostname of first node
386 (hostname,qemubox) = self.all_node_infos()[0]
387 print "export KVMHOST=%s.%s"%(qemubox,domain)
388 print "export NODE=%s"%(hostname)
392 always_display_keys=['PLC_WWW_HOST','nodes','sites',]
393 def show_pass (self,passno):
394 for (key,val) in self.plc_spec.iteritems():
395 if not self.options.verbose and key not in TestPlc.always_display_keys: continue
399 self.display_site_spec(site)
400 for node in site['nodes']:
401 self.display_node_spec(node)
402 elif key=='initscripts':
403 for initscript in val:
404 self.display_initscript_spec (initscript)
407 self.display_slice_spec (slice)
410 self.display_key_spec (key)
412 if key not in ['sites','initscripts','slices','keys', 'sfa']:
413 print '+ ',key,':',val
415 def display_site_spec (self,site):
416 print '+ ======== site',site['site_fields']['name']
417 for (k,v) in site.iteritems():
418 if not self.options.verbose and k not in TestPlc.always_display_keys: continue
421 print '+ ','nodes : ',
423 print node['node_fields']['hostname'],'',
429 print user['name'],'',
431 elif k == 'site_fields':
432 print '+ login_base',':',v['login_base']
433 elif k == 'address_fields':
439 def display_initscript_spec (self,initscript):
440 print '+ ======== initscript',initscript['initscript_fields']['name']
442 def display_key_spec (self,key):
443 print '+ ======== key',key['name']
445 def display_slice_spec (self,slice):
446 print '+ ======== slice',slice['slice_fields']['name']
447 for (k,v) in slice.iteritems():
460 elif k=='slice_fields':
461 print '+ fields',':',
462 print 'max_nodes=',v['max_nodes'],
467 def display_node_spec (self,node):
468 print "+ node=%s host_box=%s"%(node['name'],node['host_box']),
469 print "hostname=",node['node_fields']['hostname'],
470 print "ip=",node['interface_fields']['ip']
471 if self.options.verbose:
472 utils.pprint("node details",node,depth=3)
474 # another entry point for just showing the boxes involved
475 def display_mapping (self):
476 TestPlc.display_mapping_plc(self.plc_spec)
480 def display_mapping_plc (plc_spec):
481 print '+ MyPLC',plc_spec['name']
482 # WARNING this would not be right for lxc-based PLC's - should be harmless though
483 print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['host_box'],plc_spec['vservername'])
484 print '+\tIP = %s/%s'%(plc_spec['PLC_API_HOST'],plc_spec['vserverip'])
485 for site_spec in plc_spec['sites']:
486 for node_spec in site_spec['nodes']:
487 TestPlc.display_mapping_node(node_spec)
490 def display_mapping_node (node_spec):
491 print '+ NODE %s'%(node_spec['name'])
492 print '+\tqemu box %s'%node_spec['host_box']
493 print '+\thostname=%s'%node_spec['node_fields']['hostname']
495 # write a timestamp in /vservers/<>.timestamp
496 # cannot be inside the vserver, that causes vserver .. build to cough
497 def timestamp_vs (self):
499 # TODO-lxc check this one
500 # a first approx. is to store the timestamp close to the VM root like vs does
501 stamp_path=self.vm_timestamp_path ()
502 stamp_dir = os.path.dirname (stamp_path)
503 utils.system(self.test_ssh.actual_command("mkdir -p %s"%stamp_dir))
504 return utils.system(self.test_ssh.actual_command("echo %d > %s"%(now,stamp_path)))==0
506 # this is called inconditionnally at the beginning of the test sequence
507 # just in case this is a rerun, so if the vm is not running it's fine
509 "vserver delete the test myplc"
510 stamp_path=self.vm_timestamp_path()
511 self.run_in_host("rm -f %s"%stamp_path)
512 if self.options.plcs_use_lxc:
513 self.run_in_host("lxc-stop --name %s"%self.vservername)
514 self.run_in_host("lxc-destroy --name %s"%self.vservername)
517 self.run_in_host("vserver --silent %s delete"%self.vservername)
521 # historically the build was being fetched by the tests
522 # now the build pushes itself as a subdir of the tests workdir
523 # so that the tests do not have to worry about extracting the build (svn, git, or whatever)
524 def vs_create (self):
525 "vserver creation (no install done)"
526 # push the local build/ dir to the testplc box
528 # a full path for the local calls
529 build_dir=os.path.dirname(sys.argv[0])
530 # sometimes this is empty - set to "." in such a case
531 if not build_dir: build_dir="."
532 build_dir += "/build"
534 # use a standard name - will be relative to remote buildname
536 # remove for safety; do *not* mkdir first, otherwise we end up with build/build/
537 self.test_ssh.rmdir(build_dir)
538 self.test_ssh.copy(build_dir,recursive=True)
539 # the repo url is taken from arch-rpms-url
540 # with the last step (i386) removed
541 repo_url = self.options.arch_rpms_url
542 for level in [ 'arch' ]:
543 repo_url = os.path.dirname(repo_url)
544 # pass the vbuild-nightly options to vtest-init-vserver
546 test_env_options += " -p %s"%self.options.personality
547 test_env_options += " -d %s"%self.options.pldistro
548 test_env_options += " -f %s"%self.options.fcdistro
549 if self.options.plcs_use_lxc:
550 script="vtest-init-lxc.sh"
552 script="vtest-init-vserver.sh"
553 vserver_name = self.vservername
554 vserver_options="--netdev eth0 --interface %s"%self.vserverip
556 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
557 vserver_options += " --hostname %s"%vserver_hostname
559 print "Cannot reverse lookup %s"%self.vserverip
560 print "This is considered fatal, as this might pollute the test results"
562 create_vserver="%(build_dir)s/%(script)s %(test_env_options)s %(vserver_name)s %(repo_url)s -- %(vserver_options)s"%locals()
563 return self.run_in_host(create_vserver) == 0
566 def plc_install(self):
567 "yum install myplc, noderepo, and the plain bootstrapfs"
569 # workaround for getting pgsql8.2 on centos5
570 if self.options.fcdistro == "centos5":
571 self.run_in_guest("rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-3.noarch.rpm")
574 if self.options.personality == "linux32":
576 elif self.options.personality == "linux64":
579 raise Exception, "Unsupported personality %r"%self.options.personality
580 nodefamily="%s-%s-%s"%(self.options.pldistro,self.options.fcdistro,arch)
583 pkgs_list.append ("slicerepo-%s"%nodefamily)
584 pkgs_list.append ("myplc")
585 pkgs_list.append ("noderepo-%s"%nodefamily)
586 pkgs_list.append ("nodeimage-%s-plain"%nodefamily)
587 pkgs_string=" ".join(pkgs_list)
588 return self.yum_install (pkgs_list)
591 def plc_configure(self):
593 tmpname='%s.plc-config-tty'%(self.name())
594 fileconf=open(tmpname,'w')
595 for var in [ 'PLC_NAME',
600 'PLC_MAIL_SUPPORT_ADDRESS',
603 # Above line was added for integrating SFA Testing
609 'PLC_RESERVATION_GRANULARITY',
611 'PLC_OMF_XMPP_SERVER',
613 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
614 fileconf.write('w\n')
615 fileconf.write('q\n')
617 utils.system('cat %s'%tmpname)
618 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
619 utils.system('rm %s'%tmpname)
624 self.run_in_guest('service plc start')
629 self.run_in_guest('service plc stop')
633 "start the PLC vserver"
638 "stop the PLC vserver"
642 # stores the keys from the config for further use
643 def keys_store(self):
644 "stores test users ssh keys in keys/"
645 for key_spec in self.plc_spec['keys']:
646 TestKey(self,key_spec).store_key()
649 def keys_clean(self):
650 "removes keys cached in keys/"
651 utils.system("rm -rf ./keys")
654 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
655 # for later direct access to the nodes
656 def keys_fetch(self):
657 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
659 if not os.path.isdir(dir):
661 vservername=self.vservername
662 vm_root=self.vm_root_in_host()
664 prefix = 'debug_ssh_key'
665 for ext in [ 'pub', 'rsa' ] :
666 src="%(vm_root)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
667 dst="keys/%(vservername)s-debug.%(ext)s"%locals()
668 if self.test_ssh.fetch(src,dst) != 0: overall=False
672 "create sites with PLCAPI"
673 return self.do_sites()
675 def delete_sites (self):
676 "delete sites with PLCAPI"
677 return self.do_sites(action="delete")
679 def do_sites (self,action="add"):
680 for site_spec in self.plc_spec['sites']:
681 test_site = TestSite (self,site_spec)
682 if (action != "add"):
683 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
684 test_site.delete_site()
685 # deleted with the site
686 #test_site.delete_users()
689 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
690 test_site.create_site()
691 test_site.create_users()
694 def delete_all_sites (self):
695 "Delete all sites in PLC, and related objects"
696 print 'auth_root',self.auth_root()
697 sites = self.apiserver.GetSites(self.auth_root(), {}, ['site_id'])
699 # keep automatic site - otherwise we shoot in our own foot, root_auth is not valid anymore
700 if site['login_base']==self.plc_spec['PLC_SLICE_PREFIX']: continue
701 site_id=site['site_id']
702 print 'Deleting site_id',site_id
703 self.apiserver.DeleteSite(self.auth_root(),site_id)
707 "create nodes with PLCAPI"
708 return self.do_nodes()
709 def delete_nodes (self):
710 "delete nodes with PLCAPI"
711 return self.do_nodes(action="delete")
713 def do_nodes (self,action="add"):
714 for site_spec in self.plc_spec['sites']:
715 test_site = TestSite (self,site_spec)
717 utils.header("Deleting nodes in site %s"%test_site.name())
718 for node_spec in site_spec['nodes']:
719 test_node=TestNode(self,test_site,node_spec)
720 utils.header("Deleting %s"%test_node.name())
721 test_node.delete_node()
723 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
724 for node_spec in site_spec['nodes']:
725 utils.pprint('Creating node %s'%node_spec,node_spec)
726 test_node = TestNode (self,test_site,node_spec)
727 test_node.create_node ()
730 def nodegroups (self):
731 "create nodegroups with PLCAPI"
732 return self.do_nodegroups("add")
733 def delete_nodegroups (self):
734 "delete nodegroups with PLCAPI"
735 return self.do_nodegroups("delete")
739 def translate_timestamp (start,grain,timestamp):
740 if timestamp < TestPlc.YEAR: return start+timestamp*grain
741 else: return timestamp
744 def timestamp_printable (timestamp):
745 return time.strftime('%m-%d %H:%M:%S UTC',time.gmtime(timestamp))
748 "create leases (on reservable nodes only, use e.g. run -c default -c resa)"
750 grain=self.apiserver.GetLeaseGranularity(self.auth_root())
751 print 'API answered grain=',grain
752 start=(now/grain)*grain
754 # find out all nodes that are reservable
755 nodes=self.all_reservable_nodenames()
757 utils.header ("No reservable node found - proceeding without leases")
760 # attach them to the leases as specified in plc_specs
761 # this is where the 'leases' field gets interpreted as relative of absolute
762 for lease_spec in self.plc_spec['leases']:
763 # skip the ones that come with a null slice id
764 if not lease_spec['slice']: continue
765 lease_spec['t_from']=TestPlc.translate_timestamp(start,grain,lease_spec['t_from'])
766 lease_spec['t_until']=TestPlc.translate_timestamp(start,grain,lease_spec['t_until'])
767 lease_addition=self.apiserver.AddLeases(self.auth_root(),nodes,
768 lease_spec['slice'],lease_spec['t_from'],lease_spec['t_until'])
769 if lease_addition['errors']:
770 utils.header("Cannot create leases, %s"%lease_addition['errors'])
773 utils.header('Leases on nodes %r for %s from %d (%s) until %d (%s)'%\
774 (nodes,lease_spec['slice'],
775 lease_spec['t_from'],TestPlc.timestamp_printable(lease_spec['t_from']),
776 lease_spec['t_until'],TestPlc.timestamp_printable(lease_spec['t_until'])))
780 def delete_leases (self):
781 "remove all leases in the myplc side"
782 lease_ids= [ l['lease_id'] for l in self.apiserver.GetLeases(self.auth_root())]
783 utils.header("Cleaning leases %r"%lease_ids)
784 self.apiserver.DeleteLeases(self.auth_root(),lease_ids)
787 def list_leases (self):
788 "list all leases known to the myplc"
789 leases = self.apiserver.GetLeases(self.auth_root())
792 current=l['t_until']>=now
793 if self.options.verbose or current:
794 utils.header("%s %s from %s until %s"%(l['hostname'],l['name'],
795 TestPlc.timestamp_printable(l['t_from']),
796 TestPlc.timestamp_printable(l['t_until'])))
799 # create nodegroups if needed, and populate
800 def do_nodegroups (self, action="add"):
801 # 1st pass to scan contents
803 for site_spec in self.plc_spec['sites']:
804 test_site = TestSite (self,site_spec)
805 for node_spec in site_spec['nodes']:
806 test_node=TestNode (self,test_site,node_spec)
807 if node_spec.has_key('nodegroups'):
808 nodegroupnames=node_spec['nodegroups']
809 if isinstance(nodegroupnames,StringTypes):
810 nodegroupnames = [ nodegroupnames ]
811 for nodegroupname in nodegroupnames:
812 if not groups_dict.has_key(nodegroupname):
813 groups_dict[nodegroupname]=[]
814 groups_dict[nodegroupname].append(test_node.name())
815 auth=self.auth_root()
817 for (nodegroupname,group_nodes) in groups_dict.iteritems():
819 print 'nodegroups:','dealing with nodegroup',nodegroupname,'on nodes',group_nodes
820 # first, check if the nodetagtype is here
821 tag_types = self.apiserver.GetTagTypes(auth,{'tagname':nodegroupname})
823 tag_type_id = tag_types[0]['tag_type_id']
825 tag_type_id = self.apiserver.AddTagType(auth,
826 {'tagname':nodegroupname,
827 'description': 'for nodegroup %s'%nodegroupname,
829 print 'located tag (type)',nodegroupname,'as',tag_type_id
831 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
833 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
834 print 'created nodegroup',nodegroupname,'from tagname',nodegroupname,'and value','yes'
835 # set node tag on all nodes, value='yes'
836 for nodename in group_nodes:
838 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
840 traceback.print_exc()
841 print 'node',nodename,'seems to already have tag',nodegroupname
844 expect_yes = self.apiserver.GetNodeTags(auth,
845 {'hostname':nodename,
846 'tagname':nodegroupname},
847 ['value'])[0]['value']
848 if expect_yes != "yes":
849 print 'Mismatch node tag on node',nodename,'got',expect_yes
852 if not self.options.dry_run:
853 print 'Cannot find tag',nodegroupname,'on node',nodename
857 print 'cleaning nodegroup',nodegroupname
858 self.apiserver.DeleteNodeGroup(auth,nodegroupname)
860 traceback.print_exc()
864 # a list of TestNode objs
865 def all_nodes (self):
867 for site_spec in self.plc_spec['sites']:
868 test_site = TestSite (self,site_spec)
869 for node_spec in site_spec['nodes']:
870 nodes.append(TestNode (self,test_site,node_spec))
873 # return a list of tuples (nodename,qemuname)
874 def all_node_infos (self) :
876 for site_spec in self.plc_spec['sites']:
877 node_infos += [ (node_spec['node_fields']['hostname'],node_spec['host_box']) \
878 for node_spec in site_spec['nodes'] ]
881 def all_nodenames (self): return [ x[0] for x in self.all_node_infos() ]
882 def all_reservable_nodenames (self):
884 for site_spec in self.plc_spec['sites']:
885 for node_spec in site_spec['nodes']:
886 node_fields=node_spec['node_fields']
887 if 'node_type' in node_fields and node_fields['node_type']=='reservable':
888 res.append(node_fields['hostname'])
891 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
892 def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period=15):
893 if self.options.dry_run:
897 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
898 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
899 # the nodes that haven't checked yet - start with a full list and shrink over time
900 tocheck = self.all_hostnames()
901 utils.header("checking nodes %r"%tocheck)
902 # create a dict hostname -> status
903 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
906 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
908 for array in tocheck_status:
909 hostname=array['hostname']
910 boot_state=array['boot_state']
911 if boot_state == target_boot_state:
912 utils.header ("%s has reached the %s state"%(hostname,target_boot_state))
914 # if it's a real node, never mind
915 (site_spec,node_spec)=self.locate_hostname(hostname)
916 if TestNode.is_real_model(node_spec['node_fields']['model']):
917 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
919 boot_state = target_boot_state
920 elif datetime.datetime.now() > graceout:
921 utils.header ("%s still in '%s' state"%(hostname,boot_state))
922 graceout=datetime.datetime.now()+datetime.timedelta(1)
923 status[hostname] = boot_state
925 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != target_boot_state ]
928 if datetime.datetime.now() > timeout:
929 for hostname in tocheck:
930 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
932 # otherwise, sleep for a while
934 # only useful in empty plcs
937 def nodes_booted(self):
938 return self.nodes_check_boot_state('boot',timeout_minutes=30,silent_minutes=28)
940 def check_nodes_ssh(self,debug,timeout_minutes,silent_minutes,period=15):
942 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
943 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
944 vservername=self.vservername
947 local_key = "keys/%(vservername)s-debug.rsa"%locals()
950 local_key = "keys/key1.rsa"
951 node_infos = self.all_node_infos()
952 utils.header("checking ssh access (expected in %s mode) to nodes:"%message)
953 for (nodename,qemuname) in node_infos:
954 utils.header("hostname=%s -- qemubox=%s"%(nodename,qemuname))
955 utils.header("max timeout is %d minutes, silent for %d minutes (period is %s)"%\
956 (timeout_minutes,silent_minutes,period))
958 for node_info in node_infos:
959 (hostname,qemuname) = node_info
960 # try to run 'hostname' in the node
961 command = TestSsh (hostname,key=local_key).actual_command("hostname;uname -a")
962 # don't spam logs - show the command only after the grace period
963 success = utils.system ( command, silent=datetime.datetime.now() < graceout)
965 utils.header('Successfully entered root@%s (%s)'%(hostname,message))
967 node_infos.remove(node_info)
969 # we will have tried real nodes once, in case they're up - but if not, just skip
970 (site_spec,node_spec)=self.locate_hostname(hostname)
971 if TestNode.is_real_model(node_spec['node_fields']['model']):
972 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
973 node_infos.remove(node_info)
976 if datetime.datetime.now() > timeout:
977 for (hostname,qemuname) in node_infos:
978 utils.header("FAILURE to ssh into %s (on %s)"%(hostname,qemuname))
980 # otherwise, sleep for a while
982 # only useful in empty plcs
985 def ssh_node_debug(self):
986 "Tries to ssh into nodes in debug mode with the debug ssh key"
987 return self.check_nodes_ssh(debug=True,timeout_minutes=10,silent_minutes=8)
989 def ssh_node_boot(self):
990 "Tries to ssh into nodes in production mode with the root ssh key"
991 return self.check_nodes_ssh(debug=False,timeout_minutes=40,silent_minutes=38)
994 def qemu_local_init (self):
995 "all nodes : init a clean local directory for holding node-dep stuff like iso image..."
999 "all nodes: invoke GetBootMedium and store result locally"
1002 def qemu_local_config (self):
1003 "all nodes: compute qemu config qemu.conf and store it locally"
1006 def nodestate_reinstall (self):
1007 "all nodes: mark PLCAPI boot_state as reinstall"
1010 def nodestate_safeboot (self):
1011 "all nodes: mark PLCAPI boot_state as safeboot"
1014 def nodestate_boot (self):
1015 "all nodes: mark PLCAPI boot_state as boot"
1018 def nodestate_show (self):
1019 "all nodes: show PLCAPI boot_state"
1022 def qemu_export (self):
1023 "all nodes: push local node-dep directory on the qemu box"
1026 ### check hooks : invoke scripts from hooks/{node,slice}
1027 def check_hooks_node (self):
1028 return self.locate_first_node().check_hooks()
1029 def check_hooks_sliver (self) :
1030 return self.locate_first_sliver().check_hooks()
1032 def check_hooks (self):
1033 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
1034 return self.check_hooks_node() and self.check_hooks_sliver()
1037 def do_check_initscripts(self):
1039 for slice_spec in self.plc_spec['slices']:
1040 if not slice_spec.has_key('initscriptstamp'):
1042 stamp=slice_spec['initscriptstamp']
1043 for nodename in slice_spec['nodenames']:
1044 (site,node) = self.locate_node (nodename)
1045 # xxx - passing the wrong site - probably harmless
1046 test_site = TestSite (self,site)
1047 test_slice = TestSlice (self,test_site,slice_spec)
1048 test_node = TestNode (self,test_site,node)
1049 test_sliver = TestSliver (self, test_node, test_slice)
1050 if not test_sliver.check_initscript_stamp(stamp):
1054 def check_initscripts(self):
1055 "check that the initscripts have triggered"
1056 return self.do_check_initscripts()
1058 def initscripts (self):
1059 "create initscripts with PLCAPI"
1060 for initscript in self.plc_spec['initscripts']:
1061 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
1062 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
1065 def delete_initscripts (self):
1066 "delete initscripts with PLCAPI"
1067 for initscript in self.plc_spec['initscripts']:
1068 initscript_name = initscript['initscript_fields']['name']
1069 print('Attempting to delete %s in plc %s'%(initscript_name,self.plc_spec['name']))
1071 self.apiserver.DeleteInitScript(self.auth_root(),initscript_name)
1072 print initscript_name,'deleted'
1074 print 'deletion went wrong - probably did not exist'
1079 "create slices with PLCAPI"
1080 return self.do_slices(action="add")
1082 def delete_slices (self):
1083 "delete slices with PLCAPI"
1084 return self.do_slices(action="delete")
1086 def fill_slices (self):
1087 "add nodes in slices with PLCAPI"
1088 return self.do_slices(action="fill")
1090 def empty_slices (self):
1091 "remove nodes from slices with PLCAPI"
1092 return self.do_slices(action="empty")
1094 def do_slices (self, action="add"):
1095 for slice in self.plc_spec['slices']:
1096 site_spec = self.locate_site (slice['sitename'])
1097 test_site = TestSite(self,site_spec)
1098 test_slice=TestSlice(self,test_site,slice)
1099 if action == "delete":
1100 test_slice.delete_slice()
1101 elif action=="fill":
1102 test_slice.add_nodes()
1103 elif action=="empty":
1104 test_slice.delete_nodes()
1106 test_slice.create_slice()
1110 def ssh_slice(self):
1111 "tries to ssh-enter the slice with the user key, to ensure slice creation"
1115 def ssh_slice_off (self):
1116 "tries to ssh-enter the slice with the user key, expecting it to be unreachable"
1120 def keys_clear_known_hosts (self):
1121 "remove test nodes entries from the local known_hosts file"
1124 def speed_up_slices (self):
1125 "tweak nodemanager settings on all nodes using a conf file"
1126 # create the template on the server-side
1127 template="%s.nodemanager"%self.name()
1128 template_file = open (template,"w")
1129 template_file.write('OPTIONS="-p 30 -r 11 -d"\n')
1130 template_file.close()
1131 in_vm="/var/www/html/PlanetLabConf/nodemanager"
1132 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1133 self.test_ssh.copy_abs(template,remote)
1135 self.apiserver.AddConfFile (self.auth_root(),
1136 {'dest':'/etc/sysconfig/nodemanager',
1137 'source':'PlanetLabConf/nodemanager',
1138 'postinstall_cmd':'service nm restart',})
1142 def qemu_start (self) :
1143 "all nodes: start the qemu instance (also runs qemu-bridge-init start)"
1147 def timestamp_qemu (self) :
1148 "all nodes: start the qemu instance (also runs qemu-bridge-init start)"
1151 def check_tcp (self):
1152 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
1153 specs = self.plc_spec['tcp_test']
1158 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
1159 if not s_test_sliver.run_tcp_server(port,timeout=10):
1163 # idem for the client side
1164 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
1165 if not c_test_sliver.run_tcp_client(s_test_sliver.test_node.name(),port):
1169 # painfully enough, we need to allow for some time as netflow might show up last
1170 def check_sys_slice (self):
1171 "all nodes: check that a system slice is alive"
1172 # would probably make more sense to check for netflow,
1173 # but that one is currently not working in the lxc distro
1174 # return self.check_systemslice ('netflow')
1175 return self.check_systemslice ('drl')
1177 # we have the slices up already here, so it should not take too long
1178 def check_systemslice (self, slicename, timeout_minutes=5, period=15):
1179 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
1180 test_nodes=self.all_nodes()
1182 for test_node in test_nodes:
1183 if test_node.check_systemslice (slicename,dry_run=self.options.dry_run):
1185 test_nodes.remove(test_node)
1190 if datetime.datetime.now () > timeout:
1191 for test_node in test_nodes:
1192 utils.header ("can't find system slice %s in %s"%(slicename,test_node.name()))
1197 def plcsh_stress_test (self):
1198 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
1199 # install the stress-test in the plc image
1200 location = "/usr/share/plc_api/plcsh_stress_test.py"
1201 remote="%s/%s"%(self.vm_root_in_host(),location)
1202 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1204 command += " -- --check"
1205 if self.options.size == 1:
1206 command += " --tiny"
1207 return ( self.run_in_guest(command) == 0)
1209 # populate runs the same utility without slightly different options
1210 # in particular runs with --preserve (dont cleanup) and without --check
1211 # also it gets run twice, once with the --foreign option for creating fake foreign entries
1213 def sfa_install_all (self):
1214 "yum install sfa sfa-plc sfa-sfatables sfa-client"
1215 return self.yum_install ("sfa sfa-plc sfa-sfatables sfa-client")
1217 def sfa_install_core(self):
1219 return self.yum_install ("sfa")
1221 def sfa_install_plc(self):
1222 "yum install sfa-plc"
1223 return self.yum_install("sfa-plc")
1225 def sfa_install_sfatables(self):
1226 "yum install sfa-sfatables"
1227 return self.yum_install ("sfa-sfatables")
1229 # for some very odd reason, this sometimes fails with the following symptom
1230 # # yum install sfa-client
1231 # Setting up Install Process
1233 # Downloading Packages:
1234 # Running rpm_check_debug
1235 # Running Transaction Test
1236 # Transaction Test Succeeded
1237 # Running Transaction
1238 # Transaction couldn't start:
1239 # installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem
1240 # [('installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem', (9, '/', 69632L))]
1241 # even though in the same context I have
1242 # [2012.05.23--f14-32-sfastd1-1-vplc07] / # df -h
1243 # Filesystem Size Used Avail Use% Mounted on
1244 # /dev/hdv1 806G 264G 501G 35% /
1245 # none 16M 36K 16M 1% /tmp
1247 # so as a workaround, we first try yum install, and then invoke rpm on the cached rpm...
1248 def sfa_install_client(self):
1249 "yum install sfa-client"
1250 first_try=self.yum_install("sfa-client")
1251 if first_try: return True
1252 utils.header ("********** Regular yum failed - special workaround in place, 2nd chance")
1253 (code,cached_rpm_path)=utils.output_of(self.actual_command_in_guest('find /var/cache/yum -name sfa-client\*.rpm'))
1255 self.run_in_guest("rpm -i %s"%cached_rpm_path)
1256 return self.yum_check_installed ("sfa-client")
1258 def sfa_dbclean(self):
1259 "thoroughly wipes off the SFA database"
1260 return self.run_in_guest("sfaadmin.py registry nuke")==0 or \
1261 self.run_in_guest("sfa-nuke.py")==0 or \
1262 self.run_in_guest("sfa-nuke-plc.py")==0
1264 def sfa_plcclean(self):
1265 "cleans the PLC entries that were created as a side effect of running the script"
1267 sfa_spec=self.plc_spec['sfa']
1269 for sfa_slice_spec in sfa_spec['sfa_slice_specs']:
1270 login_base=sfa_slice_spec['login_base']
1271 try: self.apiserver.DeleteSite (self.auth_root(),login_base)
1272 except: print "Site %s already absent from PLC db"%login_base
1274 for key in ['piuser','regularuser']:
1275 username="%s@%s"%(sfa_slice_spec[key],sfa_slice_spec['domain'])
1276 try: self.apiserver.DeletePerson(self.auth_root(),username)
1277 except: print "User %s already absent from PLC db"%username
1279 print "REMEMBER TO RUN sfa_import AGAIN"
1282 def sfa_uninstall(self):
1283 "uses rpm to uninstall sfa - ignore result"
1284 self.run_in_guest("rpm -e sfa sfa-sfatables sfa-client sfa-plc")
1285 self.run_in_guest("rm -rf /var/lib/sfa")
1286 self.run_in_guest("rm -rf /etc/sfa")
1287 self.run_in_guest("rm -rf /var/log/sfa_access.log /var/log/sfa_import_plc.log /var/log/sfa.daemon")
1289 self.run_in_guest("rpm -e --noscripts sfa-plc")
1292 ### run unit tests for SFA
1293 # NOTE: for some reason on f14/i386, yum install sfa-tests fails for no reason
1294 # Running Transaction
1295 # Transaction couldn't start:
1296 # installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem
1297 # [('installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem', (9, '/', 208896L))]
1298 # no matter how many Gbs are available on the testplc
1299 # could not figure out what's wrong, so...
1300 # if the yum install phase fails, consider the test is successful
1301 # other combinations will eventually run it hopefully
1302 def sfa_utest(self):
1303 "yum install sfa-tests and run SFA unittests"
1304 self.run_in_guest("yum -y install sfa-tests")
1305 # failed to install - forget it
1306 if self.run_in_guest("rpm -q sfa-tests")!=0:
1307 utils.header("WARNING: SFA unit tests failed to install, ignoring")
1309 return self.run_in_guest("/usr/share/sfa/tests/testAll.py")==0
1313 dirname="conf.%s"%self.plc_spec['name']
1314 if not os.path.isdir(dirname):
1315 utils.system("mkdir -p %s"%dirname)
1316 if not os.path.isdir(dirname):
1317 raise "Cannot create config dir for plc %s"%self.name()
1320 def conffile(self,filename):
1321 return "%s/%s"%(self.confdir(),filename)
1322 def confsubdir(self,dirname,clean,dry_run=False):
1323 subdirname="%s/%s"%(self.confdir(),dirname)
1325 utils.system("rm -rf %s"%subdirname)
1326 if not os.path.isdir(subdirname):
1327 utils.system("mkdir -p %s"%subdirname)
1328 if not dry_run and not os.path.isdir(subdirname):
1329 raise "Cannot create config subdir %s for plc %s"%(dirname,self.name())
1332 def conffile_clean (self,filename):
1333 filename=self.conffile(filename)
1334 return utils.system("rm -rf %s"%filename)==0
1337 def sfa_configure(self):
1338 "run sfa-config-tty"
1339 tmpname=self.conffile("sfa-config-tty")
1340 fileconf=open(tmpname,'w')
1341 for var in [ 'SFA_REGISTRY_ROOT_AUTH',
1342 'SFA_INTERFACE_HRN',
1343 'SFA_REGISTRY_LEVEL1_AUTH',
1344 'SFA_REGISTRY_HOST',
1345 'SFA_AGGREGATE_HOST',
1355 'SFA_GENERIC_FLAVOUR',
1356 'SFA_AGGREGATE_ENABLED',
1358 if self.plc_spec['sfa'].has_key(var):
1359 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec['sfa'][var]))
1360 # the way plc_config handles booleans just sucks..
1363 if self.plc_spec['sfa'][var]: val='true'
1364 fileconf.write ('e %s\n%s\n'%(var,val))
1365 fileconf.write('w\n')
1366 fileconf.write('R\n')
1367 fileconf.write('q\n')
1369 utils.system('cat %s'%tmpname)
1370 self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
1373 def aggregate_xml_line(self):
1374 port=self.plc_spec['sfa']['neighbours-port']
1375 return '<aggregate addr="%s" hrn="%s" port="%r"/>' % \
1376 (self.vserverip,self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH'],port)
1378 def registry_xml_line(self):
1379 return '<registry addr="%s" hrn="%s" port="12345"/>' % \
1380 (self.vserverip,self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH'])
1383 # a cross step that takes all other plcs in argument
1384 def cross_sfa_configure(self, other_plcs):
1385 "writes aggregates.xml and registries.xml that point to all other PLCs in the test"
1386 # of course with a single plc, other_plcs is an empty list
1389 agg_fname=self.conffile("agg.xml")
1390 file(agg_fname,"w").write("<aggregates>%s</aggregates>\n" % \
1391 " ".join([ plc.aggregate_xml_line() for plc in other_plcs ]))
1392 utils.header ("(Over)wrote %s"%agg_fname)
1393 reg_fname=self.conffile("reg.xml")
1394 file(reg_fname,"w").write("<registries>%s</registries>\n" % \
1395 " ".join([ plc.registry_xml_line() for plc in other_plcs ]))
1396 utils.header ("(Over)wrote %s"%reg_fname)
1397 return self.test_ssh.copy_abs(agg_fname,'/%s/etc/sfa/aggregates.xml'%self.vm_root_in_host())==0 \
1398 and self.test_ssh.copy_abs(reg_fname,'/%s/etc/sfa/registries.xml'%self.vm_root_in_host())==0
1400 def sfa_import(self):
1402 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1403 return self.run_in_guest('sfa-import.py')==0 or \
1404 self.run_in_guest('sfa-import-plc.py')==0 or \
1405 self.run_in_guest('sfaadmin.py registry import_registry')==0
1406 # not needed anymore
1407 # self.run_in_guest('cp /etc/sfa/authorities/%s/%s.pkey /etc/sfa/authorities/server.key'%(auth,auth))
1409 def sfa_start(self):
1411 return self.run_in_guest('service sfa start')==0
1413 def sfi_configure(self):
1414 "Create /root/sfi on the plc side for sfi client configuration"
1415 if self.options.dry_run:
1416 utils.header("DRY RUN - skipping step")
1418 sfa_spec=self.plc_spec['sfa']
1419 # cannot use sfa_slice_mapper to pass dir_name
1420 for slice_spec in self.plc_spec['sfa']['sfa_slice_specs']:
1421 test_slice=TestSliceSfa(self,slice_spec)
1422 dir_basename=os.path.basename(test_slice.sfi_path())
1423 dir_name=self.confsubdir("dot-sfi/%s"%dir_basename,clean=True,dry_run=self.options.dry_run)
1424 test_slice.sfi_config(dir_name)
1425 # push into the remote /root/sfi area
1426 location = test_slice.sfi_path()
1427 remote="%s/%s"%(self.vm_root_in_host(),location)
1428 self.test_ssh.mkdir(remote,abs=True)
1429 # need to strip last level or remote otherwise we get an extra dir level
1430 self.test_ssh.copy_abs(dir_name, os.path.dirname(remote), recursive=True)
1434 def sfi_clean (self):
1435 "clean up /root/sfi on the plc side"
1436 self.run_in_guest("rm -rf /root/sfi")
1440 def sfa_add_site (self):
1441 "bootstrap a site using sfaadmin"
1445 def sfa_add_pi (self):
1446 "bootstrap a PI user for that site"
1450 def sfa_add_user(self):
1455 def sfa_update_user(self):
1459 def sfa_add_slice(self):
1460 "run sfi.py add (on Registry) from slice.xml"
1464 def sfa_discover(self):
1465 "discover resources into resouces_in.rspec"
1469 def sfa_create_slice(self):
1470 "run sfi.py create (on SM) - 1st time"
1474 def sfa_check_slice_plc(self):
1475 "check sfa_create_slice at the plcs - all local nodes should be in slice"
1479 def sfa_update_slice(self):
1480 "run sfi.py create (on SM) on existing object"
1485 "various registry-related calls"
1489 def ssh_slice_sfa(self):
1490 "tries to ssh-enter the SFA slice"
1494 def sfa_delete_user(self):
1499 def sfa_delete_slice(self):
1500 "run sfi.py delete (on SM), sfi.py remove (on Registry) to clean slices"
1505 self.run_in_guest('service sfa stop')==0
1508 def populate (self):
1509 "creates random entries in the PLCAPI"
1510 # install the stress-test in the plc image
1511 location = "/usr/share/plc_api/plcsh_stress_test.py"
1512 remote="%s/%s"%(self.vm_root_in_host(),location)
1513 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1515 command += " -- --preserve --short-names"
1516 local = (self.run_in_guest(command) == 0);
1517 # second run with --foreign
1518 command += ' --foreign'
1519 remote = (self.run_in_guest(command) == 0);
1520 return ( local and remote)
1522 def gather_logs (self):
1523 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1524 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1525 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1526 # (1.c) get the plc's /root/sfi -> logs/sfi.<plcname>/
1527 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1528 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1529 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1531 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1532 self.gather_var_logs ()
1534 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1535 self.gather_pgsql_logs ()
1537 print "-------------------- TestPlc.gather_logs : PLC's /root/sfi/"
1538 self.gather_root_sfi ()
1540 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1541 for site_spec in self.plc_spec['sites']:
1542 test_site = TestSite (self,site_spec)
1543 for node_spec in site_spec['nodes']:
1544 test_node=TestNode(self,test_site,node_spec)
1545 test_node.gather_qemu_logs()
1547 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1548 self.gather_nodes_var_logs()
1550 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1551 self.gather_slivers_var_logs()
1554 def gather_slivers_var_logs(self):
1555 for test_sliver in self.all_sliver_objs():
1556 remote = test_sliver.tar_var_logs()
1557 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
1558 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
1559 utils.system(command)
1562 def gather_var_logs (self):
1563 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
1564 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1565 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
1566 utils.system(command)
1567 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd"%self.name()
1568 utils.system(command)
1570 def gather_pgsql_logs (self):
1571 utils.system("mkdir -p logs/myplc.pgsql-log.%s"%self.name())
1572 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1573 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -"%self.name()
1574 utils.system(command)
1576 def gather_root_sfi (self):
1577 utils.system("mkdir -p logs/sfi.%s"%self.name())
1578 to_plc = self.actual_command_in_guest("tar -C /root/sfi/ -cf - .")
1579 command = to_plc + "| tar -C logs/sfi.%s -xf -"%self.name()
1580 utils.system(command)
1582 def gather_nodes_var_logs (self):
1583 for site_spec in self.plc_spec['sites']:
1584 test_site = TestSite (self,site_spec)
1585 for node_spec in site_spec['nodes']:
1586 test_node=TestNode(self,test_site,node_spec)
1587 test_ssh = TestSsh (test_node.name(),key="keys/key1.rsa")
1588 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1589 command = command + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
1590 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
1591 utils.system(command)
1594 # returns the filename to use for sql dump/restore, using options.dbname if set
1595 def dbfile (self, database):
1596 # uses options.dbname if it is found
1598 name=self.options.dbname
1599 if not isinstance(name,StringTypes):
1602 t=datetime.datetime.now()
1605 return "/root/%s-%s.sql"%(database,name)
1607 def plc_db_dump(self):
1608 'dump the planetlab5 DB in /root in the PLC - filename has time'
1609 dump=self.dbfile("planetab5")
1610 self.run_in_guest('pg_dump -U pgsqluser planetlab5 -f '+ dump)
1611 utils.header('Dumped planetlab5 database in %s'%dump)
1614 def plc_db_restore(self):
1615 'restore the planetlab5 DB - looks broken, but run -n might help'
1616 dump=self.dbfile("planetab5")
1617 ##stop httpd service
1618 self.run_in_guest('service httpd stop')
1619 # xxx - need another wrapper
1620 self.run_in_guest_piped('echo drop database planetlab5','psql --user=pgsqluser template1')
1621 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab5')
1622 self.run_in_guest('psql -U pgsqluser planetlab5 -f '+dump)
1623 ##starting httpd service
1624 self.run_in_guest('service httpd start')
1626 utils.header('Database restored from ' + dump)
1628 def standby_1_through_20(self):
1629 """convenience function to wait for a specified number of minutes"""
1632 def standby_1(): pass
1634 def standby_2(): pass
1636 def standby_3(): pass
1638 def standby_4(): pass
1640 def standby_5(): pass
1642 def standby_6(): pass
1644 def standby_7(): pass
1646 def standby_8(): pass
1648 def standby_9(): pass
1650 def standby_10(): pass
1652 def standby_11(): pass
1654 def standby_12(): pass
1656 def standby_13(): pass
1658 def standby_14(): pass
1660 def standby_15(): pass
1662 def standby_16(): pass
1664 def standby_17(): pass
1666 def standby_18(): pass
1668 def standby_19(): pass
1670 def standby_20(): pass