1 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
2 # Copyright (C) 2010 INRIA
9 from datetime import datetime, timedelta
10 from types import StringTypes
13 from TestSite import TestSite
14 from TestNode import TestNode
15 from TestUser import TestUser
16 from TestKey import TestKey
17 from TestSlice import TestSlice
18 from TestSliver import TestSliver
19 from TestBoxQemu import TestBoxQemu
20 from TestSsh import TestSsh
21 from TestApiserver import TestApiserver
22 from TestAuthSfa import TestAuthSfa
23 from PlcapiUrlScanner import PlcapiUrlScanner
24 from Completer import Completer, CompleterTask
26 # step methods must take (self) and return a boolean (options is a member of the class)
28 def standby(minutes,dry_run):
29 utils.header('Entering StandBy for %d mn'%minutes)
33 time.sleep(60*minutes)
36 def standby_generic (func):
38 minutes=int(func.__name__.split("_")[1])
39 return standby(minutes,self.options.dry_run)
42 def node_mapper (method):
43 def actual(self,*args, **kwds):
45 node_method = TestNode.__dict__[method.__name__]
46 for test_node in self.all_nodes():
47 if not node_method(test_node, *args, **kwds): overall=False
49 # restore the doc text
50 actual.__doc__=TestNode.__dict__[method.__name__].__doc__
53 def slice_mapper (method):
56 slice_method = TestSlice.__dict__[method.__name__]
57 for slice_spec in self.plc_spec['slices']:
58 site_spec = self.locate_site (slice_spec['sitename'])
59 test_site = TestSite(self,site_spec)
60 test_slice=TestSlice(self,test_site,slice_spec)
61 if not slice_method(test_slice,self.options): overall=False
63 # restore the doc text
64 actual.__doc__=TestSlice.__dict__[method.__name__].__doc__
67 # a variant that expects the TestSlice method to return a list of CompleterTasks that
68 # are then merged into a single Completer run to avoid wating for all the slices
69 # esp. useful when a test fails of course
70 # because we need to pass arguments we use a class instead..
71 class slice_mapper__tasks (object):
72 # could not get this to work with named arguments
73 def __init__ (self,timeout_minutes,silent_minutes,period_seconds):
75 print "timeout_minutes,silent_minutes,period_seconds",timeout_minutes,silent_minutes,period_seconds
76 self.timeout=timedelta(minutes=timeout_minutes)
77 self.silent=timedelta(minutes=silent_minutes)
78 self.period=timedelta(seconds=period_seconds)
79 def __call__ (self, method):
81 # compute augmented method name
82 method_name = method.__name__ + "__tasks"
84 slice_method = TestSlice.__dict__[ method_name ]
87 for slice_spec in self.plc_spec['slices']:
88 site_spec = self.locate_site (slice_spec['sitename'])
89 test_site = TestSite(self,site_spec)
90 test_slice=TestSlice(self,test_site,slice_spec)
91 tasks += slice_method (test_slice, self.options)
92 return Completer (tasks).run (decorator_self.timeout, decorator_self.silent, decorator_self.period)
93 # restore the doc text from the TestSlice method even if a bit odd
94 wrappee.__doc__ = slice_method.__doc__
97 def auth_sfa_mapper (method):
100 auth_method = TestAuthSfa.__dict__[method.__name__]
101 for auth_spec in self.plc_spec['sfa']['auth_sfa_specs']:
102 test_auth=TestAuthSfa(self,auth_spec)
103 if not auth_method(test_auth,self.options): overall=False
105 # restore the doc text
106 actual.__doc__=TestAuthSfa.__dict__[method.__name__].__doc__
116 'vs_delete','timestamp_vs','vs_create', SEP,
117 # 'plc_install', 'mod_python', 'plc_configure', 'plc_start', SEP,
118 'plc_install', 'plc_configure', 'plc_start', SEP,
119 'keys_fetch', 'keys_store', 'keys_clear_known_hosts', SEP,
120 'plcapi_urls','speed_up_slices', SEP,
121 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', 'leases', SEP,
122 # slices created under plcsh interactively seem to be fine but these ones don't have the tags
123 # keep this our of the way for now
124 # 'check_vsys_defaults', SEP,
125 'nodestate_reinstall', 'qemu_local_init','bootcd', 'qemu_local_config', SEP,
126 'qemu_kill_mine','qemu_clean_mine', 'qemu_export', 'qemu_start', 'timestamp_qemu', SEP,
127 'sfa_install_all', 'sfa_configure', 'cross_sfa_configure', 'sfa_start', 'sfa_import', SEPSFA,
128 'sfi_configure@1', 'sfa_add_site@1','sfa_add_pi@1', SEPSFA,
129 'sfa_add_user@1', 'sfa_update_user@1', 'sfa_add_slice@1', 'sfa_renew_slice@1', SEPSFA,
130 'sfa_discover@1', 'sfa_create_slice@1', 'sfa_check_slice_plc@1', 'sfa_update_slice@1', SEPSFA,
131 'sfi_list@1', 'sfi_show@1', 'sfa_utest@1', SEPSFA,
132 # we used to run plcsh_stress_test, and then ssh_node_debug and ssh_node_boot
133 # but as the stress test might take a while, we sometimes missed the debug mode..
134 'ssh_node_debug@1', 'plcsh_stress_test@1', SEP,
135 'ssh_node_boot@1', 'node_bmlogs@1', 'ssh_slice', 'ssh_slice_basics', 'check_initscripts', SEP,
136 'ssh_slice_sfa@1', 'sfa_delete_slice@1', 'sfa_delete_user@1', SEPSFA,
137 'cross_check_tcp@1', 'check_system_slice', SEP,
138 'empty_slices', 'ssh_slice_off', 'fill_slices', SEP,
139 'force_gather_logs', SEP,
142 'export', 'show_boxes', SEP,
143 'check_hooks', 'plc_stop', 'vs_start', 'vs_stop', SEP,
144 'delete_initscripts', 'delete_nodegroups','delete_all_sites', SEP,
145 'delete_sites', 'delete_nodes', 'delete_slices', 'keys_clean', SEP,
146 'delete_leases', 'list_leases', SEP,
148 'nodestate_show','nodestate_safeboot','nodestate_boot', SEP,
149 'qemu_list_all', 'qemu_list_mine', 'qemu_kill_all', SEP,
150 'sfa_install_core', 'sfa_install_sfatables', 'sfa_install_plc', 'sfa_install_client', SEPSFA,
151 'sfa_plcclean', 'sfa_dbclean', 'sfa_stop','sfa_uninstall', 'sfi_clean', SEPSFA,
152 'plc_db_dump' , 'plc_db_restore', SEP,
153 'check_netflow','check_drl', SEP,
154 'debug_nodemanager', SEP,
155 'standby_1_through_20',SEP,
159 def printable_steps (list):
160 single_line=" ".join(list)+" "
161 return single_line.replace(" "+SEP+" "," \\\n").replace(" "+SEPSFA+" "," \\\n")
163 def valid_step (step):
164 return step != SEP and step != SEPSFA
166 # turn off the sfa-related steps when build has skipped SFA
167 # this was originally for centos5 but is still valid
168 # for up to f12 as recent SFAs with sqlalchemy won't build before f14
170 def check_whether_build_has_sfa (rpms_url):
171 utils.header ("Checking if build provides SFA package...")
172 # warning, we're now building 'sface' so let's be a bit more picky
173 retcod=os.system ("curl --silent %s/ | grep -q sfa-"%rpms_url)
174 # full builds are expected to return with 0 here
176 utils.header("build does provide SFA")
178 # move all steps containing 'sfa' from default_steps to other_steps
179 utils.header("SFA package not found - removing steps with sfa or sfi")
180 sfa_steps= [ step for step in TestPlc.default_steps if step.find('sfa')>=0 or step.find("sfi")>=0 ]
181 TestPlc.other_steps += sfa_steps
182 for step in sfa_steps: TestPlc.default_steps.remove(step)
184 def __init__ (self,plc_spec,options):
185 self.plc_spec=plc_spec
187 self.test_ssh=TestSsh(self.plc_spec['host_box'],self.options.buildname)
188 self.vserverip=plc_spec['vserverip']
189 self.vservername=plc_spec['vservername']
190 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
191 self.apiserver=TestApiserver(self.url,options.dry_run)
192 (self.ssh_node_boot_timeout,self.ssh_node_boot_silent)=plc_spec['ssh_node_boot_timers']
193 (self.ssh_node_debug_timeout,self.ssh_node_debug_silent)=plc_spec['ssh_node_debug_timers']
195 def has_addresses_api (self):
196 return self.apiserver.has_method('AddIpAddress')
199 name=self.plc_spec['name']
200 return "%s.%s"%(name,self.vservername)
203 return self.plc_spec['host_box']
206 return self.test_ssh.is_local()
208 # define the API methods on this object through xmlrpc
209 # would help, but not strictly necessary
213 def actual_command_in_guest (self,command):
214 return self.test_ssh.actual_command(self.host_to_guest(command),dry_run=self.options.dry_run)
216 def start_guest (self):
217 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host(),dry_run=self.options.dry_run))
219 def stop_guest (self):
220 return utils.system(self.test_ssh.actual_command(self.stop_guest_in_host(),dry_run=self.options.dry_run))
222 def run_in_guest (self,command):
223 return utils.system(self.actual_command_in_guest(command))
225 def run_in_host (self,command):
226 return self.test_ssh.run_in_buildname(command, dry_run=self.options.dry_run)
228 #command gets run in the plc's vm
229 def host_to_guest(self,command):
230 if self.options.plcs_use_lxc:
231 return "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s %s"%(self.vserverip,command)
233 return "vserver %s exec %s"%(self.vservername,command)
235 def vm_root_in_host(self):
236 if self.options.plcs_use_lxc:
237 return "/vservers/%s/rootfs/"%(self.vservername)
239 return "/vservers/%s"%(self.vservername)
241 def vm_timestamp_path (self):
242 if self.options.plcs_use_lxc:
243 return "/vservers/%s/%s.timestamp"%(self.vservername,self.vservername)
245 return "/vservers/%s.timestamp"%(self.vservername)
247 #start/stop the vserver
248 def start_guest_in_host(self):
249 if self.options.plcs_use_lxc:
250 return "virsh -c lxc:// start %s"%(self.vservername)
252 return "vserver %s start"%(self.vservername)
254 def stop_guest_in_host(self):
255 if self.options.plcs_use_lxc:
256 return "virsh -c lxc:// destroy %s"%(self.vservername)
258 return "vserver %s stop"%(self.vservername)
261 def run_in_guest_piped (self,local,remote):
262 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
264 def yum_check_installed (self, rpms):
265 if isinstance (rpms, list):
267 return self.run_in_guest("rpm -q %s"%rpms)==0
269 # does a yum install in the vs, ignore yum retcod, check with rpm
270 def yum_install (self, rpms):
271 if isinstance (rpms, list):
273 self.run_in_guest("yum -y install %s"%rpms)
274 # yum-complete-transaction comes with yum-utils, that is in vtest.pkgs
275 self.run_in_guest("yum-complete-transaction -y")
276 return self.yum_check_installed (rpms)
278 def auth_root (self):
279 return {'Username':self.plc_spec['PLC_ROOT_USER'],
280 'AuthMethod':'password',
281 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
282 'Role' : self.plc_spec['role']
284 def locate_site (self,sitename):
285 for site in self.plc_spec['sites']:
286 if site['site_fields']['name'] == sitename:
288 if site['site_fields']['login_base'] == sitename:
290 raise Exception,"Cannot locate site %s"%sitename
292 def locate_node (self,nodename):
293 for site in self.plc_spec['sites']:
294 for node in site['nodes']:
295 if node['name'] == nodename:
297 raise Exception,"Cannot locate node %s"%nodename
299 def locate_hostname (self,hostname):
300 for site in self.plc_spec['sites']:
301 for node in site['nodes']:
302 if node['node_fields']['hostname'] == hostname:
304 raise Exception,"Cannot locate hostname %s"%hostname
306 def locate_key (self,key_name):
307 for key in self.plc_spec['keys']:
308 if key['key_name'] == key_name:
310 raise Exception,"Cannot locate key %s"%key_name
312 def locate_private_key_from_key_names (self, key_names):
313 # locate the first avail. key
315 for key_name in key_names:
316 key_spec=self.locate_key(key_name)
317 test_key=TestKey(self,key_spec)
318 publickey=test_key.publicpath()
319 privatekey=test_key.privatepath()
320 if os.path.isfile(publickey) and os.path.isfile(privatekey):
322 if found: return privatekey
325 def locate_slice (self, slicename):
326 for slice in self.plc_spec['slices']:
327 if slice['slice_fields']['name'] == slicename:
329 raise Exception,"Cannot locate slice %s"%slicename
331 def all_sliver_objs (self):
333 for slice_spec in self.plc_spec['slices']:
334 slicename = slice_spec['slice_fields']['name']
335 for nodename in slice_spec['nodenames']:
336 result.append(self.locate_sliver_obj (nodename,slicename))
339 def locate_sliver_obj (self,nodename,slicename):
340 (site,node) = self.locate_node(nodename)
341 slice = self.locate_slice (slicename)
343 test_site = TestSite (self, site)
344 test_node = TestNode (self, test_site,node)
345 # xxx the slice site is assumed to be the node site - mhh - probably harmless
346 test_slice = TestSlice (self, test_site, slice)
347 return TestSliver (self, test_node, test_slice)
349 def locate_first_node(self):
350 nodename=self.plc_spec['slices'][0]['nodenames'][0]
351 (site,node) = self.locate_node(nodename)
352 test_site = TestSite (self, site)
353 test_node = TestNode (self, test_site,node)
356 def locate_first_sliver (self):
357 slice_spec=self.plc_spec['slices'][0]
358 slicename=slice_spec['slice_fields']['name']
359 nodename=slice_spec['nodenames'][0]
360 return self.locate_sliver_obj(nodename,slicename)
362 # all different hostboxes used in this plc
363 def get_BoxNodes(self):
364 # maps on sites and nodes, return [ (host_box,test_node) ]
366 for site_spec in self.plc_spec['sites']:
367 test_site = TestSite (self,site_spec)
368 for node_spec in site_spec['nodes']:
369 test_node = TestNode (self, test_site, node_spec)
370 if not test_node.is_real():
371 tuples.append( (test_node.host_box(),test_node) )
372 # transform into a dict { 'host_box' -> [ test_node .. ] }
374 for (box,node) in tuples:
375 if not result.has_key(box):
378 result[box].append(node)
381 # a step for checking this stuff
382 def show_boxes (self):
383 'print summary of nodes location'
384 for (box,nodes) in self.get_BoxNodes().iteritems():
385 print box,":"," + ".join( [ node.name() for node in nodes ] )
388 # make this a valid step
389 def qemu_kill_all(self):
390 'kill all qemu instances on the qemu boxes involved by this setup'
391 # this is the brute force version, kill all qemus on that host box
392 for (box,nodes) in self.get_BoxNodes().iteritems():
393 # pass the first nodename, as we don't push template-qemu on testboxes
394 nodedir=nodes[0].nodedir()
395 TestBoxQemu(box,self.options.buildname).qemu_kill_all(nodedir)
398 # make this a valid step
399 def qemu_list_all(self):
400 'list all qemu instances on the qemu boxes involved by this setup'
401 for (box,nodes) in self.get_BoxNodes().iteritems():
402 # this is the brute force version, kill all qemus on that host box
403 TestBoxQemu(box,self.options.buildname).qemu_list_all()
406 # kill only the qemus related to this test
407 def qemu_list_mine(self):
408 'list qemu instances for our nodes'
409 for (box,nodes) in self.get_BoxNodes().iteritems():
410 # the fine-grain version
415 # kill only the qemus related to this test
416 def qemu_clean_mine(self):
417 'cleanup (rm -rf) qemu instances for our nodes'
418 for (box,nodes) in self.get_BoxNodes().iteritems():
419 # the fine-grain version
424 # kill only the right qemus
425 def qemu_kill_mine(self):
426 'kill the qemu instances for our nodes'
427 for (box,nodes) in self.get_BoxNodes().iteritems():
428 # the fine-grain version
433 #################### display config
435 "show test configuration after localization"
440 # uggly hack to make sure 'run export' only reports about the 1st plc
441 # to avoid confusion - also we use 'inri_slice1' in various aliases..
444 "print cut'n paste-able stuff to export env variables to your shell"
445 # guess local domain from hostname
446 if TestPlc.exported_id>1:
447 print "export GUESTHOSTNAME%d=%s"%(TestPlc.exported_id,self.plc_spec['vservername'])
449 TestPlc.exported_id+=1
450 domain=socket.gethostname().split('.',1)[1]
451 fqdn="%s.%s"%(self.plc_spec['host_box'],domain)
452 print "export BUILD=%s"%self.options.buildname
453 if self.options.plcs_use_lxc:
454 print "export PLCHOSTLXC=%s"%fqdn
456 print "export PLCHOSTVS=%s"%fqdn
457 print "export GUESTNAME=%s"%self.plc_spec['vservername']
458 vplcname=self.plc_spec['vservername'].split('-')[-1]
459 print "export GUESTHOSTNAME=%s.%s"%(vplcname,domain)
460 # find hostname of first node
461 (hostname,qemubox) = self.all_node_infos()[0]
462 print "export KVMHOST=%s.%s"%(qemubox,domain)
463 print "export NODE=%s"%(hostname)
467 always_display_keys=['PLC_WWW_HOST','nodes','sites',]
468 def show_pass (self,passno):
469 for (key,val) in self.plc_spec.iteritems():
470 if not self.options.verbose and key not in TestPlc.always_display_keys: continue
474 self.display_site_spec(site)
475 for node in site['nodes']:
476 self.display_node_spec(node)
477 elif key=='initscripts':
478 for initscript in val:
479 self.display_initscript_spec (initscript)
482 self.display_slice_spec (slice)
485 self.display_key_spec (key)
487 if key not in ['sites','initscripts','slices','keys', 'sfa']:
488 print '+ ',key,':',val
490 def display_site_spec (self,site):
491 print '+ ======== site',site['site_fields']['name']
492 for (k,v) in site.iteritems():
493 if not self.options.verbose and k not in TestPlc.always_display_keys: continue
496 print '+ ','nodes : ',
498 print node['node_fields']['hostname'],'',
504 print user['name'],'',
506 elif k == 'site_fields':
507 print '+ login_base',':',v['login_base']
508 elif k == 'address_fields':
514 def display_initscript_spec (self,initscript):
515 print '+ ======== initscript',initscript['initscript_fields']['name']
517 def display_key_spec (self,key):
518 print '+ ======== key',key['key_name']
520 def display_slice_spec (self,slice):
521 print '+ ======== slice',slice['slice_fields']['name']
522 for (k,v) in slice.iteritems():
535 elif k=='slice_fields':
536 print '+ fields',':',
537 print 'max_nodes=',v['max_nodes'],
542 def display_node_spec (self,node):
543 print "+ node=%s host_box=%s"%(node['name'],node['host_box']),
544 print "hostname=",node['node_fields']['hostname'],
545 print "ip=",node['interface_fields']['ip']
546 if self.options.verbose:
547 utils.pprint("node details",node,depth=3)
549 # another entry point for just showing the boxes involved
550 def display_mapping (self):
551 TestPlc.display_mapping_plc(self.plc_spec)
555 def display_mapping_plc (plc_spec):
556 print '+ MyPLC',plc_spec['name']
557 # WARNING this would not be right for lxc-based PLC's - should be harmless though
558 print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['host_box'],plc_spec['vservername'])
559 print '+\tIP = %s/%s'%(plc_spec['PLC_API_HOST'],plc_spec['vserverip'])
560 for site_spec in plc_spec['sites']:
561 for node_spec in site_spec['nodes']:
562 TestPlc.display_mapping_node(node_spec)
565 def display_mapping_node (node_spec):
566 print '+ NODE %s'%(node_spec['name'])
567 print '+\tqemu box %s'%node_spec['host_box']
568 print '+\thostname=%s'%node_spec['node_fields']['hostname']
570 # write a timestamp in /vservers/<>.timestamp
571 # cannot be inside the vserver, that causes vserver .. build to cough
572 def timestamp_vs (self):
573 "Create a timestamp to remember creation date for this plc"
575 # TODO-lxc check this one
576 # a first approx. is to store the timestamp close to the VM root like vs does
577 stamp_path=self.vm_timestamp_path ()
578 stamp_dir = os.path.dirname (stamp_path)
579 utils.system(self.test_ssh.actual_command("mkdir -p %s"%stamp_dir))
580 return utils.system(self.test_ssh.actual_command("echo %d > %s"%(now,stamp_path)))==0
582 # this is called inconditionnally at the beginning of the test sequence
583 # just in case this is a rerun, so if the vm is not running it's fine
585 "vserver delete the test myplc"
586 stamp_path=self.vm_timestamp_path()
587 self.run_in_host("rm -f %s"%stamp_path)
588 if self.options.plcs_use_lxc:
589 self.run_in_host("virsh -c lxc:// destroy %s"%self.vservername)
590 self.run_in_host("virsh -c lxc:// undefine %s"%self.vservername)
591 self.run_in_host("rm -fr /vservers/%s"%self.vservername)
594 self.run_in_host("vserver --silent %s delete"%self.vservername)
598 # historically the build was being fetched by the tests
599 # now the build pushes itself as a subdir of the tests workdir
600 # so that the tests do not have to worry about extracting the build (svn, git, or whatever)
601 def vs_create (self):
602 "vserver creation (no install done)"
603 # push the local build/ dir to the testplc box
605 # a full path for the local calls
606 build_dir=os.path.dirname(sys.argv[0])
607 # sometimes this is empty - set to "." in such a case
608 if not build_dir: build_dir="."
609 build_dir += "/build"
611 # use a standard name - will be relative to remote buildname
613 # remove for safety; do *not* mkdir first, otherwise we end up with build/build/
614 self.test_ssh.rmdir(build_dir)
615 self.test_ssh.copy(build_dir,recursive=True)
616 # the repo url is taken from arch-rpms-url
617 # with the last step (i386) removed
618 repo_url = self.options.arch_rpms_url
619 for level in [ 'arch' ]:
620 repo_url = os.path.dirname(repo_url)
621 # pass the vbuild-nightly options to vtest-init-vserver
623 test_env_options += " -p %s"%self.options.personality
624 test_env_options += " -d %s"%self.options.pldistro
625 test_env_options += " -f %s"%self.options.fcdistro
626 if self.options.plcs_use_lxc:
627 script="vtest-init-lxc.sh"
629 script="vtest-init-vserver.sh"
630 vserver_name = self.vservername
631 vserver_options="--netdev eth0 --interface %s"%self.vserverip
633 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
634 vserver_options += " --hostname %s"%vserver_hostname
636 print "Cannot reverse lookup %s"%self.vserverip
637 print "This is considered fatal, as this might pollute the test results"
639 create_vserver="%(build_dir)s/%(script)s %(test_env_options)s %(vserver_name)s %(repo_url)s -- %(vserver_options)s"%locals()
640 return self.run_in_host(create_vserver) == 0
643 def plc_install(self):
644 "yum install myplc, noderepo, and the plain bootstrapfs"
646 # workaround for getting pgsql8.2 on centos5
647 if self.options.fcdistro == "centos5":
648 self.run_in_guest("rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-3.noarch.rpm")
651 if self.options.personality == "linux32":
653 elif self.options.personality == "linux64":
656 raise Exception, "Unsupported personality %r"%self.options.personality
657 nodefamily="%s-%s-%s"%(self.options.pldistro,self.options.fcdistro,arch)
660 pkgs_list.append ("slicerepo-%s"%nodefamily)
661 pkgs_list.append ("myplc")
662 pkgs_list.append ("noderepo-%s"%nodefamily)
663 pkgs_list.append ("nodeimage-%s-plain"%nodefamily)
664 pkgs_string=" ".join(pkgs_list)
665 return self.yum_install (pkgs_list)
668 def mod_python(self):
669 """yum install mod_python, useful on f18 and above so as to avoid broken wsgi"""
670 return self.yum_install ( [ 'mod_python' ] )
673 def plc_configure(self):
675 tmpname='%s.plc-config-tty'%(self.name())
676 fileconf=open(tmpname,'w')
677 for var in [ 'PLC_NAME',
682 'PLC_MAIL_SUPPORT_ADDRESS',
685 # Above line was added for integrating SFA Testing
691 'PLC_RESERVATION_GRANULARITY',
693 'PLC_OMF_XMPP_SERVER',
696 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
697 fileconf.write('w\n')
698 fileconf.write('q\n')
700 utils.system('cat %s'%tmpname)
701 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
702 utils.system('rm %s'%tmpname)
707 self.run_in_guest('service plc start')
712 self.run_in_guest('service plc stop')
716 "start the PLC vserver"
721 "stop the PLC vserver"
725 # stores the keys from the config for further use
726 def keys_store(self):
727 "stores test users ssh keys in keys/"
728 for key_spec in self.plc_spec['keys']:
729 TestKey(self,key_spec).store_key()
732 def keys_clean(self):
733 "removes keys cached in keys/"
734 utils.system("rm -rf ./keys")
737 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
738 # for later direct access to the nodes
739 def keys_fetch(self):
740 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
742 if not os.path.isdir(dir):
744 vservername=self.vservername
745 vm_root=self.vm_root_in_host()
747 prefix = 'debug_ssh_key'
748 for ext in [ 'pub', 'rsa' ] :
749 src="%(vm_root)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
750 dst="keys/%(vservername)s-debug.%(ext)s"%locals()
751 if self.test_ssh.fetch(src,dst) != 0: overall=False
755 "create sites with PLCAPI"
756 return self.do_sites()
758 def delete_sites (self):
759 "delete sites with PLCAPI"
760 return self.do_sites(action="delete")
762 def do_sites (self,action="add"):
763 for site_spec in self.plc_spec['sites']:
764 test_site = TestSite (self,site_spec)
765 if (action != "add"):
766 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
767 test_site.delete_site()
768 # deleted with the site
769 #test_site.delete_users()
772 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
773 test_site.create_site()
774 test_site.create_users()
777 def delete_all_sites (self):
778 "Delete all sites in PLC, and related objects"
779 print 'auth_root',self.auth_root()
780 sites = self.apiserver.GetSites(self.auth_root(), {}, ['site_id','login_base'])
782 # keep automatic site - otherwise we shoot in our own foot, root_auth is not valid anymore
783 if site['login_base']==self.plc_spec['PLC_SLICE_PREFIX']: continue
784 site_id=site['site_id']
785 print 'Deleting site_id',site_id
786 self.apiserver.DeleteSite(self.auth_root(),site_id)
790 "create nodes with PLCAPI"
791 return self.do_nodes()
792 def delete_nodes (self):
793 "delete nodes with PLCAPI"
794 return self.do_nodes(action="delete")
796 def do_nodes (self,action="add"):
797 for site_spec in self.plc_spec['sites']:
798 test_site = TestSite (self,site_spec)
800 utils.header("Deleting nodes in site %s"%test_site.name())
801 for node_spec in site_spec['nodes']:
802 test_node=TestNode(self,test_site,node_spec)
803 utils.header("Deleting %s"%test_node.name())
804 test_node.delete_node()
806 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
807 for node_spec in site_spec['nodes']:
808 utils.pprint('Creating node %s'%node_spec,node_spec)
809 test_node = TestNode (self,test_site,node_spec)
810 test_node.create_node ()
813 def nodegroups (self):
814 "create nodegroups with PLCAPI"
815 return self.do_nodegroups("add")
816 def delete_nodegroups (self):
817 "delete nodegroups with PLCAPI"
818 return self.do_nodegroups("delete")
822 def translate_timestamp (start,grain,timestamp):
823 if timestamp < TestPlc.YEAR: return start+timestamp*grain
824 else: return timestamp
827 def timestamp_printable (timestamp):
828 return time.strftime('%m-%d %H:%M:%S UTC',time.gmtime(timestamp))
831 "create leases (on reservable nodes only, use e.g. run -c default -c resa)"
833 grain=self.apiserver.GetLeaseGranularity(self.auth_root())
834 print 'API answered grain=',grain
835 start=(now/grain)*grain
837 # find out all nodes that are reservable
838 nodes=self.all_reservable_nodenames()
840 utils.header ("No reservable node found - proceeding without leases")
843 # attach them to the leases as specified in plc_specs
844 # this is where the 'leases' field gets interpreted as relative of absolute
845 for lease_spec in self.plc_spec['leases']:
846 # skip the ones that come with a null slice id
847 if not lease_spec['slice']: continue
848 lease_spec['t_from']=TestPlc.translate_timestamp(start,grain,lease_spec['t_from'])
849 lease_spec['t_until']=TestPlc.translate_timestamp(start,grain,lease_spec['t_until'])
850 lease_addition=self.apiserver.AddLeases(self.auth_root(),nodes,
851 lease_spec['slice'],lease_spec['t_from'],lease_spec['t_until'])
852 if lease_addition['errors']:
853 utils.header("Cannot create leases, %s"%lease_addition['errors'])
856 utils.header('Leases on nodes %r for %s from %d (%s) until %d (%s)'%\
857 (nodes,lease_spec['slice'],
858 lease_spec['t_from'],TestPlc.timestamp_printable(lease_spec['t_from']),
859 lease_spec['t_until'],TestPlc.timestamp_printable(lease_spec['t_until'])))
863 def delete_leases (self):
864 "remove all leases in the myplc side"
865 lease_ids= [ l['lease_id'] for l in self.apiserver.GetLeases(self.auth_root())]
866 utils.header("Cleaning leases %r"%lease_ids)
867 self.apiserver.DeleteLeases(self.auth_root(),lease_ids)
870 def list_leases (self):
871 "list all leases known to the myplc"
872 leases = self.apiserver.GetLeases(self.auth_root())
875 current=l['t_until']>=now
876 if self.options.verbose or current:
877 utils.header("%s %s from %s until %s"%(l['hostname'],l['name'],
878 TestPlc.timestamp_printable(l['t_from']),
879 TestPlc.timestamp_printable(l['t_until'])))
882 # create nodegroups if needed, and populate
883 def do_nodegroups (self, action="add"):
884 # 1st pass to scan contents
886 for site_spec in self.plc_spec['sites']:
887 test_site = TestSite (self,site_spec)
888 for node_spec in site_spec['nodes']:
889 test_node=TestNode (self,test_site,node_spec)
890 if node_spec.has_key('nodegroups'):
891 nodegroupnames=node_spec['nodegroups']
892 if isinstance(nodegroupnames,StringTypes):
893 nodegroupnames = [ nodegroupnames ]
894 for nodegroupname in nodegroupnames:
895 if not groups_dict.has_key(nodegroupname):
896 groups_dict[nodegroupname]=[]
897 groups_dict[nodegroupname].append(test_node.name())
898 auth=self.auth_root()
900 for (nodegroupname,group_nodes) in groups_dict.iteritems():
902 print 'nodegroups:','dealing with nodegroup',nodegroupname,'on nodes',group_nodes
903 # first, check if the nodetagtype is here
904 tag_types = self.apiserver.GetTagTypes(auth,{'tagname':nodegroupname})
906 tag_type_id = tag_types[0]['tag_type_id']
908 tag_type_id = self.apiserver.AddTagType(auth,
909 {'tagname':nodegroupname,
910 'description': 'for nodegroup %s'%nodegroupname,
912 print 'located tag (type)',nodegroupname,'as',tag_type_id
914 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
916 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
917 print 'created nodegroup',nodegroupname,'from tagname',nodegroupname,'and value','yes'
918 # set node tag on all nodes, value='yes'
919 for nodename in group_nodes:
921 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
923 traceback.print_exc()
924 print 'node',nodename,'seems to already have tag',nodegroupname
927 expect_yes = self.apiserver.GetNodeTags(auth,
928 {'hostname':nodename,
929 'tagname':nodegroupname},
930 ['value'])[0]['value']
931 if expect_yes != "yes":
932 print 'Mismatch node tag on node',nodename,'got',expect_yes
935 if not self.options.dry_run:
936 print 'Cannot find tag',nodegroupname,'on node',nodename
940 print 'cleaning nodegroup',nodegroupname
941 self.apiserver.DeleteNodeGroup(auth,nodegroupname)
943 traceback.print_exc()
947 # a list of TestNode objs
948 def all_nodes (self):
950 for site_spec in self.plc_spec['sites']:
951 test_site = TestSite (self,site_spec)
952 for node_spec in site_spec['nodes']:
953 nodes.append(TestNode (self,test_site,node_spec))
956 # return a list of tuples (nodename,qemuname)
957 def all_node_infos (self) :
959 for site_spec in self.plc_spec['sites']:
960 node_infos += [ (node_spec['node_fields']['hostname'],node_spec['host_box']) \
961 for node_spec in site_spec['nodes'] ]
964 def all_nodenames (self): return [ x[0] for x in self.all_node_infos() ]
965 def all_reservable_nodenames (self):
967 for site_spec in self.plc_spec['sites']:
968 for node_spec in site_spec['nodes']:
969 node_fields=node_spec['node_fields']
970 if 'node_type' in node_fields and node_fields['node_type']=='reservable':
971 res.append(node_fields['hostname'])
974 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
975 def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period_seconds=15):
976 if self.options.dry_run:
980 class CompleterTaskBootState (CompleterTask):
981 def __init__ (self, test_plc,hostname):
982 self.test_plc=test_plc
983 self.hostname=hostname
984 self.last_boot_state='undef'
985 def actual_run (self):
987 node = self.test_plc.apiserver.GetNodes(self.test_plc.auth_root(), [ self.hostname ],
989 self.last_boot_state = node['boot_state']
990 return self.last_boot_state == target_boot_state
994 return "CompleterTaskBootState with node %s"%self.hostname
995 def failure_message (self):
996 return "node %s in state %s - expected %s"%(self.hostname,self.last_boot_state,target_boot_state)
998 timeout = timedelta(minutes=timeout_minutes)
999 graceout = timedelta(minutes=silent_minutes)
1000 period = timedelta(seconds=period_seconds)
1001 # the nodes that haven't checked yet - start with a full list and shrink over time
1002 utils.header("checking nodes boot state (expected %s)"%target_boot_state)
1003 tasks = [ CompleterTaskBootState (self,hostname) \
1004 for (hostname,_) in self.all_node_infos() ]
1005 return Completer (tasks).run (timeout, graceout, period)
1007 def nodes_booted(self):
1008 return self.nodes_check_boot_state('boot',timeout_minutes=30,silent_minutes=28)
1010 def check_nodes_ssh(self,debug,timeout_minutes,silent_minutes,period_seconds=15):
1011 class CompleterTaskNodeSsh (CompleterTask):
1012 def __init__ (self, hostname, qemuname, boot_state, local_key):
1013 self.hostname=hostname
1014 self.qemuname=qemuname
1015 self.boot_state=boot_state
1016 self.local_key=local_key
1017 def run (self, silent):
1018 command = TestSsh (self.hostname,key=self.local_key).actual_command("hostname;uname -a")
1019 return utils.system (command, silent=silent)==0
1020 def failure_message (self):
1021 return "Cannot reach %s @ %s in %s mode"%(self.hostname, self.qemuname, self.boot_state)
1024 timeout = timedelta(minutes=timeout_minutes)
1025 graceout = timedelta(minutes=silent_minutes)
1026 period = timedelta(seconds=period_seconds)
1027 vservername=self.vservername
1030 local_key = "keys/%(vservername)s-debug.rsa"%locals()
1033 local_key = "keys/key_admin.rsa"
1034 utils.header("checking ssh access to nodes (expected in %s mode)"%message)
1035 node_infos = self.all_node_infos()
1036 tasks = [ CompleterTaskNodeSsh (nodename, qemuname, message, local_key) \
1037 for (nodename,qemuname) in node_infos ]
1038 return Completer (tasks).run (timeout, graceout, period)
1040 def ssh_node_debug(self):
1041 "Tries to ssh into nodes in debug mode with the debug ssh key"
1042 return self.check_nodes_ssh(debug=True,
1043 timeout_minutes=self.ssh_node_debug_timeout,
1044 silent_minutes=self.ssh_node_debug_silent)
1046 def ssh_node_boot(self):
1047 "Tries to ssh into nodes in production mode with the root ssh key"
1048 return self.check_nodes_ssh(debug=False,
1049 timeout_minutes=self.ssh_node_boot_timeout,
1050 silent_minutes=self.ssh_node_boot_silent)
1052 def node_bmlogs(self):
1053 "Checks that there's a non-empty dir. /var/log/bm/raw"
1054 return utils.system(self.actual_command_in_guest("ls /var/log/bm/raw"))==0
1057 def qemu_local_init (self): pass
1059 def bootcd (self): pass
1061 def qemu_local_config (self): pass
1063 def nodestate_reinstall (self): pass
1065 def nodestate_safeboot (self): pass
1067 def nodestate_boot (self): pass
1069 def nodestate_show (self): pass
1071 def qemu_export (self): pass
1073 ### check hooks : invoke scripts from hooks/{node,slice}
1074 def check_hooks_node (self):
1075 return self.locate_first_node().check_hooks()
1076 def check_hooks_sliver (self) :
1077 return self.locate_first_sliver().check_hooks()
1079 def check_hooks (self):
1080 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
1081 return self.check_hooks_node() and self.check_hooks_sliver()
1084 def do_check_initscripts(self):
1085 class CompleterTaskInitscript (CompleterTask):
1086 def __init__ (self, test_sliver, stamp):
1087 self.test_sliver=test_sliver
1089 def actual_run (self):
1090 return self.test_sliver.check_initscript_stamp (self.stamp)
1092 return "initscript checker for %s"%self.test_sliver.name()
1093 def failure_message (self):
1094 return "initscript stamp %s not found in sliver %s"%(self.stamp,self.test_sliver.name())
1097 for slice_spec in self.plc_spec['slices']:
1098 if not slice_spec.has_key('initscriptstamp'):
1100 stamp=slice_spec['initscriptstamp']
1101 slicename=slice_spec['slice_fields']['name']
1102 for nodename in slice_spec['nodenames']:
1103 print 'nodename',nodename,'slicename',slicename,'stamp',stamp
1104 (site,node) = self.locate_node (nodename)
1105 # xxx - passing the wrong site - probably harmless
1106 test_site = TestSite (self,site)
1107 test_slice = TestSlice (self,test_site,slice_spec)
1108 test_node = TestNode (self,test_site,node)
1109 test_sliver = TestSliver (self, test_node, test_slice)
1110 tasks.append ( CompleterTaskInitscript (test_sliver, stamp))
1111 return Completer (tasks).run (timedelta(minutes=5), timedelta(minutes=4), timedelta(seconds=10))
1113 def check_initscripts(self):
1114 "check that the initscripts have triggered"
1115 return self.do_check_initscripts()
1117 def initscripts (self):
1118 "create initscripts with PLCAPI"
1119 for initscript in self.plc_spec['initscripts']:
1120 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
1121 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
1124 def delete_initscripts (self):
1125 "delete initscripts with PLCAPI"
1126 for initscript in self.plc_spec['initscripts']:
1127 initscript_name = initscript['initscript_fields']['name']
1128 print('Attempting to delete %s in plc %s'%(initscript_name,self.plc_spec['name']))
1130 self.apiserver.DeleteInitScript(self.auth_root(),initscript_name)
1131 print initscript_name,'deleted'
1133 print 'deletion went wrong - probably did not exist'
1138 "create slices with PLCAPI"
1139 return self.do_slices(action="add")
1141 def delete_slices (self):
1142 "delete slices with PLCAPI"
1143 return self.do_slices(action="delete")
1145 def fill_slices (self):
1146 "add nodes in slices with PLCAPI"
1147 return self.do_slices(action="fill")
1149 def empty_slices (self):
1150 "remove nodes from slices with PLCAPI"
1151 return self.do_slices(action="empty")
1153 def do_slices (self, action="add"):
1154 for slice in self.plc_spec['slices']:
1155 site_spec = self.locate_site (slice['sitename'])
1156 test_site = TestSite(self,site_spec)
1157 test_slice=TestSlice(self,test_site,slice)
1158 if action == "delete":
1159 test_slice.delete_slice()
1160 elif action=="fill":
1161 test_slice.add_nodes()
1162 elif action=="empty":
1163 test_slice.delete_nodes()
1165 test_slice.create_slice()
1168 @slice_mapper__tasks(20,10,15)
1169 def ssh_slice(self): pass
1170 @slice_mapper__tasks(20,19,15)
1171 def ssh_slice_off (self): pass
1174 def ssh_slice_basics(self): pass
1177 def check_vsys_defaults(self): pass
1180 def keys_clear_known_hosts (self): pass
1182 def plcapi_urls (self):
1183 return PlcapiUrlScanner (self.auth_root(),ip=self.vserverip).scan()
1185 def speed_up_slices (self):
1186 "tweak nodemanager settings on all nodes using a conf file"
1187 # create the template on the server-side
1188 template="%s.nodemanager"%self.name()
1189 template_file = open (template,"w")
1190 template_file.write('OPTIONS="-p 30 -r 11 -d"\n')
1191 template_file.close()
1192 in_vm="/var/www/html/PlanetLabConf/nodemanager"
1193 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1194 self.test_ssh.copy_abs(template,remote)
1196 self.apiserver.AddConfFile (self.auth_root(),
1197 {'dest':'/etc/sysconfig/nodemanager',
1198 'source':'PlanetLabConf/nodemanager',
1199 'postinstall_cmd':'service nm restart',})
1202 def debug_nodemanager (self):
1203 "sets verbose mode for nodemanager, and speeds up cycle even more (needs speed_up_slices first)"
1204 template="%s.nodemanager"%self.name()
1205 template_file = open (template,"w")
1206 template_file.write('OPTIONS="-p 10 -r 6 -v -d"\n')
1207 template_file.close()
1208 in_vm="/var/www/html/PlanetLabConf/nodemanager"
1209 remote="%s/%s"%(self.vm_root_in_host(),in_vm)
1210 self.test_ssh.copy_abs(template,remote)
1214 def qemu_start (self) : pass
1217 def timestamp_qemu (self) : pass
1219 # when a spec refers to a node possibly on another plc
1220 def locate_sliver_obj_cross (self, nodename, slicename, other_plcs):
1221 for plc in [ self ] + other_plcs:
1223 return plc.locate_sliver_obj (nodename, slicename)
1226 raise Exception, "Cannot locate sliver %s@%s among all PLCs"%(nodename,slicename)
1228 # implement this one as a cross step so that we can take advantage of different nodes
1229 # in multi-plcs mode
1230 def cross_check_tcp (self, other_plcs):
1231 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
1232 if 'tcp_specs' not in self.plc_spec or not self.plc_spec['tcp_specs']:
1233 utils.header ("check_tcp: no/empty config found")
1235 specs = self.plc_spec['tcp_specs']
1240 s_test_sliver = self.locate_sliver_obj_cross (spec['server_node'],spec['server_slice'],other_plcs)
1241 if not s_test_sliver.run_tcp_server(port,timeout=20):
1245 # idem for the client side
1246 c_test_sliver = self.locate_sliver_obj_cross (spec['client_node'],spec['client_slice'],other_plcs)
1247 # use nodename from locatesd sliver, unless 'client_connect' is set
1248 if 'client_connect' in spec:
1249 destination = spec['client_connect']
1251 destination=s_test_sliver.test_node.name()
1252 if not c_test_sliver.run_tcp_client(destination,port):
1256 # painfully enough, we need to allow for some time as netflow might show up last
1257 def check_system_slice (self):
1258 "all nodes: check that a system slice is alive"
1259 # netflow currently not working in the lxc distro
1260 # drl not built at all in the wtx distro
1261 # if we find either of them we're happy
1262 return self.check_netflow() or self.check_drl()
1265 def check_netflow (self): return self._check_system_slice ('netflow')
1266 def check_drl (self): return self._check_system_slice ('drl')
1268 # we have the slices up already here, so it should not take too long
1269 def _check_system_slice (self, slicename, timeout_minutes=5, period_seconds=15):
1270 class CompleterTaskSystemSlice (CompleterTask):
1271 def __init__ (self, test_node, dry_run):
1272 self.test_node=test_node
1273 self.dry_run=dry_run
1274 def actual_run (self):
1275 return self.test_node._check_system_slice (slicename, dry_run=self.dry_run)
1277 return "System slice %s @ %s"%(slicename, self.test_node.name())
1278 def failure_message (self):
1279 return "COULD not find system slice %s @ %s"%(slicename, self.test_node.name())
1280 timeout = timedelta(minutes=timeout_minutes)
1281 silent = timedelta (0)
1282 period = timedelta (seconds=period_seconds)
1283 tasks = [ CompleterTaskSystemSlice (test_node, self.options.dry_run) \
1284 for test_node in self.all_nodes() ]
1285 return Completer (tasks) . run (timeout, silent, period)
1287 def plcsh_stress_test (self):
1288 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
1289 # install the stress-test in the plc image
1290 location = "/usr/share/plc_api/plcsh_stress_test.py"
1291 remote="%s/%s"%(self.vm_root_in_host(),location)
1292 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1294 command += " -- --check"
1295 if self.options.size == 1:
1296 command += " --tiny"
1297 return ( self.run_in_guest(command) == 0)
1299 # populate runs the same utility without slightly different options
1300 # in particular runs with --preserve (dont cleanup) and without --check
1301 # also it gets run twice, once with the --foreign option for creating fake foreign entries
1303 def sfa_install_all (self):
1304 "yum install sfa sfa-plc sfa-sfatables sfa-client"
1305 return self.yum_install ("sfa sfa-plc sfa-sfatables sfa-client")
1307 def sfa_install_core(self):
1309 return self.yum_install ("sfa")
1311 def sfa_install_plc(self):
1312 "yum install sfa-plc"
1313 return self.yum_install("sfa-plc")
1315 def sfa_install_sfatables(self):
1316 "yum install sfa-sfatables"
1317 return self.yum_install ("sfa-sfatables")
1319 # for some very odd reason, this sometimes fails with the following symptom
1320 # # yum install sfa-client
1321 # Setting up Install Process
1323 # Downloading Packages:
1324 # Running rpm_check_debug
1325 # Running Transaction Test
1326 # Transaction Test Succeeded
1327 # Running Transaction
1328 # Transaction couldn't start:
1329 # installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem
1330 # [('installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem', (9, '/', 69632L))]
1331 # even though in the same context I have
1332 # [2012.05.23--f14-32-sfastd1-1-vplc07] / # df -h
1333 # Filesystem Size Used Avail Use% Mounted on
1334 # /dev/hdv1 806G 264G 501G 35% /
1335 # none 16M 36K 16M 1% /tmp
1337 # so as a workaround, we first try yum install, and then invoke rpm on the cached rpm...
1338 def sfa_install_client(self):
1339 "yum install sfa-client"
1340 first_try=self.yum_install("sfa-client")
1341 if first_try: return True
1342 utils.header ("********** Regular yum failed - special workaround in place, 2nd chance")
1343 (code,cached_rpm_path)=utils.output_of(self.actual_command_in_guest('find /var/cache/yum -name sfa-client\*.rpm'))
1344 utils.header("rpm_path=<<%s>>"%rpm_path)
1346 self.run_in_guest("rpm -i %s"%cached_rpm_path)
1347 return self.yum_check_installed ("sfa-client")
1349 def sfa_dbclean(self):
1350 "thoroughly wipes off the SFA database"
1351 return self.run_in_guest("sfaadmin reg nuke")==0 or \
1352 self.run_in_guest("sfa-nuke.py")==0 or \
1353 self.run_in_guest("sfa-nuke-plc.py")==0
1355 def sfa_fsclean(self):
1356 "cleanup /etc/sfa/trusted_roots and /var/lib/sfa"
1357 self.run_in_guest("rm -rf /etc/sfa/trusted_roots /var/lib/sfa/authorities")
1360 def sfa_plcclean(self):
1361 "cleans the PLC entries that were created as a side effect of running the script"
1363 sfa_spec=self.plc_spec['sfa']
1365 for auth_sfa_spec in sfa_spec['auth_sfa_specs']:
1366 login_base=auth_sfa_spec['login_base']
1367 try: self.apiserver.DeleteSite (self.auth_root(),login_base)
1368 except: print "Site %s already absent from PLC db"%login_base
1370 for spec_name in ['pi_spec','user_spec']:
1371 user_spec=auth_sfa_spec[spec_name]
1372 username=user_spec['email']
1373 try: self.apiserver.DeletePerson(self.auth_root(),username)
1375 # this in fact is expected as sites delete their members
1376 #print "User %s already absent from PLC db"%username
1379 print "REMEMBER TO RUN sfa_import AGAIN"
1382 def sfa_uninstall(self):
1383 "uses rpm to uninstall sfa - ignore result"
1384 self.run_in_guest("rpm -e sfa sfa-sfatables sfa-client sfa-plc")
1385 self.run_in_guest("rm -rf /var/lib/sfa")
1386 self.run_in_guest("rm -rf /etc/sfa")
1387 self.run_in_guest("rm -rf /var/log/sfa_access.log /var/log/sfa_import_plc.log /var/log/sfa.daemon")
1389 self.run_in_guest("rpm -e --noscripts sfa-plc")
1392 ### run unit tests for SFA
1393 # NOTE: for some reason on f14/i386, yum install sfa-tests fails for no reason
1394 # Running Transaction
1395 # Transaction couldn't start:
1396 # installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem
1397 # [('installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem', (9, '/', 208896L))]
1398 # no matter how many Gbs are available on the testplc
1399 # could not figure out what's wrong, so...
1400 # if the yum install phase fails, consider the test is successful
1401 # other combinations will eventually run it hopefully
1402 def sfa_utest(self):
1403 "yum install sfa-tests and run SFA unittests"
1404 self.run_in_guest("yum -y install sfa-tests")
1405 # failed to install - forget it
1406 if self.run_in_guest("rpm -q sfa-tests")!=0:
1407 utils.header("WARNING: SFA unit tests failed to install, ignoring")
1409 return self.run_in_guest("/usr/share/sfa/tests/testAll.py")==0
1413 dirname="conf.%s"%self.plc_spec['name']
1414 if not os.path.isdir(dirname):
1415 utils.system("mkdir -p %s"%dirname)
1416 if not os.path.isdir(dirname):
1417 raise Exception,"Cannot create config dir for plc %s"%self.name()
1420 def conffile(self,filename):
1421 return "%s/%s"%(self.confdir(),filename)
1422 def confsubdir(self,dirname,clean,dry_run=False):
1423 subdirname="%s/%s"%(self.confdir(),dirname)
1425 utils.system("rm -rf %s"%subdirname)
1426 if not os.path.isdir(subdirname):
1427 utils.system("mkdir -p %s"%subdirname)
1428 if not dry_run and not os.path.isdir(subdirname):
1429 raise "Cannot create config subdir %s for plc %s"%(dirname,self.name())
1432 def conffile_clean (self,filename):
1433 filename=self.conffile(filename)
1434 return utils.system("rm -rf %s"%filename)==0
1437 def sfa_configure(self):
1438 "run sfa-config-tty"
1439 tmpname=self.conffile("sfa-config-tty")
1440 fileconf=open(tmpname,'w')
1441 for var in [ 'SFA_REGISTRY_ROOT_AUTH',
1442 'SFA_INTERFACE_HRN',
1443 'SFA_REGISTRY_LEVEL1_AUTH',
1444 'SFA_REGISTRY_HOST',
1445 'SFA_AGGREGATE_HOST',
1455 'SFA_GENERIC_FLAVOUR',
1456 'SFA_AGGREGATE_ENABLED',
1458 if self.plc_spec['sfa'].has_key(var):
1459 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec['sfa'][var]))
1460 # the way plc_config handles booleans just sucks..
1463 if self.plc_spec['sfa'][var]: val='true'
1464 fileconf.write ('e %s\n%s\n'%(var,val))
1465 fileconf.write('w\n')
1466 fileconf.write('R\n')
1467 fileconf.write('q\n')
1469 utils.system('cat %s'%tmpname)
1470 self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
1473 def aggregate_xml_line(self):
1474 port=self.plc_spec['sfa']['neighbours-port']
1475 return '<aggregate addr="%s" hrn="%s" port="%r"/>' % \
1476 (self.vserverip,self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH'],port)
1478 def registry_xml_line(self):
1479 return '<registry addr="%s" hrn="%s" port="12345"/>' % \
1480 (self.vserverip,self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH'])
1483 # a cross step that takes all other plcs in argument
1484 def cross_sfa_configure(self, other_plcs):
1485 "writes aggregates.xml and registries.xml that point to all other PLCs in the test"
1486 # of course with a single plc, other_plcs is an empty list
1489 agg_fname=self.conffile("agg.xml")
1490 file(agg_fname,"w").write("<aggregates>%s</aggregates>\n" % \
1491 " ".join([ plc.aggregate_xml_line() for plc in other_plcs ]))
1492 utils.header ("(Over)wrote %s"%agg_fname)
1493 reg_fname=self.conffile("reg.xml")
1494 file(reg_fname,"w").write("<registries>%s</registries>\n" % \
1495 " ".join([ plc.registry_xml_line() for plc in other_plcs ]))
1496 utils.header ("(Over)wrote %s"%reg_fname)
1497 return self.test_ssh.copy_abs(agg_fname,'/%s/etc/sfa/aggregates.xml'%self.vm_root_in_host())==0 \
1498 and self.test_ssh.copy_abs(reg_fname,'/%s/etc/sfa/registries.xml'%self.vm_root_in_host())==0
1500 def sfa_import(self):
1501 "use sfaadmin to import from plc"
1502 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1504 self.run_in_guest('sfaadmin reg import_registry')==0
1505 # not needed anymore
1506 # self.run_in_guest('cp /etc/sfa/authorities/%s/%s.pkey /etc/sfa/authorities/server.key'%(auth,auth))
1508 def sfa_start(self):
1510 return self.run_in_guest('service sfa start')==0
1512 def sfi_configure(self):
1513 "Create /root/sfi on the plc side for sfi client configuration"
1514 if self.options.dry_run:
1515 utils.header("DRY RUN - skipping step")
1517 sfa_spec=self.plc_spec['sfa']
1518 # cannot use auth_sfa_mapper to pass dir_name
1519 for slice_spec in self.plc_spec['sfa']['auth_sfa_specs']:
1520 test_slice=TestAuthSfa(self,slice_spec)
1521 dir_basename=os.path.basename(test_slice.sfi_path())
1522 dir_name=self.confsubdir("dot-sfi/%s"%dir_basename,clean=True,dry_run=self.options.dry_run)
1523 test_slice.sfi_configure(dir_name)
1524 # push into the remote /root/sfi area
1525 location = test_slice.sfi_path()
1526 remote="%s/%s"%(self.vm_root_in_host(),location)
1527 self.test_ssh.mkdir(remote,abs=True)
1528 # need to strip last level or remote otherwise we get an extra dir level
1529 self.test_ssh.copy_abs(dir_name, os.path.dirname(remote), recursive=True)
1533 def sfi_clean (self):
1534 "clean up /root/sfi on the plc side"
1535 self.run_in_guest("rm -rf /root/sfi")
1539 def sfa_add_site (self): pass
1541 def sfa_add_pi (self): pass
1543 def sfa_add_user(self): pass
1545 def sfa_update_user(self): pass
1547 def sfa_add_slice(self): pass
1549 def sfa_renew_slice(self): pass
1551 def sfa_discover(self): pass
1553 def sfa_create_slice(self): pass
1555 def sfa_check_slice_plc(self): pass
1557 def sfa_update_slice(self): pass
1559 def sfi_list(self): pass
1561 def sfi_show(self): pass
1563 def ssh_slice_sfa(self): pass
1565 def sfa_delete_user(self): pass
1567 def sfa_delete_slice(self): pass
1571 self.run_in_guest('service sfa stop')==0
1574 def populate (self):
1575 "creates random entries in the PLCAPI"
1576 # install the stress-test in the plc image
1577 location = "/usr/share/plc_api/plcsh_stress_test.py"
1578 remote="%s/%s"%(self.vm_root_in_host(),location)
1579 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1581 command += " -- --preserve --short-names"
1582 local = (self.run_in_guest(command) == 0);
1583 # second run with --foreign
1584 command += ' --foreign'
1585 remote = (self.run_in_guest(command) == 0);
1586 return ( local and remote)
1588 def gather_logs (self):
1589 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1590 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1591 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1592 # (1.c) get the plc's /root/sfi -> logs/sfi.<plcname>/
1593 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1594 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1595 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1597 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1598 self.gather_var_logs ()
1600 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1601 self.gather_pgsql_logs ()
1603 print "-------------------- TestPlc.gather_logs : PLC's /root/sfi/"
1604 self.gather_root_sfi ()
1606 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1607 for site_spec in self.plc_spec['sites']:
1608 test_site = TestSite (self,site_spec)
1609 for node_spec in site_spec['nodes']:
1610 test_node=TestNode(self,test_site,node_spec)
1611 test_node.gather_qemu_logs()
1613 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1614 self.gather_nodes_var_logs()
1616 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1617 self.gather_slivers_var_logs()
1620 def gather_slivers_var_logs(self):
1621 for test_sliver in self.all_sliver_objs():
1622 remote = test_sliver.tar_var_logs()
1623 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
1624 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
1625 utils.system(command)
1628 def gather_var_logs (self):
1629 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
1630 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1631 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
1632 utils.system(command)
1633 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd"%self.name()
1634 utils.system(command)
1636 def gather_pgsql_logs (self):
1637 utils.system("mkdir -p logs/myplc.pgsql-log.%s"%self.name())
1638 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1639 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -"%self.name()
1640 utils.system(command)
1642 def gather_root_sfi (self):
1643 utils.system("mkdir -p logs/sfi.%s"%self.name())
1644 to_plc = self.actual_command_in_guest("tar -C /root/sfi/ -cf - .")
1645 command = to_plc + "| tar -C logs/sfi.%s -xf -"%self.name()
1646 utils.system(command)
1648 def gather_nodes_var_logs (self):
1649 for site_spec in self.plc_spec['sites']:
1650 test_site = TestSite (self,site_spec)
1651 for node_spec in site_spec['nodes']:
1652 test_node=TestNode(self,test_site,node_spec)
1653 test_ssh = TestSsh (test_node.name(),key="keys/key_admin.rsa")
1654 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1655 command = command + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
1656 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
1657 utils.system(command)
1660 # returns the filename to use for sql dump/restore, using options.dbname if set
1661 def dbfile (self, database):
1662 # uses options.dbname if it is found
1664 name=self.options.dbname
1665 if not isinstance(name,StringTypes):
1671 return "/root/%s-%s.sql"%(database,name)
1673 def plc_db_dump(self):
1674 'dump the planetlab5 DB in /root in the PLC - filename has time'
1675 dump=self.dbfile("planetab5")
1676 self.run_in_guest('pg_dump -U pgsqluser planetlab5 -f '+ dump)
1677 utils.header('Dumped planetlab5 database in %s'%dump)
1680 def plc_db_restore(self):
1681 'restore the planetlab5 DB - looks broken, but run -n might help'
1682 dump=self.dbfile("planetab5")
1683 ##stop httpd service
1684 self.run_in_guest('service httpd stop')
1685 # xxx - need another wrapper
1686 self.run_in_guest_piped('echo drop database planetlab5','psql --user=pgsqluser template1')
1687 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab5')
1688 self.run_in_guest('psql -U pgsqluser planetlab5 -f '+dump)
1689 ##starting httpd service
1690 self.run_in_guest('service httpd start')
1692 utils.header('Database restored from ' + dump)
1694 def standby_1_through_20(self):
1695 """convenience function to wait for a specified number of minutes"""
1698 def standby_1(): pass
1700 def standby_2(): pass
1702 def standby_3(): pass
1704 def standby_4(): pass
1706 def standby_5(): pass
1708 def standby_6(): pass
1710 def standby_7(): pass
1712 def standby_8(): pass
1714 def standby_9(): pass
1716 def standby_10(): pass
1718 def standby_11(): pass
1720 def standby_12(): pass
1722 def standby_13(): pass
1724 def standby_14(): pass
1726 def standby_15(): pass
1728 def standby_16(): pass
1730 def standby_17(): pass
1732 def standby_18(): pass
1734 def standby_19(): pass
1736 def standby_20(): pass