7 from types import StringTypes
11 from TestSite import TestSite
12 from TestNode import TestNode
13 from TestUser import TestUser
14 from TestKey import TestKey
15 from TestSlice import TestSlice
16 from TestSliver import TestSliver
17 from TestBox import TestBox
18 from TestSsh import TestSsh
19 from TestApiserver import TestApiserver
20 from TestSliceSfa import TestSliceSfa
21 from TestUserSfa import TestUserSfa
23 # step methods must take (self) and return a boolean (options is a member of the class)
25 def standby(minutes,dry_run):
26 utils.header('Entering StandBy for %d mn'%minutes)
30 time.sleep(60*minutes)
33 def standby_generic (func):
35 minutes=int(func.__name__.split("_")[1])
36 return standby(minutes,self.options.dry_run)
39 def node_mapper (method):
42 node_method = TestNode.__dict__[method.__name__]
43 for site_spec in self.plc_spec['sites']:
44 test_site = TestSite (self,site_spec)
45 for node_spec in site_spec['nodes']:
46 test_node = TestNode (self,test_site,node_spec)
47 if not node_method(test_node): overall=False
49 # restore the doc text
50 actual.__doc__=method.__doc__
53 def slice_mapper_options (method):
56 slice_method = TestSlice.__dict__[method.__name__]
57 for slice_spec in self.plc_spec['slices']:
58 site_spec = self.locate_site (slice_spec['sitename'])
59 test_site = TestSite(self,site_spec)
60 test_slice=TestSlice(self,test_site,slice_spec)
61 if not slice_method(test_slice,self.options): overall=False
63 # restore the doc text
64 actual.__doc__=method.__doc__
67 def slice_mapper_options_sfa (method):
71 slice_method = TestSliceSfa.__dict__[method.__name__]
72 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
73 site_spec = self.locate_site (slice_spec['sitename'])
74 test_site = TestSite(self,site_spec)
75 test_slice=TestSliceSfa(test_plc,test_site,slice_spec)
76 if not slice_method(test_slice,self.options): overall=False
78 # restore the doc text
79 actual.__doc__=method.__doc__
87 'display', 'resources_pre', SEP,
88 'delete_vs','create_vs','install', 'configure', 'start', SEP,
89 'fetch_keys', 'store_keys', 'clear_known_hosts', SEP,
90 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', SEP,
91 'reinstall_node', 'init_node','bootcd', 'configure_qemu', 'export_qemu',
92 'kill_all_qemus', 'start_node', SEP,
93 # better use of time: do this now that the nodes are taking off
94 'plcsh_stress_test', SEP,
95 'install_sfa', 'configure_sfa', 'import_sfa', 'start_sfa', SEP,
96 'setup_sfa', 'add_sfa', 'update_sfa', 'view_sfa', SEP,
97 'nodes_ssh_debug', 'nodes_ssh_boot', 'check_slice', 'check_initscripts', SEP,
98 # optionally run sfa later; takes longer, but checks more about nm
99 # 'install_sfa', 'configure_sfa', 'import_sfa', 'start_sfa', SEP,
100 # 'setup_sfa', 'add_sfa', 'update_sfa', 'view_sfa', SEP,
101 'check_slice_sfa', 'delete_sfa', 'stop_sfa', SEP,
102 'check_tcp', 'check_hooks', SEP,
103 'force_gather_logs', 'force_resources_post',
106 'show_boxes', 'resources_list','resources_release','resources_release_plc','resources_release_qemu',SEP,
107 'stop', 'vs_start', SEP,
108 'clean_initscripts', 'clean_nodegroups','clean_all_sites', SEP,
109 'clean_sites', 'clean_nodes', 'clean_slices', 'clean_keys', SEP,
111 'list_all_qemus', 'list_qemus', 'kill_qemus', SEP,
112 'db_dump' , 'db_restore', SEP,
113 'standby_1 through 20',
117 def printable_steps (list):
118 return " ".join(list).replace(" "+SEP+" "," \\\n")
120 def valid_step (step):
123 # turn off the sfa-related steps when build has skipped SFA
124 # this is originally for centos5 as recent SFAs won't build on this platformb
126 def check_build_has_sfa (rpms_url):
127 retcod=os.system ("curl --silent %s/ | grep -q sfa"%rpms_url)
128 # full builds are expected to return with 0 here
130 TestPlc.default_steps = [ step for step in TestPlc.default_steps
131 if step.find('sfa') < 0 ]
133 def __init__ (self,plc_spec,options):
134 self.plc_spec=plc_spec
136 self.test_ssh=TestSsh(self.plc_spec['hostname'],self.options.buildname)
138 self.vserverip=plc_spec['vserverip']
139 self.vservername=plc_spec['vservername']
140 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
143 raise Exception,'chroot-based myplc testing is deprecated'
144 self.apiserver=TestApiserver(self.url,options.dry_run)
147 name=self.plc_spec['name']
148 return "%s.%s"%(name,self.vservername)
151 return self.plc_spec['hostname']
154 return self.test_ssh.is_local()
156 # define the API methods on this object through xmlrpc
157 # would help, but not strictly necessary
161 def actual_command_in_guest (self,command):
162 return self.test_ssh.actual_command(self.host_to_guest(command))
164 def start_guest (self):
165 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host()))
167 def run_in_guest (self,command):
168 return utils.system(self.actual_command_in_guest(command))
170 def run_in_host (self,command):
171 return self.test_ssh.run_in_buildname(command)
173 #command gets run in the vserver
174 def host_to_guest(self,command):
175 return "vserver %s exec %s"%(self.vservername,command)
177 #command gets run in the vserver
178 def start_guest_in_host(self):
179 return "vserver %s start"%(self.vservername)
182 def run_in_guest_piped (self,local,remote):
183 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
185 def auth_root (self):
186 return {'Username':self.plc_spec['PLC_ROOT_USER'],
187 'AuthMethod':'password',
188 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
189 'Role' : self.plc_spec['role']
191 def locate_site (self,sitename):
192 for site in self.plc_spec['sites']:
193 if site['site_fields']['name'] == sitename:
195 if site['site_fields']['login_base'] == sitename:
197 raise Exception,"Cannot locate site %s"%sitename
199 def locate_node (self,nodename):
200 for site in self.plc_spec['sites']:
201 for node in site['nodes']:
202 if node['name'] == nodename:
204 raise Exception,"Cannot locate node %s"%nodename
206 def locate_hostname (self,hostname):
207 for site in self.plc_spec['sites']:
208 for node in site['nodes']:
209 if node['node_fields']['hostname'] == hostname:
211 raise Exception,"Cannot locate hostname %s"%hostname
213 def locate_key (self,keyname):
214 for key in self.plc_spec['keys']:
215 if key['name'] == keyname:
217 raise Exception,"Cannot locate key %s"%keyname
219 def locate_slice (self, slicename):
220 for slice in self.plc_spec['slices']:
221 if slice['slice_fields']['name'] == slicename:
223 raise Exception,"Cannot locate slice %s"%slicename
225 def all_sliver_objs (self):
227 for slice_spec in self.plc_spec['slices']:
228 slicename = slice_spec['slice_fields']['name']
229 for nodename in slice_spec['nodenames']:
230 result.append(self.locate_sliver_obj (nodename,slicename))
233 def locate_sliver_obj (self,nodename,slicename):
234 (site,node) = self.locate_node(nodename)
235 slice = self.locate_slice (slicename)
237 test_site = TestSite (self, site)
238 test_node = TestNode (self, test_site,node)
239 # xxx the slice site is assumed to be the node site - mhh - probably harmless
240 test_slice = TestSlice (self, test_site, slice)
241 return TestSliver (self, test_node, test_slice)
243 def locate_first_node(self):
244 nodename=self.plc_spec['slices'][0]['nodenames'][0]
245 (site,node) = self.locate_node(nodename)
246 test_site = TestSite (self, site)
247 test_node = TestNode (self, test_site,node)
250 def locate_first_sliver (self):
251 slice_spec=self.plc_spec['slices'][0]
252 slicename=slice_spec['slice_fields']['name']
253 nodename=slice_spec['nodenames'][0]
254 return self.locate_sliver_obj(nodename,slicename)
256 # all different hostboxes used in this plc
257 def gather_hostBoxes(self):
258 # maps on sites and nodes, return [ (host_box,test_node) ]
260 for site_spec in self.plc_spec['sites']:
261 test_site = TestSite (self,site_spec)
262 for node_spec in site_spec['nodes']:
263 test_node = TestNode (self, test_site, node_spec)
264 if not test_node.is_real():
265 tuples.append( (test_node.host_box(),test_node) )
266 # transform into a dict { 'host_box' -> [ test_node .. ] }
268 for (box,node) in tuples:
269 if not result.has_key(box):
272 result[box].append(node)
275 # a step for checking this stuff
276 def show_boxes (self):
277 'print summary of nodes location'
278 for (box,nodes) in self.gather_hostBoxes().iteritems():
279 print box,":"," + ".join( [ node.name() for node in nodes ] )
282 # make this a valid step
283 def kill_all_qemus(self):
284 'kill all qemu instances on the qemu boxes involved by this setup'
285 # this is the brute force version, kill all qemus on that host box
286 for (box,nodes) in self.gather_hostBoxes().iteritems():
287 # pass the first nodename, as we don't push template-qemu on testboxes
288 nodedir=nodes[0].nodedir()
289 TestBox(box,self.options.buildname).kill_all_qemus(nodedir)
292 # make this a valid step
293 def list_all_qemus(self):
294 'list all qemu instances on the qemu boxes involved by this setup'
295 for (box,nodes) in self.gather_hostBoxes().iteritems():
296 # this is the brute force version, kill all qemus on that host box
297 TestBox(box,self.options.buildname).list_all_qemus()
300 # kill only the right qemus
301 def list_qemus(self):
302 'list qemu instances for our nodes'
303 for (box,nodes) in self.gather_hostBoxes().iteritems():
304 # the fine-grain version
309 # kill only the right qemus
310 def kill_qemus(self):
311 'kill the qemu instances for our nodes'
312 for (box,nodes) in self.gather_hostBoxes().iteritems():
313 # the fine-grain version
318 #################### display config
320 "show test configuration after localization"
321 self.display_pass (1)
322 self.display_pass (2)
326 always_display_keys=['PLC_WWW_HOST','nodes','sites',]
327 def display_pass (self,passno):
328 for (key,val) in self.plc_spec.iteritems():
329 if not self.options.verbose and key not in TestPlc.always_display_keys: continue
333 self.display_site_spec(site)
334 for node in site['nodes']:
335 self.display_node_spec(node)
336 elif key=='initscripts':
337 for initscript in val:
338 self.display_initscript_spec (initscript)
341 self.display_slice_spec (slice)
344 self.display_key_spec (key)
346 if key not in ['sites','initscripts','slices','keys', 'sfa']:
347 print '+ ',key,':',val
349 def display_site_spec (self,site):
350 print '+ ======== site',site['site_fields']['name']
351 for (k,v) in site.iteritems():
352 if not self.options.verbose and k not in TestPlc.always_display_keys: continue
355 print '+ ','nodes : ',
357 print node['node_fields']['hostname'],'',
363 print user['name'],'',
365 elif k == 'site_fields':
366 print '+ login_base',':',v['login_base']
367 elif k == 'address_fields':
371 PrettyPrinter(indent=8,depth=2).pprint(v)
373 def display_initscript_spec (self,initscript):
374 print '+ ======== initscript',initscript['initscript_fields']['name']
376 def display_key_spec (self,key):
377 print '+ ======== key',key['name']
379 def display_slice_spec (self,slice):
380 print '+ ======== slice',slice['slice_fields']['name']
381 for (k,v) in slice.iteritems():
394 elif k=='slice_fields':
395 print '+ fields',':',
396 print 'max_nodes=',v['max_nodes'],
401 def display_node_spec (self,node):
402 print "+ node",node['name'],"host_box=",node['host_box'],
403 print "hostname=",node['node_fields']['hostname'],
404 print "ip=",node['interface_fields']['ip']
407 # another entry point for just showing the boxes involved
408 def display_mapping (self):
409 TestPlc.display_mapping_plc(self.plc_spec)
413 def display_mapping_plc (plc_spec):
414 print '+ MyPLC',plc_spec['name']
415 print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['hostname'],plc_spec['vservername'])
416 print '+\tIP = %s/%s'%(plc_spec['PLC_API_HOST'],plc_spec['vserverip'])
417 for site_spec in plc_spec['sites']:
418 for node_spec in site_spec['nodes']:
419 TestPlc.display_mapping_node(node_spec)
422 def display_mapping_node (node_spec):
423 print '+ NODE %s'%(node_spec['name'])
424 print '+\tqemu box %s'%node_spec['host_box']
425 print '+\thostname=%s'%node_spec['node_fields']['hostname']
427 def resources_pre (self):
428 "run site-dependant pre-test script as defined in LocalTestResources"
429 from LocalTestResources import local_resources
430 return local_resources.step_pre(self)
432 def resources_post (self):
433 "run site-dependant post-test script as defined in LocalTestResources"
434 from LocalTestResources import local_resources
435 return local_resources.step_post(self)
437 def resources_list (self):
438 "run site-dependant list script as defined in LocalTestResources"
439 from LocalTestResources import local_resources
440 return local_resources.step_list(self)
442 def resources_release (self):
443 "run site-dependant release script as defined in LocalTestResources"
444 from LocalTestResources import local_resources
445 return local_resources.step_release(self)
447 def resources_release_plc (self):
448 "run site-dependant release script as defined in LocalTestResources"
449 from LocalTestResources import local_resources
450 return local_resources.step_release_plc(self)
452 def resources_release_qemu (self):
453 "run site-dependant release script as defined in LocalTestResources"
454 from LocalTestResources import local_resources
455 return local_resources.step_release_qemu(self)
458 "vserver delete the test myplc"
459 self.run_in_host("vserver --silent %s delete"%self.vservername)
463 # historically the build was being fetched by the tests
464 # now the build pushes itself as a subdir of the tests workdir
465 # so that the tests do not have to worry about extracting the build (svn, git, or whatever)
466 def create_vs (self):
467 "vserver creation (no install done)"
468 # push the local build/ dir to the testplc box
470 # a full path for the local calls
471 build_dir=os.path.dirname(sys.argv[0])
472 # sometimes this is empty - set to "." in such a case
473 if not build_dir: build_dir="."
474 build_dir += "/build"
476 # use a standard name - will be relative to remote buildname
478 # remove for safety; do *not* mkdir first, otherwise we end up with build/build/
479 self.test_ssh.rmdir(build_dir)
480 self.test_ssh.copy(build_dir,recursive=True)
481 # the repo url is taken from arch-rpms-url
482 # with the last step (i386) removed
483 repo_url = self.options.arch_rpms_url
484 for level in [ 'arch' ]:
485 repo_url = os.path.dirname(repo_url)
486 # pass the vbuild-nightly options to vtest-init-vserver
488 test_env_options += " -p %s"%self.options.personality
489 test_env_options += " -d %s"%self.options.pldistro
490 test_env_options += " -f %s"%self.options.fcdistro
491 script="vtest-init-vserver.sh"
492 vserver_name = self.vservername
493 vserver_options="--netdev eth0 --interface %s"%self.vserverip
495 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
496 vserver_options += " --hostname %s"%vserver_hostname
498 print "Cannot reverse lookup %s"%self.vserverip
499 print "This is considered fatal, as this might pollute the test results"
501 create_vserver="%(build_dir)s/%(script)s %(test_env_options)s %(vserver_name)s %(repo_url)s -- %(vserver_options)s"%locals()
502 return self.run_in_host(create_vserver) == 0
506 "yum install myplc, noderepo, and the plain bootstrapfs"
508 # workaround for getting pgsql8.2 on centos5
509 if self.options.fcdistro == "centos5":
510 self.run_in_guest("rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-3.noarch.rpm")
512 if self.options.personality == "linux32":
514 elif self.options.personality == "linux64":
517 raise Exception, "Unsupported personality %r"%self.options.personality
519 nodefamily="%s-%s-%s"%(self.options.pldistro,self.options.fcdistro,arch)
521 # try to install slicerepo - not fatal yet
522 self.run_in_guest("yum -y install slicerepo-%s"%nodefamily)
525 self.run_in_guest("yum -y install myplc")==0 and \
526 self.run_in_guest("yum -y install noderepo-%s"%nodefamily)==0 and \
527 self.run_in_guest("yum -y install bootstrapfs-%s-plain"%nodefamily)==0
532 tmpname='%s.plc-config-tty'%(self.name())
533 fileconf=open(tmpname,'w')
534 for var in [ 'PLC_NAME',
538 'PLC_MAIL_SUPPORT_ADDRESS',
541 # Above line was added for integrating SFA Testing
547 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
548 fileconf.write('w\n')
549 fileconf.write('q\n')
551 utils.system('cat %s'%tmpname)
552 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
553 utils.system('rm %s'%tmpname)
558 self.run_in_guest('service plc start')
563 self.run_in_guest('service plc stop')
567 "start the PLC vserver"
571 # stores the keys from the config for further use
572 def store_keys(self):
573 "stores test users ssh keys in keys/"
574 for key_spec in self.plc_spec['keys']:
575 TestKey(self,key_spec).store_key()
578 def clean_keys(self):
579 "removes keys cached in keys/"
580 utils.system("rm -rf %s/keys/"%os.path(sys.argv[0]))
582 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
583 # for later direct access to the nodes
584 def fetch_keys(self):
585 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
587 if not os.path.isdir(dir):
589 vservername=self.vservername
591 prefix = 'debug_ssh_key'
592 for ext in [ 'pub', 'rsa' ] :
593 src="/vservers/%(vservername)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
594 dst="keys/%(vservername)s-debug.%(ext)s"%locals()
595 if self.test_ssh.fetch(src,dst) != 0: overall=False
599 "create sites with PLCAPI"
600 return self.do_sites()
602 def clean_sites (self):
603 "delete sites with PLCAPI"
604 return self.do_sites(action="delete")
606 def do_sites (self,action="add"):
607 for site_spec in self.plc_spec['sites']:
608 test_site = TestSite (self,site_spec)
609 if (action != "add"):
610 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
611 test_site.delete_site()
612 # deleted with the site
613 #test_site.delete_users()
616 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
617 test_site.create_site()
618 test_site.create_users()
621 def clean_all_sites (self):
622 "Delete all sites in PLC, and related objects"
623 print 'auth_root',self.auth_root()
624 site_ids = [s['site_id'] for s in self.apiserver.GetSites(self.auth_root(), {}, ['site_id'])]
625 for site_id in site_ids:
626 print 'Deleting site_id',site_id
627 self.apiserver.DeleteSite(self.auth_root(),site_id)
630 "create nodes with PLCAPI"
631 return self.do_nodes()
632 def clean_nodes (self):
633 "delete nodes with PLCAPI"
634 return self.do_nodes(action="delete")
636 def do_nodes (self,action="add"):
637 for site_spec in self.plc_spec['sites']:
638 test_site = TestSite (self,site_spec)
640 utils.header("Deleting nodes in site %s"%test_site.name())
641 for node_spec in site_spec['nodes']:
642 test_node=TestNode(self,test_site,node_spec)
643 utils.header("Deleting %s"%test_node.name())
644 test_node.delete_node()
646 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
647 for node_spec in site_spec['nodes']:
648 utils.pprint('Creating node %s'%node_spec,node_spec)
649 test_node = TestNode (self,test_site,node_spec)
650 test_node.create_node ()
653 def nodegroups (self):
654 "create nodegroups with PLCAPI"
655 return self.do_nodegroups("add")
656 def clean_nodegroups (self):
657 "delete nodegroups with PLCAPI"
658 return self.do_nodegroups("delete")
660 # create nodegroups if needed, and populate
661 def do_nodegroups (self, action="add"):
662 # 1st pass to scan contents
664 for site_spec in self.plc_spec['sites']:
665 test_site = TestSite (self,site_spec)
666 for node_spec in site_spec['nodes']:
667 test_node=TestNode (self,test_site,node_spec)
668 if node_spec.has_key('nodegroups'):
669 nodegroupnames=node_spec['nodegroups']
670 if isinstance(nodegroupnames,StringTypes):
671 nodegroupnames = [ nodegroupnames ]
672 for nodegroupname in nodegroupnames:
673 if not groups_dict.has_key(nodegroupname):
674 groups_dict[nodegroupname]=[]
675 groups_dict[nodegroupname].append(test_node.name())
676 auth=self.auth_root()
678 for (nodegroupname,group_nodes) in groups_dict.iteritems():
680 print 'nodegroups:','dealing with nodegroup',nodegroupname,'on nodes',group_nodes
681 # first, check if the nodetagtype is here
682 tag_types = self.apiserver.GetTagTypes(auth,{'tagname':nodegroupname})
684 tag_type_id = tag_types[0]['tag_type_id']
686 tag_type_id = self.apiserver.AddTagType(auth,
687 {'tagname':nodegroupname,
688 'description': 'for nodegroup %s'%nodegroupname,
691 print 'located tag (type)',nodegroupname,'as',tag_type_id
693 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
695 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
696 print 'created nodegroup',nodegroupname,'from tagname',nodegroupname,'and value','yes'
697 # set node tag on all nodes, value='yes'
698 for nodename in group_nodes:
700 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
702 traceback.print_exc()
703 print 'node',nodename,'seems to already have tag',nodegroupname
706 expect_yes = self.apiserver.GetNodeTags(auth,
707 {'hostname':nodename,
708 'tagname':nodegroupname},
709 ['value'])[0]['value']
710 if expect_yes != "yes":
711 print 'Mismatch node tag on node',nodename,'got',expect_yes
714 if not self.options.dry_run:
715 print 'Cannot find tag',nodegroupname,'on node',nodename
719 print 'cleaning nodegroup',nodegroupname
720 self.apiserver.DeleteNodeGroup(auth,nodegroupname)
722 traceback.print_exc()
726 # return a list of tuples (nodename,qemuname)
727 def all_node_infos (self) :
729 for site_spec in self.plc_spec['sites']:
730 node_infos += [ (node_spec['node_fields']['hostname'],node_spec['host_box']) \
731 for node_spec in site_spec['nodes'] ]
734 def all_nodenames (self): return [ x[0] for x in self.all_node_infos() ]
736 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
737 def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period=15):
738 if self.options.dry_run:
742 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
743 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
744 # the nodes that haven't checked yet - start with a full list and shrink over time
745 tocheck = self.all_hostnames()
746 utils.header("checking nodes %r"%tocheck)
747 # create a dict hostname -> status
748 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
751 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
753 for array in tocheck_status:
754 hostname=array['hostname']
755 boot_state=array['boot_state']
756 if boot_state == target_boot_state:
757 utils.header ("%s has reached the %s state"%(hostname,target_boot_state))
759 # if it's a real node, never mind
760 (site_spec,node_spec)=self.locate_hostname(hostname)
761 if TestNode.is_real_model(node_spec['node_fields']['model']):
762 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
764 boot_state = target_boot_state
765 elif datetime.datetime.now() > graceout:
766 utils.header ("%s still in '%s' state"%(hostname,boot_state))
767 graceout=datetime.datetime.now()+datetime.timedelta(1)
768 status[hostname] = boot_state
770 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != target_boot_state ]
773 if datetime.datetime.now() > timeout:
774 for hostname in tocheck:
775 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
777 # otherwise, sleep for a while
779 # only useful in empty plcs
782 def nodes_booted(self):
783 return self.nodes_check_boot_state('boot',timeout_minutes=30,silent_minutes=20)
785 def check_nodes_ssh(self,debug,timeout_minutes,silent_minutes,period=15):
787 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
788 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
789 vservername=self.vservername
792 local_key = "keys/%(vservername)s-debug.rsa"%locals()
795 local_key = "keys/key1.rsa"
796 node_infos = self.all_node_infos()
797 utils.header("checking ssh access (expected in %s mode) to nodes:"%message)
798 for (nodename,qemuname) in node_infos:
799 utils.header("hostname=%s -- qemubox=%s"%(nodename,qemuname))
800 utils.header("max timeout is %d minutes, silent for %d minutes (period is %s)"%\
801 (timeout_minutes,silent_minutes,period))
803 for node_info in node_infos:
804 (hostname,qemuname) = node_info
805 # try to run 'hostname' in the node
806 command = TestSsh (hostname,key=local_key).actual_command("hostname;uname -a")
807 # don't spam logs - show the command only after the grace period
808 success = utils.system ( command, silent=datetime.datetime.now() < graceout)
810 utils.header('Successfully entered root@%s (%s)'%(hostname,message))
812 node_infos.remove(node_info)
814 # we will have tried real nodes once, in case they're up - but if not, just skip
815 (site_spec,node_spec)=self.locate_hostname(hostname)
816 if TestNode.is_real_model(node_spec['node_fields']['model']):
817 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
818 node_infos.remove(node_info)
821 if datetime.datetime.now() > timeout:
822 for (hostname,qemuname) in node_infos:
823 utils.header("FAILURE to ssh into %s (on %s)"%(hostname,qemuname))
825 # otherwise, sleep for a while
827 # only useful in empty plcs
830 def nodes_ssh_debug(self):
831 "Tries to ssh into nodes in debug mode with the debug ssh key"
832 return self.check_nodes_ssh(debug=True,timeout_minutes=30,silent_minutes=5)
834 def nodes_ssh_boot(self):
835 "Tries to ssh into nodes in production mode with the root ssh key"
836 return self.check_nodes_ssh(debug=False,timeout_minutes=30,silent_minutes=15)
839 def init_node (self):
840 "all nodes : init a clean local directory for holding node-dep stuff like iso image..."
844 "all nodes: invoke GetBootMedium and store result locally"
847 def configure_qemu (self):
848 "all nodes: compute qemu config qemu.conf and store it locally"
851 def reinstall_node (self):
852 "all nodes: mark PLCAPI boot_state as reinstall"
855 def export_qemu (self):
856 "all nodes: push local node-dep directory on the qemu box"
859 ### check hooks : invoke scripts from hooks/{node,slice}
860 def check_hooks_node (self):
861 return self.locate_first_node().check_hooks()
862 def check_hooks_sliver (self) :
863 return self.locate_first_sliver().check_hooks()
865 def check_hooks (self):
866 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
867 return self.check_hooks_node() and self.check_hooks_sliver()
870 def do_check_initscripts(self):
872 for slice_spec in self.plc_spec['slices']:
873 if not slice_spec.has_key('initscriptname'):
875 initscript=slice_spec['initscriptname']
876 for nodename in slice_spec['nodenames']:
877 (site,node) = self.locate_node (nodename)
878 # xxx - passing the wrong site - probably harmless
879 test_site = TestSite (self,site)
880 test_slice = TestSlice (self,test_site,slice_spec)
881 test_node = TestNode (self,test_site,node)
882 test_sliver = TestSliver (self, test_node, test_slice)
883 if not test_sliver.check_initscript(initscript):
887 def check_initscripts(self):
888 "check that the initscripts have triggered"
889 return self.do_check_initscripts()
891 def initscripts (self):
892 "create initscripts with PLCAPI"
893 for initscript in self.plc_spec['initscripts']:
894 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
895 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
898 def clean_initscripts (self):
899 "delete initscripts with PLCAPI"
900 for initscript in self.plc_spec['initscripts']:
901 initscript_name = initscript['initscript_fields']['name']
902 print('Attempting to delete %s in plc %s'%(initscript_name,self.plc_spec['name']))
904 self.apiserver.DeleteInitScript(self.auth_root(),initscript_name)
905 print initscript_name,'deleted'
907 print 'deletion went wrong - probably did not exist'
912 "create slices with PLCAPI"
913 return self.do_slices()
915 def clean_slices (self):
916 "delete slices with PLCAPI"
917 return self.do_slices("delete")
919 def do_slices (self, action="add"):
920 for slice in self.plc_spec['slices']:
921 site_spec = self.locate_site (slice['sitename'])
922 test_site = TestSite(self,site_spec)
923 test_slice=TestSlice(self,test_site,slice)
925 utils.header("Deleting slices in site %s"%test_site.name())
926 test_slice.delete_slice()
928 utils.pprint("Creating slice",slice)
929 test_slice.create_slice()
930 utils.header('Created Slice %s'%slice['slice_fields']['name'])
933 @slice_mapper_options
934 def check_slice(self):
935 "tries to ssh-enter the slice with the user key, to ensure slice creation"
939 def clear_known_hosts (self):
940 "remove test nodes entries from the local known_hosts file"
944 def start_node (self) :
945 "all nodes: start the qemu instance (also runs qemu-bridge-init start)"
948 def check_tcp (self):
949 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
950 specs = self.plc_spec['tcp_test']
955 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
956 if not s_test_sliver.run_tcp_server(port,timeout=10):
960 # idem for the client side
961 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
962 if not c_test_sliver.run_tcp_client(s_test_sliver.test_node.name(),port):
966 def plcsh_stress_test (self):
967 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
968 # install the stress-test in the plc image
969 location = "/usr/share/plc_api/plcsh_stress_test.py"
970 remote="/vservers/%s/%s"%(self.vservername,location)
971 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
973 command += " -- --check"
974 if self.options.size == 1:
976 return ( self.run_in_guest(command) == 0)
978 # populate runs the same utility without slightly different options
979 # in particular runs with --preserve (dont cleanup) and without --check
980 # also it gets run twice, once with the --foreign option for creating fake foreign entries
983 def install_sfa(self):
984 "yum install sfa, sfa-plc and sfa-client"
985 if self.options.personality == "linux32":
987 elif self.options.personality == "linux64":
990 raise Exception, "Unsupported personality %r"%self.options.personality
991 return self.run_in_guest("yum -y install sfa sfa-client sfa-plc sfa-sfatables")==0
994 def configure_sfa(self):
996 tmpname='%s.sfa-config-tty'%(self.name())
997 fileconf=open(tmpname,'w')
998 for var in [ 'SFA_REGISTRY_ROOT_AUTH',
999 'SFA_REGISTRY_LEVEL1_AUTH',
1000 'SFA_REGISTRY_HOST',
1001 'SFA_AGGREGATE_HOST',
1007 'SFA_PLC_DB_PASSWORD',
1009 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec['sfa'][var]))
1010 fileconf.write('w\n')
1011 fileconf.write('R\n')
1012 fileconf.write('q\n')
1014 utils.system('cat %s'%tmpname)
1015 self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
1016 utils.system('rm %s'%tmpname)
1019 def import_sfa(self):
1021 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1022 return self.run_in_guest('sfa-import-plc.py')==0
1023 # not needed anymore
1024 # self.run_in_guest('cp /etc/sfa/authorities/%s/%s.pkey /etc/sfa/authorities/server.key'%(auth,auth))
1026 def start_sfa(self):
1028 return self.run_in_guest('service sfa start')==0
1030 def setup_sfa(self):
1031 "sfi client configuration"
1033 if os.path.exists(dir_name):
1034 utils.system('rm -rf %s'%dir_name)
1035 utils.system('mkdir %s'%dir_name)
1036 file_name=dir_name + os.sep + 'fake-pi1.pkey'
1037 fileconf=open(file_name,'w')
1038 fileconf.write (self.plc_spec['keys'][0]['private'])
1041 file_name=dir_name + os.sep + 'sfi_config'
1042 fileconf=open(file_name,'w')
1043 SFI_AUTH=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']+".main"
1044 fileconf.write ("SFI_AUTH='%s'"%SFI_AUTH)
1045 fileconf.write('\n')
1046 SFI_USER=SFI_AUTH+'.fake-pi1'
1047 fileconf.write ("SFI_USER='%s'"%SFI_USER)
1048 fileconf.write('\n')
1049 SFI_REGISTRY='http://' + self.plc_spec['sfa']['SFA_PLC_DB_HOST'] + ':12345/'
1050 fileconf.write ("SFI_REGISTRY='%s'"%SFI_REGISTRY)
1051 fileconf.write('\n')
1052 SFI_SM='http://' + self.plc_spec['sfa']['SFA_PLC_DB_HOST'] + ':12347/'
1053 fileconf.write ("SFI_SM='%s'"%SFI_SM)
1054 fileconf.write('\n')
1057 file_name=dir_name + os.sep + 'person.xml'
1058 fileconf=open(file_name,'w')
1059 for record in self.plc_spec['sfa']['sfa_person_xml']:
1060 person_record=record
1061 fileconf.write(person_record)
1062 fileconf.write('\n')
1065 file_name=dir_name + os.sep + 'slice.xml'
1066 fileconf=open(file_name,'w')
1067 for record in self.plc_spec['sfa']['sfa_slice_xml']:
1069 #slice_record=self.plc_spec['sfa']['sfa_slice_xml']
1070 fileconf.write(slice_record)
1071 fileconf.write('\n')
1074 file_name=dir_name + os.sep + 'slice.rspec'
1075 fileconf=open(file_name,'w')
1077 for (key, value) in self.plc_spec['sfa']['sfa_slice_rspec'].items():
1079 fileconf.write(slice_rspec)
1080 fileconf.write('\n')
1083 remote="/vservers/%s/%s"%(self.vservername,location)
1084 self.test_ssh.copy_abs(dir_name, remote, recursive=True)
1086 #utils.system('cat %s'%tmpname)
1087 utils.system('rm -rf %s'%dir_name)
1091 "run sfi.py add (on Registry) and sfi.py create (on SM) to form new objects"
1093 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1094 success=test_user_sfa.add_user()
1096 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1097 site_spec = self.locate_site (slice_spec['sitename'])
1098 test_site = TestSite(self,site_spec)
1099 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1100 success1=test_slice_sfa.add_slice()
1101 success2=test_slice_sfa.create_slice()
1102 return success and success1 and success2
1104 def update_sfa(self):
1105 "run sfi.py update (on Registry) and sfi.py create (on SM) on existing objects"
1107 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1108 success1=test_user_sfa.update_user()
1110 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1111 site_spec = self.locate_site (slice_spec['sitename'])
1112 test_site = TestSite(self,site_spec)
1113 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1114 success2=test_slice_sfa.update_slice()
1115 return success1 and success2
1118 "run sfi.py list and sfi.py show (both on Registry) and sfi.py slices and sfi.py resources (both on SM)"
1119 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1121 self.run_in_guest("sfi.py -d /root/.sfi/ list %s.main"%auth)==0 and \
1122 self.run_in_guest("sfi.py -d /root/.sfi/ show %s.main"%auth)==0 and \
1123 self.run_in_guest("sfi.py -d /root/.sfi/ slices")==0 and \
1124 self.run_in_guest("sfi.py -d /root/.sfi/ resources -o resources")==0
1126 @slice_mapper_options_sfa
1127 def check_slice_sfa(self):
1128 "tries to ssh-enter the SFA slice"
1131 def delete_sfa(self):
1132 "run sfi.py delete (on SM), sfi.py remove (on Registry)"
1134 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1135 success1=test_user_sfa.delete_user()
1136 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1137 site_spec = self.locate_site (slice_spec['sitename'])
1138 test_site = TestSite(self,site_spec)
1139 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1140 success2=test_slice_sfa.delete_slice()
1142 return success1 and success2
1146 return self.run_in_guest('service sfa stop')==0
1148 def populate (self):
1149 "creates random entries in the PLCAPI"
1150 # install the stress-test in the plc image
1151 location = "/usr/share/plc_api/plcsh_stress_test.py"
1152 remote="/vservers/%s/%s"%(self.vservername,location)
1153 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1155 command += " -- --preserve --short-names"
1156 local = (self.run_in_guest(command) == 0);
1157 # second run with --foreign
1158 command += ' --foreign'
1159 remote = (self.run_in_guest(command) == 0);
1160 return ( local and remote)
1162 def gather_logs (self):
1163 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1164 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1165 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1166 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1167 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1168 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1170 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1171 self.gather_var_logs ()
1173 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1174 self.gather_pgsql_logs ()
1176 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1177 for site_spec in self.plc_spec['sites']:
1178 test_site = TestSite (self,site_spec)
1179 for node_spec in site_spec['nodes']:
1180 test_node=TestNode(self,test_site,node_spec)
1181 test_node.gather_qemu_logs()
1183 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1184 self.gather_nodes_var_logs()
1186 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1187 self.gather_slivers_var_logs()
1190 def gather_slivers_var_logs(self):
1191 for test_sliver in self.all_sliver_objs():
1192 remote = test_sliver.tar_var_logs()
1193 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
1194 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
1195 utils.system(command)
1198 def gather_var_logs (self):
1199 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
1200 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1201 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
1202 utils.system(command)
1203 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd"%self.name()
1204 utils.system(command)
1206 def gather_pgsql_logs (self):
1207 utils.system("mkdir -p logs/myplc.pgsql-log.%s"%self.name())
1208 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1209 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -"%self.name()
1210 utils.system(command)
1212 def gather_nodes_var_logs (self):
1213 for site_spec in self.plc_spec['sites']:
1214 test_site = TestSite (self,site_spec)
1215 for node_spec in site_spec['nodes']:
1216 test_node=TestNode(self,test_site,node_spec)
1217 test_ssh = TestSsh (test_node.name(),key="keys/key1.rsa")
1218 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1219 command = command + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
1220 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
1221 utils.system(command)
1224 # returns the filename to use for sql dump/restore, using options.dbname if set
1225 def dbfile (self, database):
1226 # uses options.dbname if it is found
1228 name=self.options.dbname
1229 if not isinstance(name,StringTypes):
1232 t=datetime.datetime.now()
1235 return "/root/%s-%s.sql"%(database,name)
1238 'dump the planetlab5 DB in /root in the PLC - filename has time'
1239 dump=self.dbfile("planetab5")
1240 self.run_in_guest('pg_dump -U pgsqluser planetlab5 -f '+ dump)
1241 utils.header('Dumped planetlab5 database in %s'%dump)
1244 def db_restore(self):
1245 'restore the planetlab5 DB - looks broken, but run -n might help'
1246 dump=self.dbfile("planetab5")
1247 ##stop httpd service
1248 self.run_in_guest('service httpd stop')
1249 # xxx - need another wrapper
1250 self.run_in_guest_piped('echo drop database planetlab5','psql --user=pgsqluser template1')
1251 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab5')
1252 self.run_in_guest('psql -U pgsqluser planetlab5 -f '+dump)
1253 ##starting httpd service
1254 self.run_in_guest('service httpd start')
1256 utils.header('Database restored from ' + dump)
1259 def standby_1(): pass
1261 def standby_2(): pass
1263 def standby_3(): pass
1265 def standby_4(): pass
1267 def standby_5(): pass
1269 def standby_6(): pass
1271 def standby_7(): pass
1273 def standby_8(): pass
1275 def standby_9(): pass
1277 def standby_10(): pass
1279 def standby_11(): pass
1281 def standby_12(): pass
1283 def standby_13(): pass
1285 def standby_14(): pass
1287 def standby_15(): pass
1289 def standby_16(): pass
1291 def standby_17(): pass
1293 def standby_18(): pass
1295 def standby_19(): pass
1297 def standby_20(): pass