7 from types import StringTypes
11 from TestSite import TestSite
12 from TestNode import TestNode
13 from TestUser import TestUser
14 from TestKey import TestKey
15 from TestSlice import TestSlice
16 from TestSliver import TestSliver
17 from TestBox import TestBox
18 from TestSsh import TestSsh
19 from TestApiserver import TestApiserver
20 from TestSliceSfa import TestSliceSfa
21 from TestUserSfa import TestUserSfa
23 # step methods must take (self) and return a boolean (options is a member of the class)
25 def standby(minutes,dry_run):
26 utils.header('Entering StandBy for %d mn'%minutes)
30 time.sleep(60*minutes)
33 def standby_generic (func):
35 minutes=int(func.__name__.split("_")[1])
36 return standby(minutes,self.options.dry_run)
39 def node_mapper (method):
42 node_method = TestNode.__dict__[method.__name__]
43 for site_spec in self.plc_spec['sites']:
44 test_site = TestSite (self,site_spec)
45 for node_spec in site_spec['nodes']:
46 test_node = TestNode (self,test_site,node_spec)
47 if not node_method(test_node): overall=False
49 # restore the doc text
50 actual.__doc__=method.__doc__
53 def slice_mapper_options (method):
56 slice_method = TestSlice.__dict__[method.__name__]
57 for slice_spec in self.plc_spec['slices']:
58 site_spec = self.locate_site (slice_spec['sitename'])
59 test_site = TestSite(self,site_spec)
60 test_slice=TestSlice(self,test_site,slice_spec)
61 if not slice_method(test_slice,self.options): overall=False
63 # restore the doc text
64 actual.__doc__=method.__doc__
67 def slice_mapper_options_sfa (method):
71 slice_method = TestSliceSfa.__dict__[method.__name__]
72 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
73 site_spec = self.locate_site (slice_spec['sitename'])
74 test_site = TestSite(self,site_spec)
75 test_slice=TestSliceSfa(test_plc,test_site,slice_spec)
76 if not slice_method(test_slice,self.options): overall=False
78 # restore the doc text
79 actual.__doc__=method.__doc__
87 'display', 'local_pre', SEP,
88 'delete','create','install', 'configure', 'start', SEP,
89 'fetch_keys', 'store_keys', 'clear_known_hosts', SEP,
90 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', SEP,
91 'reinstall_node', 'init_node','bootcd', 'configure_qemu', 'export_qemu',
92 'kill_all_qemus', 'start_node', SEP,
93 # better use of time: do this now that the nodes are taking off
94 'plcsh_stress_test', SEP,
95 'nodes_ssh_debug', 'nodes_ssh_boot', 'check_slice', 'check_initscripts', SEP,
96 'install_sfa', 'configure_sfa', 'import_sfa', 'start_sfa', SEP,
97 'setup_sfa', 'add_sfa', 'update_sfa', SEP,
98 'view_sfa', 'check_slice_sfa', 'delete_sfa', 'stop_sfa', SEP,
99 'check_tcp', 'check_hooks', SEP,
100 'force_gather_logs', 'force_local_post',
103 'fresh_install', 'stop', 'vs_start', SEP,
104 'clean_initscripts', 'clean_nodegroups','clean_all_sites', SEP,
105 'clean_sites', 'clean_nodes', 'clean_slices', 'clean_keys', SEP,
107 'show_boxes', 'list_all_qemus', 'list_qemus', 'kill_qemus', SEP,
108 'db_dump' , 'db_restore', SEP,
109 'local_list','local_cleanup',SEP,
110 'standby_1 through 20',
114 def printable_steps (list):
115 return " ".join(list).replace(" "+SEP+" "," \\\n")
117 def valid_step (step):
120 def __init__ (self,plc_spec,options):
121 self.plc_spec=plc_spec
123 self.test_ssh=TestSsh(self.plc_spec['hostname'],self.options.buildname)
125 self.vserverip=plc_spec['vserverip']
126 self.vservername=plc_spec['vservername']
127 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
130 raise Exception,'chroot-based myplc testing is deprecated'
131 self.apiserver=TestApiserver(self.url,options.dry_run)
134 name=self.plc_spec['name']
135 return "%s.%s"%(name,self.vservername)
138 return self.plc_spec['hostname']
141 return self.test_ssh.is_local()
143 # define the API methods on this object through xmlrpc
144 # would help, but not strictly necessary
148 def actual_command_in_guest (self,command):
149 return self.test_ssh.actual_command(self.host_to_guest(command))
151 def start_guest (self):
152 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host()))
154 def run_in_guest (self,command):
155 return utils.system(self.actual_command_in_guest(command))
157 def run_in_host (self,command):
158 return self.test_ssh.run_in_buildname(command)
160 #command gets run in the vserver
161 def host_to_guest(self,command):
162 return "vserver %s exec %s"%(self.vservername,command)
164 #command gets run in the vserver
165 def start_guest_in_host(self):
166 return "vserver %s start"%(self.vservername)
169 def run_in_guest_piped (self,local,remote):
170 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
172 def auth_root (self):
173 return {'Username':self.plc_spec['PLC_ROOT_USER'],
174 'AuthMethod':'password',
175 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
176 'Role' : self.plc_spec['role']
178 def locate_site (self,sitename):
179 for site in self.plc_spec['sites']:
180 if site['site_fields']['name'] == sitename:
182 if site['site_fields']['login_base'] == sitename:
184 raise Exception,"Cannot locate site %s"%sitename
186 def locate_node (self,nodename):
187 for site in self.plc_spec['sites']:
188 for node in site['nodes']:
189 if node['name'] == nodename:
191 raise Exception,"Cannot locate node %s"%nodename
193 def locate_hostname (self,hostname):
194 for site in self.plc_spec['sites']:
195 for node in site['nodes']:
196 if node['node_fields']['hostname'] == hostname:
198 raise Exception,"Cannot locate hostname %s"%hostname
200 def locate_key (self,keyname):
201 for key in self.plc_spec['keys']:
202 if key['name'] == keyname:
204 raise Exception,"Cannot locate key %s"%keyname
206 def locate_slice (self, slicename):
207 for slice in self.plc_spec['slices']:
208 if slice['slice_fields']['name'] == slicename:
210 raise Exception,"Cannot locate slice %s"%slicename
212 def all_sliver_objs (self):
214 for slice_spec in self.plc_spec['slices']:
215 slicename = slice_spec['slice_fields']['name']
216 for nodename in slice_spec['nodenames']:
217 result.append(self.locate_sliver_obj (nodename,slicename))
220 def locate_sliver_obj (self,nodename,slicename):
221 (site,node) = self.locate_node(nodename)
222 slice = self.locate_slice (slicename)
224 test_site = TestSite (self, site)
225 test_node = TestNode (self, test_site,node)
226 # xxx the slice site is assumed to be the node site - mhh - probably harmless
227 test_slice = TestSlice (self, test_site, slice)
228 return TestSliver (self, test_node, test_slice)
230 def locate_first_node(self):
231 nodename=self.plc_spec['slices'][0]['nodenames'][0]
232 (site,node) = self.locate_node(nodename)
233 test_site = TestSite (self, site)
234 test_node = TestNode (self, test_site,node)
237 def locate_first_sliver (self):
238 slice_spec=self.plc_spec['slices'][0]
239 slicename=slice_spec['slice_fields']['name']
240 nodename=slice_spec['nodenames'][0]
241 return self.locate_sliver_obj(nodename,slicename)
243 # all different hostboxes used in this plc
244 def gather_hostBoxes(self):
245 # maps on sites and nodes, return [ (host_box,test_node) ]
247 for site_spec in self.plc_spec['sites']:
248 test_site = TestSite (self,site_spec)
249 for node_spec in site_spec['nodes']:
250 test_node = TestNode (self, test_site, node_spec)
251 if not test_node.is_real():
252 tuples.append( (test_node.host_box(),test_node) )
253 # transform into a dict { 'host_box' -> [ test_node .. ] }
255 for (box,node) in tuples:
256 if not result.has_key(box):
259 result[box].append(node)
262 # a step for checking this stuff
263 def show_boxes (self):
264 for (box,nodes) in self.gather_hostBoxes().iteritems():
265 print box,":"," + ".join( [ node.name() for node in nodes ] )
268 # make this a valid step
269 def kill_all_qemus(self):
270 "all qemu boxes: kill all running qemus (even of former runs)"
271 # this is the brute force version, kill all qemus on that host box
272 for (box,nodes) in self.gather_hostBoxes().iteritems():
273 # pass the first nodename, as we don't push template-qemu on testboxes
274 nodedir=nodes[0].nodedir()
275 TestBox(box,self.options.buildname).kill_all_qemus(nodedir)
278 # make this a valid step
279 def list_all_qemus(self):
280 for (box,nodes) in self.gather_hostBoxes().iteritems():
281 # this is the brute force version, kill all qemus on that host box
282 TestBox(box,self.options.buildname).list_all_qemus()
285 # kill only the right qemus
286 def list_qemus(self):
287 for (box,nodes) in self.gather_hostBoxes().iteritems():
288 # the fine-grain version
293 # kill only the right qemus
294 def kill_qemus(self):
295 for (box,nodes) in self.gather_hostBoxes().iteritems():
296 # the fine-grain version
301 #################### display config
303 "show test configuration after localization"
304 self.display_pass (1)
305 self.display_pass (2)
309 def display_pass (self,passno):
310 for (key,val) in self.plc_spec.iteritems():
314 self.display_site_spec(site)
315 for node in site['nodes']:
316 self.display_node_spec(node)
317 elif key=='initscripts':
318 for initscript in val:
319 self.display_initscript_spec (initscript)
322 self.display_slice_spec (slice)
325 self.display_key_spec (key)
327 if key not in ['sites','initscripts','slices','keys']:
328 print '+ ',key,':',val
330 def display_site_spec (self,site):
331 print '+ ======== site',site['site_fields']['name']
332 for (k,v) in site.iteritems():
335 print '+ ','nodes : ',
337 print node['node_fields']['hostname'],'',
343 print user['name'],'',
345 elif k == 'site_fields':
346 print '+ login_base',':',v['login_base']
347 elif k == 'address_fields':
351 PrettyPrinter(indent=8,depth=2).pprint(v)
353 def display_initscript_spec (self,initscript):
354 print '+ ======== initscript',initscript['initscript_fields']['name']
356 def display_key_spec (self,key):
357 print '+ ======== key',key['name']
359 def display_slice_spec (self,slice):
360 print '+ ======== slice',slice['slice_fields']['name']
361 for (k,v) in slice.iteritems():
374 elif k=='slice_fields':
375 print '+ fields',':',
376 print 'max_nodes=',v['max_nodes'],
381 def display_node_spec (self,node):
382 print "+ node",node['name'],"host_box=",node['host_box'],
383 print "hostname=",node['node_fields']['hostname'],
384 print "ip=",node['interface_fields']['ip']
387 # another entry point for just showing the boxes involved
388 def display_mapping (self):
389 TestPlc.display_mapping_plc(self.plc_spec)
393 def display_mapping_plc (plc_spec):
394 print '+ MyPLC',plc_spec['name']
395 print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['hostname'],plc_spec['vservername'])
396 print '+\tIP = %s/%s'%(plc_spec['PLC_API_HOST'],plc_spec['vserverip'])
397 for site_spec in plc_spec['sites']:
398 for node_spec in site_spec['nodes']:
399 TestPlc.display_mapping_node(node_spec)
402 def display_mapping_node (node_spec):
403 print '+ NODE %s'%(node_spec['name'])
404 print '+\tqemu box %s'%node_spec['host_box']
405 print '+\thostname=%s'%node_spec['node_fields']['hostname']
407 def local_pre (self):
408 "run site-dependant pre-test script as defined in LocalTestResources"
409 from LocalTestResources import local_resources
410 return local_resources.step_pre(self)
412 def local_post (self):
413 "run site-dependant post-test script as defined in LocalTestResources"
414 from LocalTestResources import local_resources
415 return local_resources.step_post(self)
417 def local_list (self):
418 "run site-dependant list script as defined in LocalTestResources"
419 from LocalTestResources import local_resources
420 return local_resources.step_list(self)
422 def local_cleanup (self):
423 "run site-dependant cleanup script as defined in LocalTestResources"
424 from LocalTestResources import local_resources
425 return local_resources.step_cleanup(self)
428 "vserver delete the test myplc"
429 self.run_in_host("vserver --silent %s delete"%self.vservername)
434 "vserver creation (no install done)"
436 # a full path for the local calls
437 build_dir=os.path.dirname(sys.argv[0])
438 # sometimes this is empty - set to "." in such a case
439 if not build_dir: build_dir="."
440 build_dir += "/build"
442 # use a standard name - will be relative to remote buildname
444 # run checkout in any case - would do an update if already exists
445 build_checkout = "svn checkout %s %s"%(self.options.build_url,build_dir)
446 if self.run_in_host(build_checkout) != 0:
448 # the repo url is taken from arch-rpms-url
449 # with the last step (i386) removed
450 repo_url = self.options.arch_rpms_url
451 for level in [ 'arch' ]:
452 repo_url = os.path.dirname(repo_url)
453 # pass the vbuild-nightly options to vtest-init-vserver
455 test_env_options += " -p %s"%self.options.personality
456 test_env_options += " -d %s"%self.options.pldistro
457 test_env_options += " -f %s"%self.options.fcdistro
458 script="vtest-init-vserver.sh"
459 vserver_name = self.vservername
460 vserver_options="--netdev eth0 --interface %s"%self.vserverip
462 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
463 vserver_options += " --hostname %s"%vserver_hostname
466 create_vserver="%(build_dir)s/%(script)s %(test_env_options)s %(vserver_name)s %(repo_url)s -- %(vserver_options)s"%locals()
467 return self.run_in_host(create_vserver) == 0
471 "yum install myplc, noderepo, and the plain bootstrapfs"
472 if self.options.personality == "linux32":
474 elif self.options.personality == "linux64":
477 raise Exception, "Unsupported personality %r"%self.options.personality
479 nodefamily="%s-%s-%s"%(self.options.pldistro,self.options.fcdistro,arch)
481 self.run_in_guest("yum -y install myplc")==0 and \
482 self.run_in_guest("yum -y install noderepo-%s"%nodefamily)==0 and \
483 self.run_in_guest("yum -y install bootstrapfs-%s-plain"%nodefamily)==0
488 tmpname='%s.plc-config-tty'%(self.name())
489 fileconf=open(tmpname,'w')
490 for var in [ 'PLC_NAME',
494 'PLC_MAIL_SUPPORT_ADDRESS',
497 # Above line was added for integrating SFA Testing
503 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
504 fileconf.write('w\n')
505 fileconf.write('q\n')
507 utils.system('cat %s'%tmpname)
508 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
509 utils.system('rm %s'%tmpname)
514 self.run_in_guest('service plc start')
519 self.run_in_guest('service plc stop')
526 # stores the keys from the config for further use
527 def store_keys(self):
528 "stores test users ssh keys in keys/"
529 for key_spec in self.plc_spec['keys']:
530 TestKey(self,key_spec).store_key()
533 def clean_keys(self):
534 utils.system("rm -rf %s/keys/"%os.path(sys.argv[0]))
536 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
537 # for later direct access to the nodes
538 def fetch_keys(self):
539 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
541 if not os.path.isdir(dir):
543 vservername=self.vservername
545 prefix = 'debug_ssh_key'
546 for ext in [ 'pub', 'rsa' ] :
547 src="/vservers/%(vservername)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
548 dst="keys/%(vservername)s-debug.%(ext)s"%locals()
549 if self.test_ssh.fetch(src,dst) != 0: overall=False
553 "create sites with PLCAPI"
554 return self.do_sites()
556 def clean_sites (self):
557 "delete sites with PLCAPI"
558 return self.do_sites(action="delete")
560 def do_sites (self,action="add"):
561 for site_spec in self.plc_spec['sites']:
562 test_site = TestSite (self,site_spec)
563 if (action != "add"):
564 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
565 test_site.delete_site()
566 # deleted with the site
567 #test_site.delete_users()
570 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
571 test_site.create_site()
572 test_site.create_users()
575 def clean_all_sites (self):
576 print 'auth_root',self.auth_root()
577 site_ids = [s['site_id'] for s in self.apiserver.GetSites(self.auth_root(), {}, ['site_id'])]
578 for site_id in site_ids:
579 print 'Deleting site_id',site_id
580 self.apiserver.DeleteSite(self.auth_root(),site_id)
583 "create nodes with PLCAPI"
584 return self.do_nodes()
585 def clean_nodes (self):
586 "delete nodes with PLCAPI"
587 return self.do_nodes(action="delete")
589 def do_nodes (self,action="add"):
590 for site_spec in self.plc_spec['sites']:
591 test_site = TestSite (self,site_spec)
593 utils.header("Deleting nodes in site %s"%test_site.name())
594 for node_spec in site_spec['nodes']:
595 test_node=TestNode(self,test_site,node_spec)
596 utils.header("Deleting %s"%test_node.name())
597 test_node.delete_node()
599 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
600 for node_spec in site_spec['nodes']:
601 utils.pprint('Creating node %s'%node_spec,node_spec)
602 test_node = TestNode (self,test_site,node_spec)
603 test_node.create_node ()
606 def nodegroups (self):
607 "create nodegroups with PLCAPI"
608 return self.do_nodegroups("add")
609 def clean_nodegroups (self):
610 "delete nodegroups with PLCAPI"
611 return self.do_nodegroups("delete")
613 # create nodegroups if needed, and populate
614 def do_nodegroups (self, action="add"):
615 # 1st pass to scan contents
617 for site_spec in self.plc_spec['sites']:
618 test_site = TestSite (self,site_spec)
619 for node_spec in site_spec['nodes']:
620 test_node=TestNode (self,test_site,node_spec)
621 if node_spec.has_key('nodegroups'):
622 nodegroupnames=node_spec['nodegroups']
623 if isinstance(nodegroupnames,StringTypes):
624 nodegroupnames = [ nodegroupnames ]
625 for nodegroupname in nodegroupnames:
626 if not groups_dict.has_key(nodegroupname):
627 groups_dict[nodegroupname]=[]
628 groups_dict[nodegroupname].append(test_node.name())
629 auth=self.auth_root()
631 for (nodegroupname,group_nodes) in groups_dict.iteritems():
633 print 'nodegroups:','dealing with nodegroup',nodegroupname,'on nodes',group_nodes
634 # first, check if the nodetagtype is here
635 tag_types = self.apiserver.GetTagTypes(auth,{'tagname':nodegroupname})
637 tag_type_id = tag_types[0]['tag_type_id']
639 tag_type_id = self.apiserver.AddTagType(auth,
640 {'tagname':nodegroupname,
641 'description': 'for nodegroup %s'%nodegroupname,
644 print 'located tag (type)',nodegroupname,'as',tag_type_id
646 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
648 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
649 print 'created nodegroup',nodegroupname,'from tagname',nodegroupname,'and value','yes'
650 # set node tag on all nodes, value='yes'
651 for nodename in group_nodes:
653 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
655 traceback.print_exc()
656 print 'node',nodename,'seems to already have tag',nodegroupname
659 expect_yes = self.apiserver.GetNodeTags(auth,
660 {'hostname':nodename,
661 'tagname':nodegroupname},
662 ['value'])[0]['value']
663 if expect_yes != "yes":
664 print 'Mismatch node tag on node',nodename,'got',expect_yes
667 if not self.options.dry_run:
668 print 'Cannot find tag',nodegroupname,'on node',nodename
672 print 'cleaning nodegroup',nodegroupname
673 self.apiserver.DeleteNodeGroup(auth,nodegroupname)
675 traceback.print_exc()
679 def all_hostnames (self) :
681 for site_spec in self.plc_spec['sites']:
682 hostnames += [ node_spec['node_fields']['hostname'] \
683 for node_spec in site_spec['nodes'] ]
686 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
687 def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period=15):
688 if self.options.dry_run:
692 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
693 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
694 # the nodes that haven't checked yet - start with a full list and shrink over time
695 tocheck = self.all_hostnames()
696 utils.header("checking nodes %r"%tocheck)
697 # create a dict hostname -> status
698 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
701 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
703 for array in tocheck_status:
704 hostname=array['hostname']
705 boot_state=array['boot_state']
706 if boot_state == target_boot_state:
707 utils.header ("%s has reached the %s state"%(hostname,target_boot_state))
709 # if it's a real node, never mind
710 (site_spec,node_spec)=self.locate_hostname(hostname)
711 if TestNode.is_real_model(node_spec['node_fields']['model']):
712 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
714 boot_state = target_boot_state
715 elif datetime.datetime.now() > graceout:
716 utils.header ("%s still in '%s' state"%(hostname,boot_state))
717 graceout=datetime.datetime.now()+datetime.timedelta(1)
718 status[hostname] = boot_state
720 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != target_boot_state ]
723 if datetime.datetime.now() > timeout:
724 for hostname in tocheck:
725 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
727 # otherwise, sleep for a while
729 # only useful in empty plcs
732 def nodes_booted(self):
733 return self.nodes_check_boot_state('boot',timeout_minutes=30,silent_minutes=20)
735 def check_nodes_ssh(self,debug,timeout_minutes,silent_minutes,period=15):
737 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
738 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
739 vservername=self.vservername
742 local_key = "keys/%(vservername)s-debug.rsa"%locals()
745 local_key = "keys/key1.rsa"
746 tocheck = self.all_hostnames()
747 utils.header("checking ssh access (expected in %s mode) to nodes %r"%(message,tocheck))
748 utils.header("max timeout is %d minutes, silent for %d minutes (period is %s)"%\
749 (timeout_minutes,silent_minutes,period))
751 for hostname in tocheck:
752 # try to run 'hostname' in the node
753 command = TestSsh (hostname,key=local_key).actual_command("hostname;uname -a")
754 # don't spam logs - show the command only after the grace period
755 success = utils.system ( command, silent=datetime.datetime.now() < graceout)
757 utils.header('Successfully entered root@%s (%s)'%(hostname,message))
759 tocheck.remove(hostname)
761 # we will have tried real nodes once, in case they're up - but if not, just skip
762 (site_spec,node_spec)=self.locate_hostname(hostname)
763 if TestNode.is_real_model(node_spec['node_fields']['model']):
764 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
765 tocheck.remove(hostname)
768 if datetime.datetime.now() > timeout:
769 for hostname in tocheck:
770 utils.header("FAILURE to ssh into %s"%hostname)
772 # otherwise, sleep for a while
774 # only useful in empty plcs
777 def nodes_ssh_debug(self):
778 "Tries to ssh into nodes in debug mode with the debug ssh key"
779 return self.check_nodes_ssh(debug=True,timeout_minutes=30,silent_minutes=10)
781 def nodes_ssh_boot(self):
782 "Tries to ssh into nodes in production mode with the root ssh key"
783 return self.check_nodes_ssh(debug=False,timeout_minutes=30,silent_minutes=10)
786 def init_node (self):
787 "all nodes : init a clean local directory for holding node-dep stuff like iso image..."
791 "all nodes: invoke GetBootMedium and store result locally"
794 def configure_qemu (self):
795 "all nodes: compute qemu config qemu.conf and store it locally"
798 def reinstall_node (self):
799 "all nodes: mark PLCAPI boot_state as reinstall"
802 def export_qemu (self):
803 "all nodes: push local node-dep directory on the qemu box"
806 ### check hooks : invoke scripts from hooks/{node,slice}
807 def check_hooks_node (self):
808 return self.locate_first_node().check_hooks()
809 def check_hooks_sliver (self) :
810 return self.locate_first_sliver().check_hooks()
812 def check_hooks (self):
813 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
814 return self.check_hooks_node() and self.check_hooks_sliver()
817 def do_check_initscripts(self):
819 for slice_spec in self.plc_spec['slices']:
820 if not slice_spec.has_key('initscriptname'):
822 initscript=slice_spec['initscriptname']
823 for nodename in slice_spec['nodenames']:
824 (site,node) = self.locate_node (nodename)
825 # xxx - passing the wrong site - probably harmless
826 test_site = TestSite (self,site)
827 test_slice = TestSlice (self,test_site,slice_spec)
828 test_node = TestNode (self,test_site,node)
829 test_sliver = TestSliver (self, test_node, test_slice)
830 if not test_sliver.check_initscript(initscript):
834 def check_initscripts(self):
835 "check that the initscripts have triggered"
836 return self.do_check_initscripts()
838 def initscripts (self):
839 "create initscripts with PLCAPI"
840 for initscript in self.plc_spec['initscripts']:
841 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
842 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
845 def clean_initscripts (self):
846 "delete initscripts with PLCAPI"
847 for initscript in self.plc_spec['initscripts']:
848 initscript_name = initscript['initscript_fields']['name']
849 print('Attempting to delete %s in plc %s'%(initscript_name,self.plc_spec['name']))
851 self.apiserver.DeleteInitScript(self.auth_root(),initscript_name)
852 print initscript_name,'deleted'
854 print 'deletion went wrong - probably did not exist'
859 "create slices with PLCAPI"
860 return self.do_slices()
862 def clean_slices (self):
863 "delete slices with PLCAPI"
864 return self.do_slices("delete")
866 def do_slices (self, action="add"):
867 for slice in self.plc_spec['slices']:
868 site_spec = self.locate_site (slice['sitename'])
869 test_site = TestSite(self,site_spec)
870 test_slice=TestSlice(self,test_site,slice)
872 utils.header("Deleting slices in site %s"%test_site.name())
873 test_slice.delete_slice()
875 utils.pprint("Creating slice",slice)
876 test_slice.create_slice()
877 utils.header('Created Slice %s'%slice['slice_fields']['name'])
880 @slice_mapper_options
881 def check_slice(self):
882 "tries to ssh-enter the slice with the user key, to ensure slice creation"
886 def clear_known_hosts (self):
887 "remove test nodes entries from the local known_hosts file"
891 def start_node (self) :
892 "all nodes: start the qemu instance (also runs qemu-bridge-init start)"
895 def check_tcp (self):
896 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
897 specs = self.plc_spec['tcp_test']
902 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
903 if not s_test_sliver.run_tcp_server(port,timeout=10):
907 # idem for the client side
908 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
909 if not c_test_sliver.run_tcp_client(s_test_sliver.test_node.name(),port):
913 def plcsh_stress_test (self):
914 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
915 # install the stress-test in the plc image
916 location = "/usr/share/plc_api/plcsh_stress_test.py"
917 remote="/vservers/%s/%s"%(self.vservername,location)
918 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
920 command += " -- --check"
921 if self.options.size == 1:
923 return ( self.run_in_guest(command) == 0)
925 # populate runs the same utility without slightly different options
926 # in particular runs with --preserve (dont cleanup) and without --check
927 # also it gets run twice, once with the --foreign option for creating fake foreign entries
930 def install_sfa(self):
931 "yum install sfa, sfa-plc and sfa-client"
932 if self.options.personality == "linux32":
934 elif self.options.personality == "linux64":
937 raise Exception, "Unsupported personality %r"%self.options.personality
938 return self.run_in_guest("yum -y install sfa sfa-client sfa-plc sfa-sfatables")==0
941 def configure_sfa(self):
943 tmpname='%s.sfa-config-tty'%(self.name())
944 fileconf=open(tmpname,'w')
945 for var in [ 'SFA_REGISTRY_ROOT_AUTH',
946 'SFA_REGISTRY_LEVEL1_AUTH',
948 'SFA_AGGREGATE_HOST',
954 'SFA_PLC_DB_PASSWORD',
956 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec['sfa'][var]))
957 fileconf.write('w\n')
958 fileconf.write('R\n')
959 fileconf.write('q\n')
961 utils.system('cat %s'%tmpname)
962 self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
963 utils.system('rm %s'%tmpname)
966 def import_sfa(self):
968 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
969 self.run_in_guest('sfa-import-plc.py')
971 # self.run_in_guest('cp /etc/sfa/authorities/%s/%s.pkey /etc/sfa/authorities/server.key'%(auth,auth))
976 self.run_in_guest('service sfa start')
980 "sfi client configuration"
982 if os.path.exists(dir_name):
983 utils.system('rm -rf %s'%dir_name)
984 utils.system('mkdir %s'%dir_name)
985 file_name=dir_name + os.sep + 'fake-pi1.pkey'
986 fileconf=open(file_name,'w')
987 fileconf.write (self.plc_spec['keys'][0]['private'])
990 file_name=dir_name + os.sep + 'sfi_config'
991 fileconf=open(file_name,'w')
992 SFI_AUTH=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']+".main"
993 fileconf.write ("SFI_AUTH='%s'"%SFI_AUTH)
995 SFI_USER=SFI_AUTH+'.fake-pi1'
996 fileconf.write ("SFI_USER='%s'"%SFI_USER)
998 SFI_REGISTRY='http://' + self.plc_spec['sfa']['SFA_PLC_DB_HOST'] + ':12345/'
999 fileconf.write ("SFI_REGISTRY='%s'"%SFI_REGISTRY)
1000 fileconf.write('\n')
1001 SFI_SM='http://' + self.plc_spec['sfa']['SFA_PLC_DB_HOST'] + ':12347/'
1002 fileconf.write ("SFI_SM='%s'"%SFI_SM)
1003 fileconf.write('\n')
1006 file_name=dir_name + os.sep + 'person.xml'
1007 fileconf=open(file_name,'w')
1008 for record in self.plc_spec['sfa']['sfa_person_xml']:
1009 person_record=record
1010 fileconf.write(person_record)
1011 fileconf.write('\n')
1014 file_name=dir_name + os.sep + 'slice.xml'
1015 fileconf=open(file_name,'w')
1016 for record in self.plc_spec['sfa']['sfa_slice_xml']:
1018 #slice_record=self.plc_spec['sfa']['sfa_slice_xml']
1019 fileconf.write(slice_record)
1020 fileconf.write('\n')
1023 file_name=dir_name + os.sep + 'slice.rspec'
1024 fileconf=open(file_name,'w')
1026 for (key, value) in self.plc_spec['sfa']['sfa_slice_rspec'].items():
1028 fileconf.write(slice_rspec)
1029 fileconf.write('\n')
1032 remote="/vservers/%s/%s"%(self.vservername,location)
1033 self.test_ssh.copy_abs(dir_name, remote, recursive=True)
1035 #utils.system('cat %s'%tmpname)
1036 utils.system('rm -rf %s'%dir_name)
1040 "run sfi.py add (on Registry) and sfi.py create (on SM) to form new objects"
1042 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1043 success=test_user_sfa.add_user()
1045 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1046 site_spec = self.locate_site (slice_spec['sitename'])
1047 test_site = TestSite(self,site_spec)
1048 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1049 success1=test_slice_sfa.add_slice()
1050 success2=test_slice_sfa.create_slice()
1051 return success and success1 and success2
1053 def update_sfa(self):
1054 "run sfi.py update (on Registry) and sfi.py create (on SM) on existing objects"
1056 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1057 success1=test_user_sfa.update_user()
1059 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1060 site_spec = self.locate_site (slice_spec['sitename'])
1061 test_site = TestSite(self,site_spec)
1062 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1063 success2=test_slice_sfa.update_slice()
1064 return success1 and success2
1067 "run sfi.py list and sfi.py show (both on Registry) and sfi.py slices and sfi.py resources (both on SM)"
1068 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1070 self.run_in_guest("sfi.py -d /root/.sfi/ list %s.main"%auth)==0 and \
1071 self.run_in_guest("sfi.py -d /root/.sfi/ show %s.main"%auth)==0 and \
1072 self.run_in_guest("sfi.py -d /root/.sfi/ slices")==0 and \
1073 self.run_in_guest("sfi.py -d /root/.sfi/ resources -o resources")==0
1075 @slice_mapper_options_sfa
1076 def check_slice_sfa(self):
1077 "tries to ssh-enter the SFA slice"
1080 def delete_sfa(self):
1081 "run sfi.py delete (on SM), sfi.py remove (on Registry)"
1083 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1084 success1=test_user_sfa.delete_user()
1085 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1086 site_spec = self.locate_site (slice_spec['sitename'])
1087 test_site = TestSite(self,site_spec)
1088 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1089 success2=test_slice_sfa.delete_slice()
1091 return success1 and success2
1095 self.run_in_guest('service sfa stop')
1098 def populate (self):
1099 "creates random entries in the PLCAPI"
1100 # install the stress-test in the plc image
1101 location = "/usr/share/plc_api/plcsh_stress_test.py"
1102 remote="/vservers/%s/%s"%(self.vservername,location)
1103 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1105 command += " -- --preserve --short-names"
1106 local = (self.run_in_guest(command) == 0);
1107 # second run with --foreign
1108 command += ' --foreign'
1109 remote = (self.run_in_guest(command) == 0);
1110 return ( local and remote)
1112 def gather_logs (self):
1113 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1114 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1115 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1116 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1117 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1118 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1120 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1121 self.gather_var_logs ()
1123 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1124 self.gather_pgsql_logs ()
1126 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1127 for site_spec in self.plc_spec['sites']:
1128 test_site = TestSite (self,site_spec)
1129 for node_spec in site_spec['nodes']:
1130 test_node=TestNode(self,test_site,node_spec)
1131 test_node.gather_qemu_logs()
1133 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1134 self.gather_nodes_var_logs()
1136 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1137 self.gather_slivers_var_logs()
1140 def gather_slivers_var_logs(self):
1141 for test_sliver in self.all_sliver_objs():
1142 remote = test_sliver.tar_var_logs()
1143 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
1144 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
1145 utils.system(command)
1148 def gather_var_logs (self):
1149 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
1150 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1151 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
1152 utils.system(command)
1153 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd"%self.name()
1154 utils.system(command)
1156 def gather_pgsql_logs (self):
1157 utils.system("mkdir -p logs/myplc.pgsql-log.%s"%self.name())
1158 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1159 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -"%self.name()
1160 utils.system(command)
1162 def gather_nodes_var_logs (self):
1163 for site_spec in self.plc_spec['sites']:
1164 test_site = TestSite (self,site_spec)
1165 for node_spec in site_spec['nodes']:
1166 test_node=TestNode(self,test_site,node_spec)
1167 test_ssh = TestSsh (test_node.name(),key="keys/key1.rsa")
1168 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1169 command = command + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
1170 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
1171 utils.system(command)
1174 # returns the filename to use for sql dump/restore, using options.dbname if set
1175 def dbfile (self, database):
1176 # uses options.dbname if it is found
1178 name=self.options.dbname
1179 if not isinstance(name,StringTypes):
1182 t=datetime.datetime.now()
1185 return "/root/%s-%s.sql"%(database,name)
1188 dump=self.dbfile("planetab4")
1189 self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
1190 utils.header('Dumped planetlab4 database in %s'%dump)
1193 def db_restore(self):
1194 dump=self.dbfile("planetab4")
1195 ##stop httpd service
1196 self.run_in_guest('service httpd stop')
1197 # xxx - need another wrapper
1198 self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
1199 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
1200 self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
1201 ##starting httpd service
1202 self.run_in_guest('service httpd start')
1204 utils.header('Database restored from ' + dump)
1207 def standby_1(): pass
1209 def standby_2(): pass
1211 def standby_3(): pass
1213 def standby_4(): pass
1215 def standby_5(): pass
1217 def standby_6(): pass
1219 def standby_7(): pass
1221 def standby_8(): pass
1223 def standby_9(): pass
1225 def standby_10(): pass
1227 def standby_11(): pass
1229 def standby_12(): pass
1231 def standby_13(): pass
1233 def standby_14(): pass
1235 def standby_15(): pass
1237 def standby_16(): pass
1239 def standby_17(): pass
1241 def standby_18(): pass
1243 def standby_19(): pass
1245 def standby_20(): pass