7 from types import StringTypes
11 from TestSite import TestSite
12 from TestNode import TestNode
13 from TestUser import TestUser
14 from TestKey import TestKey
15 from TestSlice import TestSlice
16 from TestSliver import TestSliver
17 from TestBox import TestBox
18 from TestSsh import TestSsh
19 from TestApiserver import TestApiserver
20 from TestSliceSfa import TestSliceSfa
21 from TestUserSfa import TestUserSfa
23 # step methods must take (self) and return a boolean (options is a member of the class)
25 def standby(minutes,dry_run):
26 utils.header('Entering StandBy for %d mn'%minutes)
30 time.sleep(60*minutes)
33 def standby_generic (func):
35 minutes=int(func.__name__.split("_")[1])
36 return standby(minutes,self.options.dry_run)
39 def node_mapper (method):
42 node_method = TestNode.__dict__[method.__name__]
43 for site_spec in self.plc_spec['sites']:
44 test_site = TestSite (self,site_spec)
45 for node_spec in site_spec['nodes']:
46 test_node = TestNode (self,test_site,node_spec)
47 if not node_method(test_node): overall=False
49 # restore the doc text
50 actual.__doc__=method.__doc__
53 def slice_mapper_options (method):
56 slice_method = TestSlice.__dict__[method.__name__]
57 for slice_spec in self.plc_spec['slices']:
58 site_spec = self.locate_site (slice_spec['sitename'])
59 test_site = TestSite(self,site_spec)
60 test_slice=TestSlice(self,test_site,slice_spec)
61 if not slice_method(test_slice,self.options): overall=False
63 # restore the doc text
64 actual.__doc__=method.__doc__
67 def slice_mapper_options_sfa (method):
71 slice_method = TestSliceSfa.__dict__[method.__name__]
72 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
73 site_spec = self.locate_site (slice_spec['sitename'])
74 test_site = TestSite(self,site_spec)
75 test_slice=TestSliceSfa(test_plc,test_site,slice_spec)
76 if not slice_method(test_slice,self.options): overall=False
78 # restore the doc text
79 actual.__doc__=method.__doc__
87 'display', 'local_pre', SEP,
88 'delete','create','install', 'configure', 'start', SEP,
89 'fetch_keys', 'store_keys', 'clear_known_hosts', SEP,
90 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', SEP,
91 'reinstall_node', 'init_node','bootcd', 'configure_qemu', 'export_qemu',
92 'kill_all_qemus', 'start_node', SEP,
93 # better use of time: do this now that the nodes are taking off
94 'plcsh_stress_test', SEP,
95 'nodes_ssh_debug', 'nodes_ssh_boot', 'check_slice', 'check_initscripts', SEP,
96 'install_sfa', 'configure_sfa', 'import_sfa', 'start_sfa', SEP,
97 'setup_sfa', 'add_sfa', 'update_sfa', SEP,
98 'view_sfa', 'check_slice_sfa', 'delete_sfa', 'stop_sfa', SEP,
99 'check_tcp', 'check_hooks', SEP,
100 'force_gather_logs', 'force_local_post',
103 'fresh_install', 'stop', 'vs_start', SEP,
104 'clean_initscripts', 'clean_nodegroups','clean_all_sites', SEP,
105 'clean_sites', 'clean_nodes', 'clean_slices', 'clean_keys', SEP,
107 'show_boxes', 'list_all_qemus', 'list_qemus', 'kill_qemus', SEP,
108 'db_dump' , 'db_restore', SEP,
109 'local_list','local_cleanup',SEP,
110 'standby_1 through 20',
114 def printable_steps (list):
115 return " ".join(list).replace(" "+SEP+" "," \\\n")
117 def valid_step (step):
120 def __init__ (self,plc_spec,options):
121 self.plc_spec=plc_spec
123 self.test_ssh=TestSsh(self.plc_spec['hostname'],self.options.buildname)
125 self.vserverip=plc_spec['vserverip']
126 self.vservername=plc_spec['vservername']
127 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
130 raise Exception,'chroot-based myplc testing is deprecated'
131 self.apiserver=TestApiserver(self.url,options.dry_run)
134 name=self.plc_spec['name']
135 return "%s.%s"%(name,self.vservername)
138 return self.plc_spec['hostname']
141 return self.test_ssh.is_local()
143 # define the API methods on this object through xmlrpc
144 # would help, but not strictly necessary
148 def actual_command_in_guest (self,command):
149 return self.test_ssh.actual_command(self.host_to_guest(command))
151 def start_guest (self):
152 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host()))
154 def run_in_guest (self,command):
155 return utils.system(self.actual_command_in_guest(command))
157 def run_in_host (self,command):
158 return self.test_ssh.run_in_buildname(command)
160 #command gets run in the vserver
161 def host_to_guest(self,command):
162 return "vserver %s exec %s"%(self.vservername,command)
164 #command gets run in the vserver
165 def start_guest_in_host(self):
166 return "vserver %s start"%(self.vservername)
169 def run_in_guest_piped (self,local,remote):
170 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
172 def auth_root (self):
173 return {'Username':self.plc_spec['PLC_ROOT_USER'],
174 'AuthMethod':'password',
175 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
176 'Role' : self.plc_spec['role']
178 def locate_site (self,sitename):
179 for site in self.plc_spec['sites']:
180 if site['site_fields']['name'] == sitename:
182 if site['site_fields']['login_base'] == sitename:
184 raise Exception,"Cannot locate site %s"%sitename
186 def locate_node (self,nodename):
187 for site in self.plc_spec['sites']:
188 for node in site['nodes']:
189 if node['name'] == nodename:
191 raise Exception,"Cannot locate node %s"%nodename
193 def locate_hostname (self,hostname):
194 for site in self.plc_spec['sites']:
195 for node in site['nodes']:
196 if node['node_fields']['hostname'] == hostname:
198 raise Exception,"Cannot locate hostname %s"%hostname
200 def locate_key (self,keyname):
201 for key in self.plc_spec['keys']:
202 if key['name'] == keyname:
204 raise Exception,"Cannot locate key %s"%keyname
206 def locate_slice (self, slicename):
207 for slice in self.plc_spec['slices']:
208 if slice['slice_fields']['name'] == slicename:
210 raise Exception,"Cannot locate slice %s"%slicename
212 def all_sliver_objs (self):
214 for slice_spec in self.plc_spec['slices']:
215 slicename = slice_spec['slice_fields']['name']
216 for nodename in slice_spec['nodenames']:
217 result.append(self.locate_sliver_obj (nodename,slicename))
220 def locate_sliver_obj (self,nodename,slicename):
221 (site,node) = self.locate_node(nodename)
222 slice = self.locate_slice (slicename)
224 test_site = TestSite (self, site)
225 test_node = TestNode (self, test_site,node)
226 # xxx the slice site is assumed to be the node site - mhh - probably harmless
227 test_slice = TestSlice (self, test_site, slice)
228 return TestSliver (self, test_node, test_slice)
230 def locate_first_node(self):
231 nodename=self.plc_spec['slices'][0]['nodenames'][0]
232 (site,node) = self.locate_node(nodename)
233 test_site = TestSite (self, site)
234 test_node = TestNode (self, test_site,node)
237 def locate_first_sliver (self):
238 slice_spec=self.plc_spec['slices'][0]
239 slicename=slice_spec['slice_fields']['name']
240 nodename=slice_spec['nodenames'][0]
241 return self.locate_sliver_obj(nodename,slicename)
243 # all different hostboxes used in this plc
244 def gather_hostBoxes(self):
245 # maps on sites and nodes, return [ (host_box,test_node) ]
247 for site_spec in self.plc_spec['sites']:
248 test_site = TestSite (self,site_spec)
249 for node_spec in site_spec['nodes']:
250 test_node = TestNode (self, test_site, node_spec)
251 if not test_node.is_real():
252 tuples.append( (test_node.host_box(),test_node) )
253 # transform into a dict { 'host_box' -> [ test_node .. ] }
255 for (box,node) in tuples:
256 if not result.has_key(box):
259 result[box].append(node)
262 # a step for checking this stuff
263 def show_boxes (self):
264 for (box,nodes) in self.gather_hostBoxes().iteritems():
265 print box,":"," + ".join( [ node.name() for node in nodes ] )
268 # make this a valid step
269 def kill_all_qemus(self):
270 "all qemu boxes: kill all running qemus (even of former runs)"
271 # this is the brute force version, kill all qemus on that host box
272 for (box,nodes) in self.gather_hostBoxes().iteritems():
273 # pass the first nodename, as we don't push template-qemu on testboxes
274 nodedir=nodes[0].nodedir()
275 TestBox(box,self.options.buildname).kill_all_qemus(nodedir)
278 # make this a valid step
279 def list_all_qemus(self):
280 for (box,nodes) in self.gather_hostBoxes().iteritems():
281 # this is the brute force version, kill all qemus on that host box
282 TestBox(box,self.options.buildname).list_all_qemus()
285 # kill only the right qemus
286 def list_qemus(self):
287 for (box,nodes) in self.gather_hostBoxes().iteritems():
288 # the fine-grain version
293 # kill only the right qemus
294 def kill_qemus(self):
295 for (box,nodes) in self.gather_hostBoxes().iteritems():
296 # the fine-grain version
301 #################### display config
303 "show test configuration after localization"
304 self.display_pass (1)
305 self.display_pass (2)
309 def display_pass (self,passno):
310 for (key,val) in self.plc_spec.iteritems():
314 self.display_site_spec(site)
315 for node in site['nodes']:
316 self.display_node_spec(node)
317 elif key=='initscripts':
318 for initscript in val:
319 self.display_initscript_spec (initscript)
322 self.display_slice_spec (slice)
325 self.display_key_spec (key)
327 if key not in ['sites','initscripts','slices','keys']:
328 print '+ ',key,':',val
330 def display_site_spec (self,site):
331 print '+ ======== site',site['site_fields']['name']
332 for (k,v) in site.iteritems():
335 print '+ ','nodes : ',
337 print node['node_fields']['hostname'],'',
343 print user['name'],'',
345 elif k == 'site_fields':
346 print '+ login_base',':',v['login_base']
347 elif k == 'address_fields':
351 PrettyPrinter(indent=8,depth=2).pprint(v)
353 def display_initscript_spec (self,initscript):
354 print '+ ======== initscript',initscript['initscript_fields']['name']
356 def display_key_spec (self,key):
357 print '+ ======== key',key['name']
359 def display_slice_spec (self,slice):
360 print '+ ======== slice',slice['slice_fields']['name']
361 for (k,v) in slice.iteritems():
374 elif k=='slice_fields':
375 print '+ fields',':',
376 print 'max_nodes=',v['max_nodes'],
381 def display_node_spec (self,node):
382 print "+ node",node['name'],"host_box=",node['host_box'],
383 print "hostname=",node['node_fields']['hostname'],
384 print "ip=",node['interface_fields']['ip']
387 # another entry point for just showing the boxes involved
388 def display_mapping (self):
389 TestPlc.display_mapping_plc(self.plc_spec)
393 def display_mapping_plc (plc_spec):
394 print '+ MyPLC',plc_spec['name']
395 print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['hostname'],plc_spec['vservername'])
396 print '+\tIP = %s/%s'%(plc_spec['PLC_API_HOST'],plc_spec['vserverip'])
397 for site_spec in plc_spec['sites']:
398 for node_spec in site_spec['nodes']:
399 TestPlc.display_mapping_node(node_spec)
402 def display_mapping_node (node_spec):
403 print '+ NODE %s'%(node_spec['name'])
404 print '+\tqemu box %s'%node_spec['host_box']
405 print '+\thostname=%s'%node_spec['node_fields']['hostname']
407 def local_pre (self):
408 "run site-dependant pre-test script as defined in LocalTestResources"
409 from LocalTestResources import local_resources
410 return local_resources.step_pre(self)
412 def local_post (self):
413 "run site-dependant post-test script as defined in LocalTestResources"
414 from LocalTestResources import local_resources
415 return local_resources.step_post(self)
417 def local_list (self):
418 "run site-dependant list script as defined in LocalTestResources"
419 from LocalTestResources import local_resources
420 return local_resources.step_list(self)
422 def local_cleanup (self):
423 "run site-dependant cleanup script as defined in LocalTestResources"
424 from LocalTestResources import local_resources
425 return local_resources.step_cleanup(self)
428 "vserver delete the test myplc"
429 self.run_in_host("vserver --silent %s delete"%self.vservername)
434 "vserver creation (no install done)"
436 # a full path for the local calls
437 build_dir=os.path.dirname(sys.argv[0])
438 # sometimes this is empty - set to "." in such a case
439 if not build_dir: build_dir="."
440 build_dir += "/build"
442 # use a standard name - will be relative to remote buildname
444 # run checkout in any case - would do an update if already exists
445 build_checkout = "svn checkout %s %s"%(self.options.build_url,build_dir)
446 if self.run_in_host(build_checkout) != 0:
448 # the repo url is taken from arch-rpms-url
449 # with the last step (i386) removed
450 repo_url = self.options.arch_rpms_url
451 for level in [ 'arch' ]:
452 repo_url = os.path.dirname(repo_url)
453 # pass the vbuild-nightly options to vtest-init-vserver
455 test_env_options += " -p %s"%self.options.personality
456 test_env_options += " -d %s"%self.options.pldistro
457 test_env_options += " -f %s"%self.options.fcdistro
458 script="vtest-init-vserver.sh"
459 vserver_name = self.vservername
460 vserver_options="--netdev eth0 --interface %s"%self.vserverip
462 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
463 vserver_options += " --hostname %s"%vserver_hostname
466 create_vserver="%(build_dir)s/%(script)s %(test_env_options)s %(vserver_name)s %(repo_url)s -- %(vserver_options)s"%locals()
467 return self.run_in_host(create_vserver) == 0
471 "yum install myplc, noderepo, and the plain bootstrapfs"
472 if self.options.personality == "linux32":
474 elif self.options.personality == "linux64":
477 raise Exception, "Unsupported personality %r"%self.options.personality
479 self.run_in_guest("yum -y install myplc")==0 and \
480 nodefamily="%s-%s-%s"%(self.options.pldistro,self.options.fcdistro,arch)
481 self.run_in_guest("yum -y install noderepo-%s"%nodefamily)==0 and \
482 self.run_in_guest("yum -y install bootstrapfs-%s-plain"%nodefamily)==0
487 tmpname='%s.plc-config-tty'%(self.name())
488 fileconf=open(tmpname,'w')
489 for var in [ 'PLC_NAME',
493 'PLC_MAIL_SUPPORT_ADDRESS',
496 # Above line was added for integrating SFA Testing
502 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
503 fileconf.write('w\n')
504 fileconf.write('q\n')
506 utils.system('cat %s'%tmpname)
507 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
508 utils.system('rm %s'%tmpname)
513 self.run_in_guest('service plc start')
518 self.run_in_guest('service plc stop')
525 # stores the keys from the config for further use
526 def store_keys(self):
527 "stores test users ssh keys in keys/"
528 for key_spec in self.plc_spec['keys']:
529 TestKey(self,key_spec).store_key()
532 def clean_keys(self):
533 utils.system("rm -rf %s/keys/"%os.path(sys.argv[0]))
535 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
536 # for later direct access to the nodes
537 def fetch_keys(self):
538 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
540 if not os.path.isdir(dir):
542 vservername=self.vservername
544 prefix = 'debug_ssh_key'
545 for ext in [ 'pub', 'rsa' ] :
546 src="/vservers/%(vservername)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
547 dst="keys/%(vservername)s-debug.%(ext)s"%locals()
548 if self.test_ssh.fetch(src,dst) != 0: overall=False
552 "create sites with PLCAPI"
553 return self.do_sites()
555 def clean_sites (self):
556 "delete sites with PLCAPI"
557 return self.do_sites(action="delete")
559 def do_sites (self,action="add"):
560 for site_spec in self.plc_spec['sites']:
561 test_site = TestSite (self,site_spec)
562 if (action != "add"):
563 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
564 test_site.delete_site()
565 # deleted with the site
566 #test_site.delete_users()
569 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
570 test_site.create_site()
571 test_site.create_users()
574 def clean_all_sites (self):
575 print 'auth_root',self.auth_root()
576 site_ids = [s['site_id'] for s in self.apiserver.GetSites(self.auth_root(), {}, ['site_id'])]
577 for site_id in site_ids:
578 print 'Deleting site_id',site_id
579 self.apiserver.DeleteSite(self.auth_root(),site_id)
582 "create nodes with PLCAPI"
583 return self.do_nodes()
584 def clean_nodes (self):
585 "delete nodes with PLCAPI"
586 return self.do_nodes(action="delete")
588 def do_nodes (self,action="add"):
589 for site_spec in self.plc_spec['sites']:
590 test_site = TestSite (self,site_spec)
592 utils.header("Deleting nodes in site %s"%test_site.name())
593 for node_spec in site_spec['nodes']:
594 test_node=TestNode(self,test_site,node_spec)
595 utils.header("Deleting %s"%test_node.name())
596 test_node.delete_node()
598 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
599 for node_spec in site_spec['nodes']:
600 utils.pprint('Creating node %s'%node_spec,node_spec)
601 test_node = TestNode (self,test_site,node_spec)
602 test_node.create_node ()
605 def nodegroups (self):
606 "create nodegroups with PLCAPI"
607 return self.do_nodegroups("add")
608 def clean_nodegroups (self):
609 "delete nodegroups with PLCAPI"
610 return self.do_nodegroups("delete")
612 # create nodegroups if needed, and populate
613 def do_nodegroups (self, action="add"):
614 # 1st pass to scan contents
616 for site_spec in self.plc_spec['sites']:
617 test_site = TestSite (self,site_spec)
618 for node_spec in site_spec['nodes']:
619 test_node=TestNode (self,test_site,node_spec)
620 if node_spec.has_key('nodegroups'):
621 nodegroupnames=node_spec['nodegroups']
622 if isinstance(nodegroupnames,StringTypes):
623 nodegroupnames = [ nodegroupnames ]
624 for nodegroupname in nodegroupnames:
625 if not groups_dict.has_key(nodegroupname):
626 groups_dict[nodegroupname]=[]
627 groups_dict[nodegroupname].append(test_node.name())
628 auth=self.auth_root()
630 for (nodegroupname,group_nodes) in groups_dict.iteritems():
632 print 'nodegroups:','dealing with nodegroup',nodegroupname,'on nodes',group_nodes
633 # first, check if the nodetagtype is here
634 tag_types = self.apiserver.GetTagTypes(auth,{'tagname':nodegroupname})
636 tag_type_id = tag_types[0]['tag_type_id']
638 tag_type_id = self.apiserver.AddTagType(auth,
639 {'tagname':nodegroupname,
640 'description': 'for nodegroup %s'%nodegroupname,
643 print 'located tag (type)',nodegroupname,'as',tag_type_id
645 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
647 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
648 print 'created nodegroup',nodegroupname,'from tagname',nodegroupname,'and value','yes'
649 # set node tag on all nodes, value='yes'
650 for nodename in group_nodes:
652 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
654 traceback.print_exc()
655 print 'node',nodename,'seems to already have tag',nodegroupname
658 expect_yes = self.apiserver.GetNodeTags(auth,
659 {'hostname':nodename,
660 'tagname':nodegroupname},
661 ['value'])[0]['value']
662 if expect_yes != "yes":
663 print 'Mismatch node tag on node',nodename,'got',expect_yes
666 if not self.options.dry_run:
667 print 'Cannot find tag',nodegroupname,'on node',nodename
671 print 'cleaning nodegroup',nodegroupname
672 self.apiserver.DeleteNodeGroup(auth,nodegroupname)
674 traceback.print_exc()
678 def all_hostnames (self) :
680 for site_spec in self.plc_spec['sites']:
681 hostnames += [ node_spec['node_fields']['hostname'] \
682 for node_spec in site_spec['nodes'] ]
685 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
686 def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period=15):
687 if self.options.dry_run:
691 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
692 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
693 # the nodes that haven't checked yet - start with a full list and shrink over time
694 tocheck = self.all_hostnames()
695 utils.header("checking nodes %r"%tocheck)
696 # create a dict hostname -> status
697 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
700 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
702 for array in tocheck_status:
703 hostname=array['hostname']
704 boot_state=array['boot_state']
705 if boot_state == target_boot_state:
706 utils.header ("%s has reached the %s state"%(hostname,target_boot_state))
708 # if it's a real node, never mind
709 (site_spec,node_spec)=self.locate_hostname(hostname)
710 if TestNode.is_real_model(node_spec['node_fields']['model']):
711 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
713 boot_state = target_boot_state
714 elif datetime.datetime.now() > graceout:
715 utils.header ("%s still in '%s' state"%(hostname,boot_state))
716 graceout=datetime.datetime.now()+datetime.timedelta(1)
717 status[hostname] = boot_state
719 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != target_boot_state ]
722 if datetime.datetime.now() > timeout:
723 for hostname in tocheck:
724 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
726 # otherwise, sleep for a while
728 # only useful in empty plcs
731 def nodes_booted(self):
732 return self.nodes_check_boot_state('boot',timeout_minutes=30,silent_minutes=20)
734 def check_nodes_ssh(self,debug,timeout_minutes,silent_minutes,period=15):
736 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
737 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
738 vservername=self.vservername
741 local_key = "keys/%(vservername)s-debug.rsa"%locals()
744 local_key = "keys/key1.rsa"
745 tocheck = self.all_hostnames()
746 utils.header("checking ssh access (expected in %s mode) to nodes %r"%(message,tocheck))
747 utils.header("max timeout is %d minutes, silent for %d minutes (period is %s)"%\
748 (timeout_minutes,silent_minutes,period))
750 for hostname in tocheck:
751 # try to run 'hostname' in the node
752 command = TestSsh (hostname,key=local_key).actual_command("hostname;uname -a")
753 # don't spam logs - show the command only after the grace period
754 success = utils.system ( command, silent=datetime.datetime.now() < graceout)
756 utils.header('Successfully entered root@%s (%s)'%(hostname,message))
758 tocheck.remove(hostname)
760 # we will have tried real nodes once, in case they're up - but if not, just skip
761 (site_spec,node_spec)=self.locate_hostname(hostname)
762 if TestNode.is_real_model(node_spec['node_fields']['model']):
763 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
764 tocheck.remove(hostname)
767 if datetime.datetime.now() > timeout:
768 for hostname in tocheck:
769 utils.header("FAILURE to ssh into %s"%hostname)
771 # otherwise, sleep for a while
773 # only useful in empty plcs
776 def nodes_ssh_debug(self):
777 "Tries to ssh into nodes in debug mode with the debug ssh key"
778 return self.check_nodes_ssh(debug=True,timeout_minutes=30,silent_minutes=10)
780 def nodes_ssh_boot(self):
781 "Tries to ssh into nodes in production mode with the root ssh key"
782 return self.check_nodes_ssh(debug=False,timeout_minutes=30,silent_minutes=10)
785 def init_node (self):
786 "all nodes : init a clean local directory for holding node-dep stuff like iso image..."
790 "all nodes: invoke GetBootMedium and store result locally"
793 def configure_qemu (self):
794 "all nodes: compute qemu config qemu.conf and store it locally"
797 def reinstall_node (self):
798 "all nodes: mark PLCAPI boot_state as reinstall"
801 def export_qemu (self):
802 "all nodes: push local node-dep directory on the qemu box"
805 ### check hooks : invoke scripts from hooks/{node,slice}
806 def check_hooks_node (self):
807 return self.locate_first_node().check_hooks()
808 def check_hooks_sliver (self) :
809 return self.locate_first_sliver().check_hooks()
811 def check_hooks (self):
812 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
813 return self.check_hooks_node() and self.check_hooks_sliver()
816 def do_check_initscripts(self):
818 for slice_spec in self.plc_spec['slices']:
819 if not slice_spec.has_key('initscriptname'):
821 initscript=slice_spec['initscriptname']
822 for nodename in slice_spec['nodenames']:
823 (site,node) = self.locate_node (nodename)
824 # xxx - passing the wrong site - probably harmless
825 test_site = TestSite (self,site)
826 test_slice = TestSlice (self,test_site,slice_spec)
827 test_node = TestNode (self,test_site,node)
828 test_sliver = TestSliver (self, test_node, test_slice)
829 if not test_sliver.check_initscript(initscript):
833 def check_initscripts(self):
834 "check that the initscripts have triggered"
835 return self.do_check_initscripts()
837 def initscripts (self):
838 "create initscripts with PLCAPI"
839 for initscript in self.plc_spec['initscripts']:
840 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
841 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
844 def clean_initscripts (self):
845 "delete initscripts with PLCAPI"
846 for initscript in self.plc_spec['initscripts']:
847 initscript_name = initscript['initscript_fields']['name']
848 print('Attempting to delete %s in plc %s'%(initscript_name,self.plc_spec['name']))
850 self.apiserver.DeleteInitScript(self.auth_root(),initscript_name)
851 print initscript_name,'deleted'
853 print 'deletion went wrong - probably did not exist'
858 "create slices with PLCAPI"
859 return self.do_slices()
861 def clean_slices (self):
862 "delete slices with PLCAPI"
863 return self.do_slices("delete")
865 def do_slices (self, action="add"):
866 for slice in self.plc_spec['slices']:
867 site_spec = self.locate_site (slice['sitename'])
868 test_site = TestSite(self,site_spec)
869 test_slice=TestSlice(self,test_site,slice)
871 utils.header("Deleting slices in site %s"%test_site.name())
872 test_slice.delete_slice()
874 utils.pprint("Creating slice",slice)
875 test_slice.create_slice()
876 utils.header('Created Slice %s'%slice['slice_fields']['name'])
879 @slice_mapper_options
880 def check_slice(self):
881 "tries to ssh-enter the slice with the user key, to ensure slice creation"
885 def clear_known_hosts (self):
886 "remove test nodes entries from the local known_hosts file"
890 def start_node (self) :
891 "all nodes: start the qemu instance (also runs qemu-bridge-init start)"
894 def check_tcp (self):
895 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
896 specs = self.plc_spec['tcp_test']
901 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
902 if not s_test_sliver.run_tcp_server(port,timeout=10):
906 # idem for the client side
907 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
908 if not c_test_sliver.run_tcp_client(s_test_sliver.test_node.name(),port):
912 def plcsh_stress_test (self):
913 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
914 # install the stress-test in the plc image
915 location = "/usr/share/plc_api/plcsh_stress_test.py"
916 remote="/vservers/%s/%s"%(self.vservername,location)
917 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
919 command += " -- --check"
920 if self.options.size == 1:
922 return ( self.run_in_guest(command) == 0)
924 # populate runs the same utility without slightly different options
925 # in particular runs with --preserve (dont cleanup) and without --check
926 # also it gets run twice, once with the --foreign option for creating fake foreign entries
929 def install_sfa(self):
930 "yum install sfa, sfa-plc and sfa-client"
931 if self.options.personality == "linux32":
933 elif self.options.personality == "linux64":
936 raise Exception, "Unsupported personality %r"%self.options.personality
937 return self.run_in_guest("yum -y install sfa sfa-client sfa-plc sfa-sfatables")==0
940 def configure_sfa(self):
942 tmpname='%s.sfa-config-tty'%(self.name())
943 fileconf=open(tmpname,'w')
944 for var in [ 'SFA_REGISTRY_ROOT_AUTH',
945 'SFA_REGISTRY_LEVEL1_AUTH',
947 'SFA_AGGREGATE_HOST',
953 'SFA_PLC_DB_PASSWORD',
955 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec['sfa'][var]))
956 fileconf.write('w\n')
957 fileconf.write('R\n')
958 fileconf.write('q\n')
960 utils.system('cat %s'%tmpname)
961 self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
962 utils.system('rm %s'%tmpname)
965 def import_sfa(self):
967 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
968 self.run_in_guest('sfa-import-plc.py')
970 # self.run_in_guest('cp /etc/sfa/authorities/%s/%s.pkey /etc/sfa/authorities/server.key'%(auth,auth))
975 self.run_in_guest('service sfa start')
979 "sfi client configuration"
981 if os.path.exists(dir_name):
982 utils.system('rm -rf %s'%dir_name)
983 utils.system('mkdir %s'%dir_name)
984 file_name=dir_name + os.sep + 'fake-pi1.pkey'
985 fileconf=open(file_name,'w')
986 fileconf.write (self.plc_spec['keys'][0]['private'])
989 file_name=dir_name + os.sep + 'sfi_config'
990 fileconf=open(file_name,'w')
991 SFI_AUTH=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']+".main"
992 fileconf.write ("SFI_AUTH='%s'"%SFI_AUTH)
994 SFI_USER=SFI_AUTH+'.fake-pi1'
995 fileconf.write ("SFI_USER='%s'"%SFI_USER)
997 SFI_REGISTRY='http://' + self.plc_spec['sfa']['SFA_PLC_DB_HOST'] + ':12345/'
998 fileconf.write ("SFI_REGISTRY='%s'"%SFI_REGISTRY)
1000 SFI_SM='http://' + self.plc_spec['sfa']['SFA_PLC_DB_HOST'] + ':12347/'
1001 fileconf.write ("SFI_SM='%s'"%SFI_SM)
1002 fileconf.write('\n')
1005 file_name=dir_name + os.sep + 'person.xml'
1006 fileconf=open(file_name,'w')
1007 for record in self.plc_spec['sfa']['sfa_person_xml']:
1008 person_record=record
1009 fileconf.write(person_record)
1010 fileconf.write('\n')
1013 file_name=dir_name + os.sep + 'slice.xml'
1014 fileconf=open(file_name,'w')
1015 for record in self.plc_spec['sfa']['sfa_slice_xml']:
1017 #slice_record=self.plc_spec['sfa']['sfa_slice_xml']
1018 fileconf.write(slice_record)
1019 fileconf.write('\n')
1022 file_name=dir_name + os.sep + 'slice.rspec'
1023 fileconf=open(file_name,'w')
1025 for (key, value) in self.plc_spec['sfa']['sfa_slice_rspec'].items():
1027 fileconf.write(slice_rspec)
1028 fileconf.write('\n')
1031 remote="/vservers/%s/%s"%(self.vservername,location)
1032 self.test_ssh.copy_abs(dir_name, remote, recursive=True)
1034 #utils.system('cat %s'%tmpname)
1035 utils.system('rm -rf %s'%dir_name)
1039 "run sfi.py add (on Registry) and sfi.py create (on SM) to form new objects"
1041 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1042 success=test_user_sfa.add_user()
1044 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1045 site_spec = self.locate_site (slice_spec['sitename'])
1046 test_site = TestSite(self,site_spec)
1047 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1048 success1=test_slice_sfa.add_slice()
1049 success2=test_slice_sfa.create_slice()
1050 return success and success1 and success2
1052 def update_sfa(self):
1053 "run sfi.py update (on Registry) and sfi.py create (on SM) on existing objects"
1055 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1056 success1=test_user_sfa.update_user()
1058 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1059 site_spec = self.locate_site (slice_spec['sitename'])
1060 test_site = TestSite(self,site_spec)
1061 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1062 success2=test_slice_sfa.update_slice()
1063 return success1 and success2
1066 "run sfi.py list and sfi.py show (both on Registry) and sfi.py slices and sfi.py resources (both on SM)"
1067 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1069 self.run_in_guest("sfi.py -d /root/.sfi/ list %s.main"%auth)==0 and \
1070 self.run_in_guest("sfi.py -d /root/.sfi/ show %s.main"%auth)==0 and \
1071 self.run_in_guest("sfi.py -d /root/.sfi/ slices")==0 and \
1072 self.run_in_guest("sfi.py -d /root/.sfi/ resources -o resources")==0
1074 @slice_mapper_options_sfa
1075 def check_slice_sfa(self):
1076 "tries to ssh-enter the SFA slice"
1079 def delete_sfa(self):
1080 "run sfi.py delete (on SM), sfi.py remove (on Registry)"
1082 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1083 success1=test_user_sfa.delete_user()
1084 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1085 site_spec = self.locate_site (slice_spec['sitename'])
1086 test_site = TestSite(self,site_spec)
1087 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1088 success2=test_slice_sfa.delete_slice()
1090 return success1 and success2
1094 self.run_in_guest('service sfa stop')
1097 def populate (self):
1098 "creates random entries in the PLCAPI"
1099 # install the stress-test in the plc image
1100 location = "/usr/share/plc_api/plcsh_stress_test.py"
1101 remote="/vservers/%s/%s"%(self.vservername,location)
1102 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1104 command += " -- --preserve --short-names"
1105 local = (self.run_in_guest(command) == 0);
1106 # second run with --foreign
1107 command += ' --foreign'
1108 remote = (self.run_in_guest(command) == 0);
1109 return ( local and remote)
1111 def gather_logs (self):
1112 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1113 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1114 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1115 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1116 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1117 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1119 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1120 self.gather_var_logs ()
1122 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1123 self.gather_pgsql_logs ()
1125 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1126 for site_spec in self.plc_spec['sites']:
1127 test_site = TestSite (self,site_spec)
1128 for node_spec in site_spec['nodes']:
1129 test_node=TestNode(self,test_site,node_spec)
1130 test_node.gather_qemu_logs()
1132 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1133 self.gather_nodes_var_logs()
1135 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1136 self.gather_slivers_var_logs()
1139 def gather_slivers_var_logs(self):
1140 for test_sliver in self.all_sliver_objs():
1141 remote = test_sliver.tar_var_logs()
1142 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
1143 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
1144 utils.system(command)
1147 def gather_var_logs (self):
1148 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
1149 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1150 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
1151 utils.system(command)
1152 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd"%self.name()
1153 utils.system(command)
1155 def gather_pgsql_logs (self):
1156 utils.system("mkdir -p logs/myplc.pgsql-log.%s"%self.name())
1157 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1158 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -"%self.name()
1159 utils.system(command)
1161 def gather_nodes_var_logs (self):
1162 for site_spec in self.plc_spec['sites']:
1163 test_site = TestSite (self,site_spec)
1164 for node_spec in site_spec['nodes']:
1165 test_node=TestNode(self,test_site,node_spec)
1166 test_ssh = TestSsh (test_node.name(),key="keys/key1.rsa")
1167 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1168 command = command + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
1169 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
1170 utils.system(command)
1173 # returns the filename to use for sql dump/restore, using options.dbname if set
1174 def dbfile (self, database):
1175 # uses options.dbname if it is found
1177 name=self.options.dbname
1178 if not isinstance(name,StringTypes):
1181 t=datetime.datetime.now()
1184 return "/root/%s-%s.sql"%(database,name)
1187 dump=self.dbfile("planetab4")
1188 self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
1189 utils.header('Dumped planetlab4 database in %s'%dump)
1192 def db_restore(self):
1193 dump=self.dbfile("planetab4")
1194 ##stop httpd service
1195 self.run_in_guest('service httpd stop')
1196 # xxx - need another wrapper
1197 self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
1198 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
1199 self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
1200 ##starting httpd service
1201 self.run_in_guest('service httpd start')
1203 utils.header('Database restored from ' + dump)
1206 def standby_1(): pass
1208 def standby_2(): pass
1210 def standby_3(): pass
1212 def standby_4(): pass
1214 def standby_5(): pass
1216 def standby_6(): pass
1218 def standby_7(): pass
1220 def standby_8(): pass
1222 def standby_9(): pass
1224 def standby_10(): pass
1226 def standby_11(): pass
1228 def standby_12(): pass
1230 def standby_13(): pass
1232 def standby_14(): pass
1234 def standby_15(): pass
1236 def standby_16(): pass
1238 def standby_17(): pass
1240 def standby_18(): pass
1242 def standby_19(): pass
1244 def standby_20(): pass