7 from types import StringTypes
11 from TestSite import TestSite
12 from TestNode import TestNode
13 from TestUser import TestUser
14 from TestKey import TestKey
15 from TestSlice import TestSlice
16 from TestSliver import TestSliver
17 from TestBox import TestBox
18 from TestSsh import TestSsh
19 from TestApiserver import TestApiserver
20 from TestSliceSfa import TestSliceSfa
21 from TestUserSfa import TestUserSfa
23 # step methods must take (self) and return a boolean (options is a member of the class)
25 def standby(minutes,dry_run):
26 utils.header('Entering StandBy for %d mn'%minutes)
30 time.sleep(60*minutes)
33 def standby_generic (func):
35 minutes=int(func.__name__.split("_")[1])
36 return standby(minutes,self.options.dry_run)
39 def node_mapper (method):
42 node_method = TestNode.__dict__[method.__name__]
43 for site_spec in self.plc_spec['sites']:
44 test_site = TestSite (self,site_spec)
45 for node_spec in site_spec['nodes']:
46 test_node = TestNode (self,test_site,node_spec)
47 if not node_method(test_node): overall=False
49 # restore the doc text
50 actual.__doc__=method.__doc__
53 def slice_mapper_options (method):
56 slice_method = TestSlice.__dict__[method.__name__]
57 for slice_spec in self.plc_spec['slices']:
58 site_spec = self.locate_site (slice_spec['sitename'])
59 test_site = TestSite(self,site_spec)
60 test_slice=TestSlice(self,test_site,slice_spec)
61 if not slice_method(test_slice,self.options): overall=False
63 # restore the doc text
64 actual.__doc__=method.__doc__
67 def slice_mapper_options_sfa (method):
71 slice_method = TestSliceSfa.__dict__[method.__name__]
72 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
73 site_spec = self.locate_site (slice_spec['sitename'])
74 test_site = TestSite(self,site_spec)
75 test_slice=TestSliceSfa(test_plc,test_site,slice_spec)
76 if not slice_method(test_slice,self.options): overall=False
78 # restore the doc text
79 actual.__doc__=method.__doc__
87 'display', 'local_pre', SEP,
88 'delete','create','install', 'configure', 'start', SEP,
89 'fetch_keys', 'store_keys', 'clear_known_hosts', SEP,
90 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', SEP,
91 'reinstall_node', 'init_node','bootcd', 'configure_qemu', 'export_qemu',
92 'kill_all_qemus', 'start_node', SEP,
93 # better use of time: do this now that the nodes are taking off
94 'plcsh_stress_test', SEP,
95 'nodes_ssh_debug', 'nodes_ssh_boot', 'check_slice', 'check_initscripts', SEP,
96 'install_sfa', 'configure_sfa', 'import_sfa', 'start_sfa', SEP,
97 'setup_sfa', 'add_sfa', 'update_sfa', SEP,
98 'view_sfa', 'check_slice_sfa', 'delete_sfa', 'stop_sfa', SEP,
99 'check_tcp', 'check_hooks', SEP,
100 'force_gather_logs', 'force_local_post',
103 'fresh_install', 'stop', 'vs_start', SEP,
104 'clean_initscripts', 'clean_nodegroups','clean_all_sites', SEP,
105 'clean_sites', 'clean_nodes', 'clean_slices', 'clean_keys', SEP,
107 'show_boxes', 'list_all_qemus', 'list_qemus', 'kill_qemus', SEP,
108 'db_dump' , 'db_restore', SEP,
109 'local_list','local_cleanup',SEP,
110 'standby_1 through 20',
114 def printable_steps (list):
115 return " ".join(list).replace(" "+SEP+" "," \\\n")
117 def valid_step (step):
120 def __init__ (self,plc_spec,options):
121 self.plc_spec=plc_spec
123 self.test_ssh=TestSsh(self.plc_spec['hostname'],self.options.buildname)
125 self.vserverip=plc_spec['vserverip']
126 self.vservername=plc_spec['vservername']
127 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
130 raise Exception,'chroot-based myplc testing is deprecated'
131 self.apiserver=TestApiserver(self.url,options.dry_run)
134 name=self.plc_spec['name']
135 return "%s.%s"%(name,self.vservername)
138 return self.plc_spec['hostname']
141 return self.test_ssh.is_local()
143 # define the API methods on this object through xmlrpc
144 # would help, but not strictly necessary
148 def actual_command_in_guest (self,command):
149 return self.test_ssh.actual_command(self.host_to_guest(command))
151 def start_guest (self):
152 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host()))
154 def run_in_guest (self,command):
155 return utils.system(self.actual_command_in_guest(command))
157 def run_in_host (self,command):
158 return self.test_ssh.run_in_buildname(command)
160 #command gets run in the vserver
161 def host_to_guest(self,command):
162 return "vserver %s exec %s"%(self.vservername,command)
164 #command gets run in the vserver
165 def start_guest_in_host(self):
166 return "vserver %s start"%(self.vservername)
169 def run_in_guest_piped (self,local,remote):
170 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
172 def auth_root (self):
173 return {'Username':self.plc_spec['PLC_ROOT_USER'],
174 'AuthMethod':'password',
175 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
176 'Role' : self.plc_spec['role']
178 def locate_site (self,sitename):
179 for site in self.plc_spec['sites']:
180 if site['site_fields']['name'] == sitename:
182 if site['site_fields']['login_base'] == sitename:
184 raise Exception,"Cannot locate site %s"%sitename
186 def locate_node (self,nodename):
187 for site in self.plc_spec['sites']:
188 for node in site['nodes']:
189 if node['name'] == nodename:
191 raise Exception,"Cannot locate node %s"%nodename
193 def locate_hostname (self,hostname):
194 for site in self.plc_spec['sites']:
195 for node in site['nodes']:
196 if node['node_fields']['hostname'] == hostname:
198 raise Exception,"Cannot locate hostname %s"%hostname
200 def locate_key (self,keyname):
201 for key in self.plc_spec['keys']:
202 if key['name'] == keyname:
204 raise Exception,"Cannot locate key %s"%keyname
206 def locate_slice (self, slicename):
207 for slice in self.plc_spec['slices']:
208 if slice['slice_fields']['name'] == slicename:
210 raise Exception,"Cannot locate slice %s"%slicename
212 def all_sliver_objs (self):
214 for slice_spec in self.plc_spec['slices']:
215 slicename = slice_spec['slice_fields']['name']
216 for nodename in slice_spec['nodenames']:
217 result.append(self.locate_sliver_obj (nodename,slicename))
220 def locate_sliver_obj (self,nodename,slicename):
221 (site,node) = self.locate_node(nodename)
222 slice = self.locate_slice (slicename)
224 test_site = TestSite (self, site)
225 test_node = TestNode (self, test_site,node)
226 # xxx the slice site is assumed to be the node site - mhh - probably harmless
227 test_slice = TestSlice (self, test_site, slice)
228 return TestSliver (self, test_node, test_slice)
230 def locate_first_node(self):
231 nodename=self.plc_spec['slices'][0]['nodenames'][0]
232 (site,node) = self.locate_node(nodename)
233 test_site = TestSite (self, site)
234 test_node = TestNode (self, test_site,node)
237 def locate_first_sliver (self):
238 slice_spec=self.plc_spec['slices'][0]
239 slicename=slice_spec['slice_fields']['name']
240 nodename=slice_spec['nodenames'][0]
241 return self.locate_sliver_obj(nodename,slicename)
243 # all different hostboxes used in this plc
244 def gather_hostBoxes(self):
245 # maps on sites and nodes, return [ (host_box,test_node) ]
247 for site_spec in self.plc_spec['sites']:
248 test_site = TestSite (self,site_spec)
249 for node_spec in site_spec['nodes']:
250 test_node = TestNode (self, test_site, node_spec)
251 if not test_node.is_real():
252 tuples.append( (test_node.host_box(),test_node) )
253 # transform into a dict { 'host_box' -> [ test_node .. ] }
255 for (box,node) in tuples:
256 if not result.has_key(box):
259 result[box].append(node)
262 # a step for checking this stuff
263 def show_boxes (self):
264 for (box,nodes) in self.gather_hostBoxes().iteritems():
265 print box,":"," + ".join( [ node.name() for node in nodes ] )
268 # make this a valid step
269 def kill_all_qemus(self):
270 "all qemu boxes: kill all running qemus (even of former runs)"
271 # this is the brute force version, kill all qemus on that host box
272 for (box,nodes) in self.gather_hostBoxes().iteritems():
273 # pass the first nodename, as we don't push template-qemu on testboxes
274 nodedir=nodes[0].nodedir()
275 TestBox(box,self.options.buildname).kill_all_qemus(nodedir)
278 # make this a valid step
279 def list_all_qemus(self):
280 for (box,nodes) in self.gather_hostBoxes().iteritems():
281 # this is the brute force version, kill all qemus on that host box
282 TestBox(box,self.options.buildname).list_all_qemus()
285 # kill only the right qemus
286 def list_qemus(self):
287 for (box,nodes) in self.gather_hostBoxes().iteritems():
288 # the fine-grain version
293 # kill only the right qemus
294 def kill_qemus(self):
295 for (box,nodes) in self.gather_hostBoxes().iteritems():
296 # the fine-grain version
301 #################### display config
303 "show test configuration after localization"
304 self.display_pass (1)
305 self.display_pass (2)
309 def display_pass (self,passno):
310 for (key,val) in self.plc_spec.iteritems():
314 self.display_site_spec(site)
315 for node in site['nodes']:
316 self.display_node_spec(node)
317 elif key=='initscripts':
318 for initscript in val:
319 self.display_initscript_spec (initscript)
322 self.display_slice_spec (slice)
325 self.display_key_spec (key)
327 if key not in ['sites','initscripts','slices','keys']:
328 print '+ ',key,':',val
330 def display_site_spec (self,site):
331 print '+ ======== site',site['site_fields']['name']
332 for (k,v) in site.iteritems():
335 print '+ ','nodes : ',
337 print node['node_fields']['hostname'],'',
343 print user['name'],'',
345 elif k == 'site_fields':
346 print '+ login_base',':',v['login_base']
347 elif k == 'address_fields':
351 PrettyPrinter(indent=8,depth=2).pprint(v)
353 def display_initscript_spec (self,initscript):
354 print '+ ======== initscript',initscript['initscript_fields']['name']
356 def display_key_spec (self,key):
357 print '+ ======== key',key['name']
359 def display_slice_spec (self,slice):
360 print '+ ======== slice',slice['slice_fields']['name']
361 for (k,v) in slice.iteritems():
374 elif k=='slice_fields':
375 print '+ fields',':',
376 print 'max_nodes=',v['max_nodes'],
381 def display_node_spec (self,node):
382 print "+ node",node['name'],"host_box=",node['host_box'],
383 print "hostname=",node['node_fields']['hostname'],
384 print "ip=",node['interface_fields']['ip']
387 # another entry point for just showing the boxes involved
388 def display_mapping (self):
389 TestPlc.display_mapping_plc(self.plc_spec)
393 def display_mapping_plc (plc_spec):
394 print '+ MyPLC',plc_spec['name']
395 print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['hostname'],plc_spec['vservername'])
396 print '+\tIP = %s/%s'%(plc_spec['PLC_API_HOST'],plc_spec['vserverip'])
397 for site_spec in plc_spec['sites']:
398 for node_spec in site_spec['nodes']:
399 TestPlc.display_mapping_node(node_spec)
402 def display_mapping_node (node_spec):
403 print '+ NODE %s'%(node_spec['name'])
404 print '+\tqemu box %s'%node_spec['host_box']
405 print '+\thostname=%s'%node_spec['node_fields']['hostname']
407 def local_pre (self):
408 "run site-dependant pre-test script as defined in LocalTestResources"
409 from LocalTestResources import local_resources
410 return local_resources.step_pre(self)
412 def local_post (self):
413 "run site-dependant post-test script as defined in LocalTestResources"
414 from LocalTestResources import local_resources
415 return local_resources.step_post(self)
417 def local_list (self):
418 "run site-dependant list script as defined in LocalTestResources"
419 from LocalTestResources import local_resources
420 return local_resources.step_list(self)
422 def local_cleanup (self):
423 "run site-dependant cleanup script as defined in LocalTestResources"
424 from LocalTestResources import local_resources
425 return local_resources.step_cleanup(self)
428 "vserver delete the test myplc"
429 self.run_in_host("vserver --silent %s delete"%self.vservername)
434 "vserver creation (no install done)"
436 # a full path for the local calls
437 build_dir=os.path.dirname(sys.argv[0])
438 # sometimes this is empty - set to "." in such a case
439 if not build_dir: build_dir="."
440 build_dir += "/build"
442 # use a standard name - will be relative to remote buildname
444 # run checkout in any case - would do an update if already exists
445 build_checkout = "svn checkout %s %s"%(self.options.build_url,build_dir)
446 if self.run_in_host(build_checkout) != 0:
448 # the repo url is taken from arch-rpms-url
449 # with the last step (i386) removed
450 repo_url = self.options.arch_rpms_url
451 for level in [ 'arch' ]:
452 repo_url = os.path.dirname(repo_url)
453 # pass the vbuild-nightly options to vtest-init-vserver
455 test_env_options += " -p %s"%self.options.personality
456 test_env_options += " -d %s"%self.options.pldistro
457 test_env_options += " -f %s"%self.options.fcdistro
458 script="vtest-init-vserver.sh"
459 vserver_name = self.vservername
460 vserver_options="--netdev eth0 --interface %s"%self.vserverip
462 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
463 vserver_options += " --hostname %s"%vserver_hostname
466 create_vserver="%(build_dir)s/%(script)s %(test_env_options)s %(vserver_name)s %(repo_url)s -- %(vserver_options)s"%locals()
467 return self.run_in_host(create_vserver) == 0
471 "yum install myplc, noderepo, and the plain bootstrapfs"
472 if self.options.personality == "linux32":
474 elif self.options.personality == "linux64":
477 raise Exception, "Unsupported personality %r"%self.options.personality
479 self.run_in_guest("yum -y install myplc")==0 and \
480 self.run_in_guest("yum -y install noderepo-%s-%s"%(self.options.pldistro,arch))==0 and \
481 self.run_in_guest("yum -y install bootstrapfs-%s-%s-plain"%(self.options.pldistro,arch))==0
486 tmpname='%s.plc-config-tty'%(self.name())
487 fileconf=open(tmpname,'w')
488 for var in [ 'PLC_NAME',
492 'PLC_MAIL_SUPPORT_ADDRESS',
495 # Above line was added for integrating SFA Testing
501 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
502 fileconf.write('w\n')
503 fileconf.write('q\n')
505 utils.system('cat %s'%tmpname)
506 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
507 utils.system('rm %s'%tmpname)
512 self.run_in_guest('service plc start')
517 self.run_in_guest('service plc stop')
524 # stores the keys from the config for further use
525 def store_keys(self):
526 "stores test users ssh keys in keys/"
527 for key_spec in self.plc_spec['keys']:
528 TestKey(self,key_spec).store_key()
531 def clean_keys(self):
532 utils.system("rm -rf %s/keys/"%os.path(sys.argv[0]))
534 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
535 # for later direct access to the nodes
536 def fetch_keys(self):
537 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
539 if not os.path.isdir(dir):
541 vservername=self.vservername
543 prefix = 'debug_ssh_key'
544 for ext in [ 'pub', 'rsa' ] :
545 src="/vservers/%(vservername)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
546 dst="keys/%(vservername)s-debug.%(ext)s"%locals()
547 if self.test_ssh.fetch(src,dst) != 0: overall=False
551 "create sites with PLCAPI"
552 return self.do_sites()
554 def clean_sites (self):
555 "delete sites with PLCAPI"
556 return self.do_sites(action="delete")
558 def do_sites (self,action="add"):
559 for site_spec in self.plc_spec['sites']:
560 test_site = TestSite (self,site_spec)
561 if (action != "add"):
562 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
563 test_site.delete_site()
564 # deleted with the site
565 #test_site.delete_users()
568 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
569 test_site.create_site()
570 test_site.create_users()
573 def clean_all_sites (self):
574 print 'auth_root',self.auth_root()
575 site_ids = [s['site_id'] for s in self.apiserver.GetSites(self.auth_root(), {}, ['site_id'])]
576 for site_id in site_ids:
577 print 'Deleting site_id',site_id
578 self.apiserver.DeleteSite(self.auth_root(),site_id)
581 "create nodes with PLCAPI"
582 return self.do_nodes()
583 def clean_nodes (self):
584 "delete nodes with PLCAPI"
585 return self.do_nodes(action="delete")
587 def do_nodes (self,action="add"):
588 for site_spec in self.plc_spec['sites']:
589 test_site = TestSite (self,site_spec)
591 utils.header("Deleting nodes in site %s"%test_site.name())
592 for node_spec in site_spec['nodes']:
593 test_node=TestNode(self,test_site,node_spec)
594 utils.header("Deleting %s"%test_node.name())
595 test_node.delete_node()
597 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
598 for node_spec in site_spec['nodes']:
599 utils.pprint('Creating node %s'%node_spec,node_spec)
600 test_node = TestNode (self,test_site,node_spec)
601 test_node.create_node ()
604 def nodegroups (self):
605 "create nodegroups with PLCAPI"
606 return self.do_nodegroups("add")
607 def clean_nodegroups (self):
608 "delete nodegroups with PLCAPI"
609 return self.do_nodegroups("delete")
611 # create nodegroups if needed, and populate
612 def do_nodegroups (self, action="add"):
613 # 1st pass to scan contents
615 for site_spec in self.plc_spec['sites']:
616 test_site = TestSite (self,site_spec)
617 for node_spec in site_spec['nodes']:
618 test_node=TestNode (self,test_site,node_spec)
619 if node_spec.has_key('nodegroups'):
620 nodegroupnames=node_spec['nodegroups']
621 if isinstance(nodegroupnames,StringTypes):
622 nodegroupnames = [ nodegroupnames ]
623 for nodegroupname in nodegroupnames:
624 if not groups_dict.has_key(nodegroupname):
625 groups_dict[nodegroupname]=[]
626 groups_dict[nodegroupname].append(test_node.name())
627 auth=self.auth_root()
629 for (nodegroupname,group_nodes) in groups_dict.iteritems():
631 print 'nodegroups:','dealing with nodegroup',nodegroupname,'on nodes',group_nodes
632 # first, check if the nodetagtype is here
633 tag_types = self.apiserver.GetTagTypes(auth,{'tagname':nodegroupname})
635 tag_type_id = tag_types[0]['tag_type_id']
637 tag_type_id = self.apiserver.AddTagType(auth,
638 {'tagname':nodegroupname,
639 'description': 'for nodegroup %s'%nodegroupname,
642 print 'located tag (type)',nodegroupname,'as',tag_type_id
644 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
646 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
647 print 'created nodegroup',nodegroupname,'from tagname',nodegroupname,'and value','yes'
648 # set node tag on all nodes, value='yes'
649 for nodename in group_nodes:
651 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
653 traceback.print_exc()
654 print 'node',nodename,'seems to already have tag',nodegroupname
657 expect_yes = self.apiserver.GetNodeTags(auth,
658 {'hostname':nodename,
659 'tagname':nodegroupname},
660 ['value'])[0]['value']
661 if expect_yes != "yes":
662 print 'Mismatch node tag on node',nodename,'got',expect_yes
665 if not self.options.dry_run:
666 print 'Cannot find tag',nodegroupname,'on node',nodename
670 print 'cleaning nodegroup',nodegroupname
671 self.apiserver.DeleteNodeGroup(auth,nodegroupname)
673 traceback.print_exc()
677 def all_hostnames (self) :
679 for site_spec in self.plc_spec['sites']:
680 hostnames += [ node_spec['node_fields']['hostname'] \
681 for node_spec in site_spec['nodes'] ]
684 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
685 def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period=15):
686 if self.options.dry_run:
690 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
691 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
692 # the nodes that haven't checked yet - start with a full list and shrink over time
693 tocheck = self.all_hostnames()
694 utils.header("checking nodes %r"%tocheck)
695 # create a dict hostname -> status
696 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
699 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
701 for array in tocheck_status:
702 hostname=array['hostname']
703 boot_state=array['boot_state']
704 if boot_state == target_boot_state:
705 utils.header ("%s has reached the %s state"%(hostname,target_boot_state))
707 # if it's a real node, never mind
708 (site_spec,node_spec)=self.locate_hostname(hostname)
709 if TestNode.is_real_model(node_spec['node_fields']['model']):
710 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
712 boot_state = target_boot_state
713 elif datetime.datetime.now() > graceout:
714 utils.header ("%s still in '%s' state"%(hostname,boot_state))
715 graceout=datetime.datetime.now()+datetime.timedelta(1)
716 status[hostname] = boot_state
718 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != target_boot_state ]
721 if datetime.datetime.now() > timeout:
722 for hostname in tocheck:
723 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
725 # otherwise, sleep for a while
727 # only useful in empty plcs
730 def nodes_booted(self):
731 return self.nodes_check_boot_state('boot',timeout_minutes=20,silent_minutes=15)
733 def check_nodes_ssh(self,debug,timeout_minutes,silent_minutes,period=20):
735 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
736 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
737 vservername=self.vservername
740 local_key = "keys/%(vservername)s-debug.rsa"%locals()
743 local_key = "keys/key1.rsa"
744 tocheck = self.all_hostnames()
745 utils.header("checking ssh access (expected in %s mode) to nodes %r"%(message,tocheck))
746 utils.header("max timeout is %d minutes, silent for %d minutes (period is %s)"%\
747 (timeout_minutes,silent_minutes,period))
749 for hostname in tocheck:
750 # try to run 'hostname' in the node
751 command = TestSsh (hostname,key=local_key).actual_command("hostname;uname -a")
752 # don't spam logs - show the command only after the grace period
753 if datetime.datetime.now() > graceout:
754 success=utils.system(command)
756 # truly silent, just print out a dot to show we're alive
759 command += " 2>/dev/null"
760 if self.options.dry_run:
761 print 'dry_run',command
764 success=os.system(command)
766 utils.header('Successfully entered root@%s (%s)'%(hostname,message))
768 tocheck.remove(hostname)
770 # we will have tried real nodes once, in case they're up - but if not, just skip
771 (site_spec,node_spec)=self.locate_hostname(hostname)
772 if TestNode.is_real_model(node_spec['node_fields']['model']):
773 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
774 tocheck.remove(hostname)
777 if datetime.datetime.now() > timeout:
778 for hostname in tocheck:
779 utils.header("FAILURE to ssh into %s"%hostname)
781 # otherwise, sleep for a while
783 # only useful in empty plcs
786 def nodes_ssh_debug(self):
787 "Tries to ssh into nodes in debug mode with the debug ssh key"
788 return self.check_nodes_ssh(debug=True,timeout_minutes=30,silent_minutes=10)
790 def nodes_ssh_boot(self):
791 "Tries to ssh into nodes in production mode with the root ssh key"
792 return self.check_nodes_ssh(debug=False,timeout_minutes=30,silent_minutes=10)
795 def init_node (self):
796 "all nodes : init a clean local directory for holding node-dep stuff like iso image..."
800 "all nodes: invoke GetBootMedium and store result locally"
803 def configure_qemu (self):
804 "all nodes: compute qemu config qemu.conf and store it locally"
807 def reinstall_node (self):
808 "all nodes: mark PLCAPI boot_state as reinstall"
811 def export_qemu (self):
812 "all nodes: push local node-dep directory on the qemu box"
815 ### check hooks : invoke scripts from hooks/{node,slice}
816 def check_hooks_node (self):
817 return self.locate_first_node().check_hooks()
818 def check_hooks_sliver (self) :
819 return self.locate_first_sliver().check_hooks()
821 def check_hooks (self):
822 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
823 return self.check_hooks_node() and self.check_hooks_sliver()
826 def do_check_initscripts(self):
828 for slice_spec in self.plc_spec['slices']:
829 if not slice_spec.has_key('initscriptname'):
831 initscript=slice_spec['initscriptname']
832 for nodename in slice_spec['nodenames']:
833 (site,node) = self.locate_node (nodename)
834 # xxx - passing the wrong site - probably harmless
835 test_site = TestSite (self,site)
836 test_slice = TestSlice (self,test_site,slice_spec)
837 test_node = TestNode (self,test_site,node)
838 test_sliver = TestSliver (self, test_node, test_slice)
839 if not test_sliver.check_initscript(initscript):
843 def check_initscripts(self):
844 "check that the initscripts have triggered"
845 return self.do_check_initscripts()
847 def initscripts (self):
848 "create initscripts with PLCAPI"
849 for initscript in self.plc_spec['initscripts']:
850 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
851 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
854 def clean_initscripts (self):
855 "delete initscripts with PLCAPI"
856 for initscript in self.plc_spec['initscripts']:
857 initscript_name = initscript['initscript_fields']['name']
858 print('Attempting to delete %s in plc %s'%(initscript_name,self.plc_spec['name']))
860 self.apiserver.DeleteInitScript(self.auth_root(),initscript_name)
861 print initscript_name,'deleted'
863 print 'deletion went wrong - probably did not exist'
868 "create slices with PLCAPI"
869 return self.do_slices()
871 def clean_slices (self):
872 "delete slices with PLCAPI"
873 return self.do_slices("delete")
875 def do_slices (self, action="add"):
876 for slice in self.plc_spec['slices']:
877 site_spec = self.locate_site (slice['sitename'])
878 test_site = TestSite(self,site_spec)
879 test_slice=TestSlice(self,test_site,slice)
881 utils.header("Deleting slices in site %s"%test_site.name())
882 test_slice.delete_slice()
884 utils.pprint("Creating slice",slice)
885 test_slice.create_slice()
886 utils.header('Created Slice %s'%slice['slice_fields']['name'])
889 @slice_mapper_options
890 def check_slice(self):
891 "tries to ssh-enter the slice with the user key, to ensure slice creation"
895 def clear_known_hosts (self):
896 "remove test nodes entries from the local known_hosts file"
900 def start_node (self) :
901 "all nodes: start the qemu instance (also runs qemu-bridge-init start)"
904 def check_tcp (self):
905 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
906 specs = self.plc_spec['tcp_test']
911 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
912 if not s_test_sliver.run_tcp_server(port,timeout=10):
916 # idem for the client side
917 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
918 if not c_test_sliver.run_tcp_client(s_test_sliver.test_node.name(),port):
922 def plcsh_stress_test (self):
923 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
924 # install the stress-test in the plc image
925 location = "/usr/share/plc_api/plcsh_stress_test.py"
926 remote="/vservers/%s/%s"%(self.vservername,location)
927 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
929 command += " -- --check"
930 if self.options.size == 1:
932 return ( self.run_in_guest(command) == 0)
934 # populate runs the same utility without slightly different options
935 # in particular runs with --preserve (dont cleanup) and without --check
936 # also it gets run twice, once with the --foreign option for creating fake foreign entries
939 def install_sfa(self):
940 "yum install sfa, sfa-plc and sfa-client"
941 if self.options.personality == "linux32":
943 elif self.options.personality == "linux64":
946 raise Exception, "Unsupported personality %r"%self.options.personality
948 self.run_in_guest("yum -y install sfa")==0 and \
949 self.run_in_guest("yum -y install sfa-client")==0 and \
950 self.run_in_guest("yum -y install sfa-plc")==0 and \
951 self.run_in_guest("yum -y install sfa-sfatables")==0
953 def configure_sfa(self):
955 tmpname='%s.sfa-config-tty'%(self.name())
956 fileconf=open(tmpname,'w')
957 fileconf.write ('u\n')
958 for var in [ 'SFA_REGISTRY_ROOT_AUTH',
959 'SFA_REGISTRY_LEVEL1_AUTH',
964 'SFA_PLC_DB_PASSWORD']:
965 fileconf.write ('%s\n'%(self.plc_spec['sfa'][var]))
966 fileconf.write('w\n')
967 fileconf.write('q\n')
969 utils.system('cat %s'%tmpname)
970 self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
971 utils.system('rm %s'%tmpname)
974 def import_sfa(self):
976 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
977 self.run_in_guest('sfa-import-plc.py')
978 self.run_in_guest('cp /etc/sfa/authorities/%s/%s.pkey /etc/sfa/authorities/server.key'%(auth,auth))
983 self.run_in_guest('service sfa start')
987 "sfi client configuration"
989 if os.path.exists(dir_name):
990 utils.system('rm -rf %s'%dir_name)
991 utils.system('mkdir %s'%dir_name)
992 file_name=dir_name + os.sep + 'fake-pi1.pkey'
993 fileconf=open(file_name,'w')
994 fileconf.write (self.plc_spec['keys'][0]['private'])
997 file_name=dir_name + os.sep + 'sfi_config'
998 fileconf=open(file_name,'w')
999 SFI_AUTH=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']+".main"
1000 fileconf.write ("SFI_AUTH='%s'"%SFI_AUTH)
1001 fileconf.write('\n')
1002 SFI_USER=SFI_AUTH+'.fake-pi1'
1003 fileconf.write ("SFI_USER='%s'"%SFI_USER)
1004 fileconf.write('\n')
1005 fileconf.write ("SFI_REGISTRY='http://localhost:12345/'")
1006 fileconf.write('\n')
1007 fileconf.write ("SFI_SM='http://localhost:12347/'")
1008 fileconf.write('\n')
1011 file_name=dir_name + os.sep + 'person.xml'
1012 fileconf=open(file_name,'w')
1013 for record in self.plc_spec['sfa']['sfa_person_xml']:
1014 person_record=record
1015 fileconf.write(person_record)
1016 fileconf.write('\n')
1019 file_name=dir_name + os.sep + 'slice.xml'
1020 fileconf=open(file_name,'w')
1021 for record in self.plc_spec['sfa']['sfa_slice_xml']:
1023 #slice_record=self.plc_spec['sfa']['sfa_slice_xml']
1024 fileconf.write(slice_record)
1025 fileconf.write('\n')
1028 file_name=dir_name + os.sep + 'slice.rspec'
1029 fileconf=open(file_name,'w')
1031 for (key, value) in self.plc_spec['sfa']['sfa_slice_rspec'].items():
1033 fileconf.write(slice_rspec)
1034 fileconf.write('\n')
1037 remote="/vservers/%s/%s"%(self.vservername,location)
1038 self.test_ssh.copy_abs(dir_name, remote, recursive=True)
1040 #utils.system('cat %s'%tmpname)
1041 utils.system('rm -rf %s'%dir_name)
1045 "run sfi.py add (on Registry) and sfi.py create (on SM) to form new objects"
1047 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1048 success=test_user_sfa.add_user()
1050 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1051 site_spec = self.locate_site (slice_spec['sitename'])
1052 test_site = TestSite(self,site_spec)
1053 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1054 success1=test_slice_sfa.add_slice()
1055 success2=test_slice_sfa.create_slice()
1056 return success and success1 and success2
1058 def update_sfa(self):
1059 "run sfi.py update (on Registry) and sfi.py create (on SM) on existing objects"
1061 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1062 success1=test_user_sfa.update_user()
1064 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1065 site_spec = self.locate_site (slice_spec['sitename'])
1066 test_site = TestSite(self,site_spec)
1067 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1068 success2=test_slice_sfa.update_slice()
1069 return success1 and success2
1072 "run sfi.py list and sfi.py show (both on Registry) and sfi.py slices and sfi.py resources (both on SM)"
1073 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1075 self.run_in_guest("sfi.py -d /root/.sfi/ list %s.main"%auth)==0 and \
1076 self.run_in_guest("sfi.py -d /root/.sfi/ show %s.main"%auth)==0 and \
1077 self.run_in_guest("sfi.py -d /root/.sfi/ slices")==0 and \
1078 self.run_in_guest("sfi.py -d /root/.sfi/ resources")==0
1080 @slice_mapper_options_sfa
1081 def check_slice_sfa(self):
1082 "tries to ssh-enter the SFA slice"
1085 def delete_sfa(self):
1086 "run sfi.py delete (on SM), sfi.py remove (on Registry)"
1088 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1089 success1=test_user_sfa.delete_user()
1090 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1091 site_spec = self.locate_site (slice_spec['sitename'])
1092 test_site = TestSite(self,site_spec)
1093 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1094 success2=test_slice_sfa.delete_slice()
1096 return success1 and success2
1100 self.run_in_guest('service sfa stop')
1103 def populate (self):
1104 "creates random entries in the PLCAPI"
1105 # install the stress-test in the plc image
1106 location = "/usr/share/plc_api/plcsh_stress_test.py"
1107 remote="/vservers/%s/%s"%(self.vservername,location)
1108 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1110 command += " -- --preserve --short-names"
1111 local = (self.run_in_guest(command) == 0);
1112 # second run with --foreign
1113 command += ' --foreign'
1114 remote = (self.run_in_guest(command) == 0);
1115 return ( local and remote)
1117 def gather_logs (self):
1118 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1119 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1120 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1121 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1122 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1123 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1125 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1126 self.gather_var_logs ()
1128 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1129 self.gather_pgsql_logs ()
1131 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1132 for site_spec in self.plc_spec['sites']:
1133 test_site = TestSite (self,site_spec)
1134 for node_spec in site_spec['nodes']:
1135 test_node=TestNode(self,test_site,node_spec)
1136 test_node.gather_qemu_logs()
1138 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1139 self.gather_nodes_var_logs()
1141 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1142 self.gather_slivers_var_logs()
1145 def gather_slivers_var_logs(self):
1146 for test_sliver in self.all_sliver_objs():
1147 remote = test_sliver.tar_var_logs()
1148 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
1149 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
1150 utils.system(command)
1153 def gather_var_logs (self):
1154 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
1155 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1156 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
1157 utils.system(command)
1158 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd"%self.name()
1159 utils.system(command)
1161 def gather_pgsql_logs (self):
1162 utils.system("mkdir -p logs/myplc.pgsql-log.%s"%self.name())
1163 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1164 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -"%self.name()
1165 utils.system(command)
1167 def gather_nodes_var_logs (self):
1168 for site_spec in self.plc_spec['sites']:
1169 test_site = TestSite (self,site_spec)
1170 for node_spec in site_spec['nodes']:
1171 test_node=TestNode(self,test_site,node_spec)
1172 test_ssh = TestSsh (test_node.name(),key="keys/key1.rsa")
1173 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1174 command = command + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
1175 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
1176 utils.system(command)
1179 # returns the filename to use for sql dump/restore, using options.dbname if set
1180 def dbfile (self, database):
1181 # uses options.dbname if it is found
1183 name=self.options.dbname
1184 if not isinstance(name,StringTypes):
1187 t=datetime.datetime.now()
1190 return "/root/%s-%s.sql"%(database,name)
1193 dump=self.dbfile("planetab4")
1194 self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
1195 utils.header('Dumped planetlab4 database in %s'%dump)
1198 def db_restore(self):
1199 dump=self.dbfile("planetab4")
1200 ##stop httpd service
1201 self.run_in_guest('service httpd stop')
1202 # xxx - need another wrapper
1203 self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
1204 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
1205 self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
1206 ##starting httpd service
1207 self.run_in_guest('service httpd start')
1209 utils.header('Database restored from ' + dump)
1212 def standby_1(): pass
1214 def standby_2(): pass
1216 def standby_3(): pass
1218 def standby_4(): pass
1220 def standby_5(): pass
1222 def standby_6(): pass
1224 def standby_7(): pass
1226 def standby_8(): pass
1228 def standby_9(): pass
1230 def standby_10(): pass
1232 def standby_11(): pass
1234 def standby_12(): pass
1236 def standby_13(): pass
1238 def standby_14(): pass
1240 def standby_15(): pass
1242 def standby_16(): pass
1244 def standby_17(): pass
1246 def standby_18(): pass
1248 def standby_19(): pass
1250 def standby_20(): pass