7 from types import StringTypes
11 from TestSite import TestSite
12 from TestNode import TestNode
13 from TestUser import TestUser
14 from TestKey import TestKey
15 from TestSlice import TestSlice
16 from TestSliver import TestSliver
17 from TestBox import TestBox
18 from TestSsh import TestSsh
19 from TestApiserver import TestApiserver
20 from TestSliceSfa import TestSliceSfa
21 from TestUserSfa import TestUserSfa
23 # step methods must take (self) and return a boolean (options is a member of the class)
25 def standby(minutes,dry_run):
26 utils.header('Entering StandBy for %d mn'%minutes)
30 time.sleep(60*minutes)
33 def standby_generic (func):
35 minutes=int(func.__name__.split("_")[1])
36 return standby(minutes,self.options.dry_run)
39 def node_mapper (method):
42 node_method = TestNode.__dict__[method.__name__]
43 for site_spec in self.plc_spec['sites']:
44 test_site = TestSite (self,site_spec)
45 for node_spec in site_spec['nodes']:
46 test_node = TestNode (self,test_site,node_spec)
47 if not node_method(test_node): overall=False
49 # restore the doc text
50 actual.__doc__=method.__doc__
53 def slice_mapper_options (method):
56 slice_method = TestSlice.__dict__[method.__name__]
57 for slice_spec in self.plc_spec['slices']:
58 site_spec = self.locate_site (slice_spec['sitename'])
59 test_site = TestSite(self,site_spec)
60 test_slice=TestSlice(self,test_site,slice_spec)
61 if not slice_method(test_slice,self.options): overall=False
63 # restore the doc text
64 actual.__doc__=method.__doc__
67 def slice_mapper_options_sfa (method):
71 slice_method = TestSliceSfa.__dict__[method.__name__]
72 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
73 site_spec = self.locate_site (slice_spec['sitename'])
74 test_site = TestSite(self,site_spec)
75 test_slice=TestSliceSfa(test_plc,test_site,slice_spec)
76 if not slice_method(test_slice,self.options): overall=False
78 # restore the doc text
79 actual.__doc__=method.__doc__
87 'display', 'local_pre', SEP,
88 'delete','create','install', 'configure', 'start', SEP,
89 'fetch_keys', 'store_keys', 'clear_known_hosts', SEP,
90 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', SEP,
91 'reinstall_node', 'init_node','bootcd', 'configure_qemu', 'export_qemu',
92 'kill_all_qemus', 'start_node', SEP,
93 # better use of time: do this now that the nodes are taking off
94 'plcsh_stress_test', SEP,
95 'nodes_ssh_debug', 'nodes_ssh_boot', 'check_slice', 'check_initscripts', SEP,
96 'install_sfa', 'configure_sfa', 'import_sfa', 'start_sfa', SEP,
97 'setup_sfa', 'add_sfa', 'update_sfa', SEP,
98 'view_sfa', 'check_slice_sfa', 'delete_sfa', 'stop_sfa', SEP,
99 'check_tcp', 'check_hooks', SEP,
100 'force_gather_logs', 'force_local_post',
103 'fresh_install', 'stop', 'vs_start', SEP,
104 'clean_initscripts', 'clean_nodegroups','clean_all_sites', SEP,
105 'clean_sites', 'clean_nodes', 'clean_slices', 'clean_keys', SEP,
107 'show_boxes', 'list_all_qemus', 'list_qemus', 'kill_qemus', SEP,
108 'db_dump' , 'db_restore', SEP,
109 'local_list','local_cleanup',SEP,
110 'standby_1 through 20',
114 def printable_steps (list):
115 return " ".join(list).replace(" "+SEP+" "," \\\n")
117 def valid_step (step):
120 def __init__ (self,plc_spec,options):
121 self.plc_spec=plc_spec
123 self.test_ssh=TestSsh(self.plc_spec['hostname'],self.options.buildname)
125 self.vserverip=plc_spec['vserverip']
126 self.vservername=plc_spec['vservername']
127 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
130 raise Exception,'chroot-based myplc testing is deprecated'
131 self.apiserver=TestApiserver(self.url,options.dry_run)
134 name=self.plc_spec['name']
135 return "%s.%s"%(name,self.vservername)
138 return self.plc_spec['hostname']
141 return self.test_ssh.is_local()
143 # define the API methods on this object through xmlrpc
144 # would help, but not strictly necessary
148 def actual_command_in_guest (self,command):
149 return self.test_ssh.actual_command(self.host_to_guest(command))
151 def start_guest (self):
152 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host()))
154 def run_in_guest (self,command):
155 return utils.system(self.actual_command_in_guest(command))
157 def run_in_host (self,command):
158 return self.test_ssh.run_in_buildname(command)
160 #command gets run in the vserver
161 def host_to_guest(self,command):
162 return "vserver %s exec %s"%(self.vservername,command)
164 #command gets run in the vserver
165 def start_guest_in_host(self):
166 return "vserver %s start"%(self.vservername)
169 def run_in_guest_piped (self,local,remote):
170 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
172 def auth_root (self):
173 return {'Username':self.plc_spec['PLC_ROOT_USER'],
174 'AuthMethod':'password',
175 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
176 'Role' : self.plc_spec['role']
178 def locate_site (self,sitename):
179 for site in self.plc_spec['sites']:
180 if site['site_fields']['name'] == sitename:
182 if site['site_fields']['login_base'] == sitename:
184 raise Exception,"Cannot locate site %s"%sitename
186 def locate_node (self,nodename):
187 for site in self.plc_spec['sites']:
188 for node in site['nodes']:
189 if node['name'] == nodename:
191 raise Exception,"Cannot locate node %s"%nodename
193 def locate_hostname (self,hostname):
194 for site in self.plc_spec['sites']:
195 for node in site['nodes']:
196 if node['node_fields']['hostname'] == hostname:
198 raise Exception,"Cannot locate hostname %s"%hostname
200 def locate_key (self,keyname):
201 for key in self.plc_spec['keys']:
202 if key['name'] == keyname:
204 raise Exception,"Cannot locate key %s"%keyname
206 def locate_slice (self, slicename):
207 for slice in self.plc_spec['slices']:
208 if slice['slice_fields']['name'] == slicename:
210 raise Exception,"Cannot locate slice %s"%slicename
212 def all_sliver_objs (self):
214 for slice_spec in self.plc_spec['slices']:
215 slicename = slice_spec['slice_fields']['name']
216 for nodename in slice_spec['nodenames']:
217 result.append(self.locate_sliver_obj (nodename,slicename))
220 def locate_sliver_obj (self,nodename,slicename):
221 (site,node) = self.locate_node(nodename)
222 slice = self.locate_slice (slicename)
224 test_site = TestSite (self, site)
225 test_node = TestNode (self, test_site,node)
226 # xxx the slice site is assumed to be the node site - mhh - probably harmless
227 test_slice = TestSlice (self, test_site, slice)
228 return TestSliver (self, test_node, test_slice)
230 def locate_first_node(self):
231 nodename=self.plc_spec['slices'][0]['nodenames'][0]
232 (site,node) = self.locate_node(nodename)
233 test_site = TestSite (self, site)
234 test_node = TestNode (self, test_site,node)
237 def locate_first_sliver (self):
238 slice_spec=self.plc_spec['slices'][0]
239 slicename=slice_spec['slice_fields']['name']
240 nodename=slice_spec['nodenames'][0]
241 return self.locate_sliver_obj(nodename,slicename)
243 # all different hostboxes used in this plc
244 def gather_hostBoxes(self):
245 # maps on sites and nodes, return [ (host_box,test_node) ]
247 for site_spec in self.plc_spec['sites']:
248 test_site = TestSite (self,site_spec)
249 for node_spec in site_spec['nodes']:
250 test_node = TestNode (self, test_site, node_spec)
251 if not test_node.is_real():
252 tuples.append( (test_node.host_box(),test_node) )
253 # transform into a dict { 'host_box' -> [ test_node .. ] }
255 for (box,node) in tuples:
256 if not result.has_key(box):
259 result[box].append(node)
262 # a step for checking this stuff
263 def show_boxes (self):
264 for (box,nodes) in self.gather_hostBoxes().iteritems():
265 print box,":"," + ".join( [ node.name() for node in nodes ] )
268 # make this a valid step
269 def kill_all_qemus(self):
270 "all qemu boxes: kill all running qemus (even of former runs)"
271 # this is the brute force version, kill all qemus on that host box
272 for (box,nodes) in self.gather_hostBoxes().iteritems():
273 # pass the first nodename, as we don't push template-qemu on testboxes
274 nodedir=nodes[0].nodedir()
275 TestBox(box,self.options.buildname).kill_all_qemus(nodedir)
278 # make this a valid step
279 def list_all_qemus(self):
280 for (box,nodes) in self.gather_hostBoxes().iteritems():
281 # this is the brute force version, kill all qemus on that host box
282 TestBox(box,self.options.buildname).list_all_qemus()
285 # kill only the right qemus
286 def list_qemus(self):
287 for (box,nodes) in self.gather_hostBoxes().iteritems():
288 # the fine-grain version
293 # kill only the right qemus
294 def kill_qemus(self):
295 for (box,nodes) in self.gather_hostBoxes().iteritems():
296 # the fine-grain version
301 #################### display config
303 "show test configuration after localization"
304 self.display_pass (1)
305 self.display_pass (2)
309 def display_pass (self,passno):
310 for (key,val) in self.plc_spec.iteritems():
314 self.display_site_spec(site)
315 for node in site['nodes']:
316 self.display_node_spec(node)
317 elif key=='initscripts':
318 for initscript in val:
319 self.display_initscript_spec (initscript)
322 self.display_slice_spec (slice)
325 self.display_key_spec (key)
327 if key not in ['sites','initscripts','slices','keys']:
328 print '+ ',key,':',val
330 def display_site_spec (self,site):
331 print '+ ======== site',site['site_fields']['name']
332 for (k,v) in site.iteritems():
335 print '+ ','nodes : ',
337 print node['node_fields']['hostname'],'',
343 print user['name'],'',
345 elif k == 'site_fields':
346 print '+ login_base',':',v['login_base']
347 elif k == 'address_fields':
351 PrettyPrinter(indent=8,depth=2).pprint(v)
353 def display_initscript_spec (self,initscript):
354 print '+ ======== initscript',initscript['initscript_fields']['name']
356 def display_key_spec (self,key):
357 print '+ ======== key',key['name']
359 def display_slice_spec (self,slice):
360 print '+ ======== slice',slice['slice_fields']['name']
361 for (k,v) in slice.iteritems():
374 elif k=='slice_fields':
375 print '+ fields',':',
376 print 'max_nodes=',v['max_nodes'],
381 def display_node_spec (self,node):
382 print "+ node",node['name'],"host_box=",node['host_box'],
383 print "hostname=",node['node_fields']['hostname'],
384 print "ip=",node['interface_fields']['ip']
387 # another entry point for just showing the boxes involved
388 def display_mapping (self):
389 TestPlc.display_mapping_plc(self.plc_spec)
393 def display_mapping_plc (plc_spec):
394 print '+ MyPLC',plc_spec['name']
395 print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['hostname'],plc_spec['vservername'])
396 print '+\tIP = %s/%s'%(plc_spec['PLC_API_HOST'],plc_spec['vserverip'])
397 for site_spec in plc_spec['sites']:
398 for node_spec in site_spec['nodes']:
399 TestPlc.display_mapping_node(node_spec)
402 def display_mapping_node (node_spec):
403 print '+ NODE %s'%(node_spec['name'])
404 print '+\tqemu box %s'%node_spec['host_box']
405 print '+\thostname=%s'%node_spec['node_fields']['hostname']
407 def local_pre (self):
408 "run site-dependant pre-test script as defined in LocalTestResources"
409 from LocalTestResources import local_resources
410 return local_resources.step_pre(self)
412 def local_post (self):
413 "run site-dependant post-test script as defined in LocalTestResources"
414 from LocalTestResources import local_resources
415 return local_resources.step_post(self)
417 def local_list (self):
418 "run site-dependant list script as defined in LocalTestResources"
419 from LocalTestResources import local_resources
420 return local_resources.step_list(self)
422 def local_cleanup (self):
423 "run site-dependant cleanup script as defined in LocalTestResources"
424 from LocalTestResources import local_resources
425 return local_resources.step_cleanup(self)
428 "vserver delete the test myplc"
429 self.run_in_host("vserver --silent %s delete"%self.vservername)
434 "vserver creation (no install done)"
436 # a full path for the local calls
437 build_dir=os.path.dirname(sys.argv[0])
438 # sometimes this is empty - set to "." in such a case
439 if not build_dir: build_dir="."
440 build_dir += "/build"
442 # use a standard name - will be relative to remote buildname
444 # run checkout in any case - would do an update if already exists
445 build_checkout = "svn checkout %s %s"%(self.options.build_url,build_dir)
446 if self.run_in_host(build_checkout) != 0:
448 # the repo url is taken from arch-rpms-url
449 # with the last step (i386) removed
450 repo_url = self.options.arch_rpms_url
451 for level in [ 'arch' ]:
452 repo_url = os.path.dirname(repo_url)
453 # pass the vbuild-nightly options to vtest-init-vserver
455 test_env_options += " -p %s"%self.options.personality
456 test_env_options += " -d %s"%self.options.pldistro
457 test_env_options += " -f %s"%self.options.fcdistro
458 script="vtest-init-vserver.sh"
459 vserver_name = self.vservername
460 vserver_options="--netdev eth0 --interface %s"%self.vserverip
462 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
463 vserver_options += " --hostname %s"%vserver_hostname
466 create_vserver="%(build_dir)s/%(script)s %(test_env_options)s %(vserver_name)s %(repo_url)s -- %(vserver_options)s"%locals()
467 return self.run_in_host(create_vserver) == 0
471 "yum install myplc, noderepo, and the plain bootstrapfs"
472 if self.options.personality == "linux32":
474 elif self.options.personality == "linux64":
477 raise Exception, "Unsupported personality %r"%self.options.personality
479 self.run_in_guest("yum -y install myplc")==0 and \
480 self.run_in_guest("yum -y install noderepo-%s-%s"%(self.options.pldistro,arch))==0 and \
481 self.run_in_guest("yum -y install bootstrapfs-%s-%s-plain"%(self.options.pldistro,arch))==0
486 tmpname='%s.plc-config-tty'%(self.name())
487 fileconf=open(tmpname,'w')
488 for var in [ 'PLC_NAME',
492 'PLC_MAIL_SUPPORT_ADDRESS',
495 # Above line was added for integrating SFA Testing
501 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
502 fileconf.write('w\n')
503 fileconf.write('q\n')
505 utils.system('cat %s'%tmpname)
506 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
507 utils.system('rm %s'%tmpname)
512 self.run_in_guest('service plc start')
517 self.run_in_guest('service plc stop')
524 # stores the keys from the config for further use
525 def store_keys(self):
526 "stores test users ssh keys in keys/"
527 for key_spec in self.plc_spec['keys']:
528 TestKey(self,key_spec).store_key()
531 def clean_keys(self):
532 utils.system("rm -rf %s/keys/"%os.path(sys.argv[0]))
534 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
535 # for later direct access to the nodes
536 def fetch_keys(self):
537 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
539 if not os.path.isdir(dir):
541 vservername=self.vservername
543 prefix = 'root_ssh_key'
544 for ext in [ 'pub', 'rsa' ] :
545 src="/vservers/%(vservername)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
546 dst="keys/%(vservername)s.%(ext)s"%locals()
547 if self.test_ssh.fetch(src,dst) != 0: overall=False
548 prefix = 'debug_ssh_key'
549 for ext in [ 'pub', 'rsa' ] :
550 src="/vservers/%(vservername)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
551 dst="keys/%(vservername)s-debug.%(ext)s"%locals()
552 if self.test_ssh.fetch(src,dst) != 0: overall=False
556 "create sites with PLCAPI"
557 return self.do_sites()
559 def clean_sites (self):
560 "delete sites with PLCAPI"
561 return self.do_sites(action="delete")
563 def do_sites (self,action="add"):
564 for site_spec in self.plc_spec['sites']:
565 test_site = TestSite (self,site_spec)
566 if (action != "add"):
567 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
568 test_site.delete_site()
569 # deleted with the site
570 #test_site.delete_users()
573 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
574 test_site.create_site()
575 test_site.create_users()
578 def clean_all_sites (self):
579 print 'auth_root',self.auth_root()
580 site_ids = [s['site_id'] for s in self.apiserver.GetSites(self.auth_root(), {}, ['site_id'])]
581 for site_id in site_ids:
582 print 'Deleting site_id',site_id
583 self.apiserver.DeleteSite(self.auth_root(),site_id)
586 "create nodes with PLCAPI"
587 return self.do_nodes()
588 def clean_nodes (self):
589 "delete nodes with PLCAPI"
590 return self.do_nodes(action="delete")
592 def do_nodes (self,action="add"):
593 for site_spec in self.plc_spec['sites']:
594 test_site = TestSite (self,site_spec)
596 utils.header("Deleting nodes in site %s"%test_site.name())
597 for node_spec in site_spec['nodes']:
598 test_node=TestNode(self,test_site,node_spec)
599 utils.header("Deleting %s"%test_node.name())
600 test_node.delete_node()
602 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
603 for node_spec in site_spec['nodes']:
604 utils.pprint('Creating node %s'%node_spec,node_spec)
605 test_node = TestNode (self,test_site,node_spec)
606 test_node.create_node ()
609 def nodegroups (self):
610 "create nodegroups with PLCAPI"
611 return self.do_nodegroups("add")
612 def clean_nodegroups (self):
613 "delete nodegroups with PLCAPI"
614 return self.do_nodegroups("delete")
616 # create nodegroups if needed, and populate
617 def do_nodegroups (self, action="add"):
618 # 1st pass to scan contents
620 for site_spec in self.plc_spec['sites']:
621 test_site = TestSite (self,site_spec)
622 for node_spec in site_spec['nodes']:
623 test_node=TestNode (self,test_site,node_spec)
624 if node_spec.has_key('nodegroups'):
625 nodegroupnames=node_spec['nodegroups']
626 if isinstance(nodegroupnames,StringTypes):
627 nodegroupnames = [ nodegroupnames ]
628 for nodegroupname in nodegroupnames:
629 if not groups_dict.has_key(nodegroupname):
630 groups_dict[nodegroupname]=[]
631 groups_dict[nodegroupname].append(test_node.name())
632 auth=self.auth_root()
634 for (nodegroupname,group_nodes) in groups_dict.iteritems():
636 print 'nodegroups:','dealing with nodegroup',nodegroupname,'on nodes',group_nodes
637 # first, check if the nodetagtype is here
638 tag_types = self.apiserver.GetTagTypes(auth,{'tagname':nodegroupname})
640 tag_type_id = tag_types[0]['tag_type_id']
642 tag_type_id = self.apiserver.AddTagType(auth,
643 {'tagname':nodegroupname,
644 'description': 'for nodegroup %s'%nodegroupname,
647 print 'located tag (type)',nodegroupname,'as',tag_type_id
649 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
651 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
652 print 'created nodegroup',nodegroupname,'from tagname',nodegroupname,'and value','yes'
653 # set node tag on all nodes, value='yes'
654 for nodename in group_nodes:
656 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
658 traceback.print_exc()
659 print 'node',nodename,'seems to already have tag',nodegroupname
662 expect_yes = self.apiserver.GetNodeTags(auth,
663 {'hostname':nodename,
664 'tagname':nodegroupname},
665 ['value'])[0]['value']
666 if expect_yes != "yes":
667 print 'Mismatch node tag on node',nodename,'got',expect_yes
670 if not self.options.dry_run:
671 print 'Cannot find tag',nodegroupname,'on node',nodename
675 print 'cleaning nodegroup',nodegroupname
676 self.apiserver.DeleteNodeGroup(auth,nodegroupname)
678 traceback.print_exc()
682 def all_hostnames (self) :
684 for site_spec in self.plc_spec['sites']:
685 hostnames += [ node_spec['node_fields']['hostname'] \
686 for node_spec in site_spec['nodes'] ]
689 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
690 def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period=15):
691 if self.options.dry_run:
695 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
696 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
697 # the nodes that haven't checked yet - start with a full list and shrink over time
698 tocheck = self.all_hostnames()
699 utils.header("checking nodes %r"%tocheck)
700 # create a dict hostname -> status
701 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
704 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
706 for array in tocheck_status:
707 hostname=array['hostname']
708 boot_state=array['boot_state']
709 if boot_state == target_boot_state:
710 utils.header ("%s has reached the %s state"%(hostname,target_boot_state))
712 # if it's a real node, never mind
713 (site_spec,node_spec)=self.locate_hostname(hostname)
714 if TestNode.is_real_model(node_spec['node_fields']['model']):
715 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
717 boot_state = target_boot_state
718 elif datetime.datetime.now() > graceout:
719 utils.header ("%s still in '%s' state"%(hostname,boot_state))
720 graceout=datetime.datetime.now()+datetime.timedelta(1)
721 status[hostname] = boot_state
723 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != target_boot_state ]
726 if datetime.datetime.now() > timeout:
727 for hostname in tocheck:
728 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
730 # otherwise, sleep for a while
732 # only useful in empty plcs
735 def nodes_booted(self):
736 return self.nodes_check_boot_state('boot',timeout_minutes=20,silent_minutes=15)
738 def check_nodes_ssh(self,debug,timeout_minutes,silent_minutes,period=20):
740 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
741 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
742 vservername=self.vservername
745 local_key = "keys/%(vservername)s-debug.rsa"%locals()
748 local_key = "keys/%(vservername)s.rsa"%locals()
749 tocheck = self.all_hostnames()
750 utils.header("checking ssh access (expected in %s mode) to nodes %r"%(message,tocheck))
751 utils.header("max timeout is %d minutes, silent for %d minutes (period is %s)"%\
752 (timeout_minutes,silent_minutes,period))
754 for hostname in tocheck:
755 # try to run 'hostname' in the node
756 command = TestSsh (hostname,key=local_key).actual_command("hostname;uname -a")
757 # don't spam logs - show the command only after the grace period
758 if datetime.datetime.now() > graceout:
759 success=utils.system(command)
761 # truly silent, just print out a dot to show we're alive
764 command += " 2>/dev/null"
765 if self.options.dry_run:
766 print 'dry_run',command
769 success=os.system(command)
771 utils.header('Successfully entered root@%s (%s)'%(hostname,message))
773 tocheck.remove(hostname)
775 # we will have tried real nodes once, in case they're up - but if not, just skip
776 (site_spec,node_spec)=self.locate_hostname(hostname)
777 if TestNode.is_real_model(node_spec['node_fields']['model']):
778 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
779 tocheck.remove(hostname)
782 if datetime.datetime.now() > timeout:
783 for hostname in tocheck:
784 utils.header("FAILURE to ssh into %s"%hostname)
786 # otherwise, sleep for a while
788 # only useful in empty plcs
791 def nodes_ssh_debug(self):
792 "Tries to ssh into nodes in debug mode with the debug ssh key"
793 return self.check_nodes_ssh(debug=True,timeout_minutes=30,silent_minutes=10)
795 def nodes_ssh_boot(self):
796 "Tries to ssh into nodes in production mode with the root ssh key"
797 return self.check_nodes_ssh(debug=False,timeout_minutes=30,silent_minutes=10)
800 def init_node (self):
801 "all nodes : init a clean local directory for holding node-dep stuff like iso image..."
805 "all nodes: invoke GetBootMedium and store result locally"
808 def configure_qemu (self):
809 "all nodes: compute qemu config qemu.conf and store it locally"
812 def reinstall_node (self):
813 "all nodes: mark PLCAPI boot_state as reinstall"
816 def export_qemu (self):
817 "all nodes: push local node-dep directory on the qemu box"
820 ### check hooks : invoke scripts from hooks/{node,slice}
821 def check_hooks_node (self):
822 return self.locate_first_node().check_hooks()
823 def check_hooks_sliver (self) :
824 return self.locate_first_sliver().check_hooks()
826 def check_hooks (self):
827 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
828 return self.check_hooks_node() and self.check_hooks_sliver()
831 def do_check_initscripts(self):
833 for slice_spec in self.plc_spec['slices']:
834 if not slice_spec.has_key('initscriptname'):
836 initscript=slice_spec['initscriptname']
837 for nodename in slice_spec['nodenames']:
838 (site,node) = self.locate_node (nodename)
839 # xxx - passing the wrong site - probably harmless
840 test_site = TestSite (self,site)
841 test_slice = TestSlice (self,test_site,slice_spec)
842 test_node = TestNode (self,test_site,node)
843 test_sliver = TestSliver (self, test_node, test_slice)
844 if not test_sliver.check_initscript(initscript):
848 def check_initscripts(self):
849 "check that the initscripts have triggered"
850 return self.do_check_initscripts()
852 def initscripts (self):
853 "create initscripts with PLCAPI"
854 for initscript in self.plc_spec['initscripts']:
855 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
856 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
859 def clean_initscripts (self):
860 "delete initscripts with PLCAPI"
861 for initscript in self.plc_spec['initscripts']:
862 initscript_name = initscript['initscript_fields']['name']
863 print('Attempting to delete %s in plc %s'%(initscript_name,self.plc_spec['name']))
865 self.apiserver.DeleteInitScript(self.auth_root(),initscript_name)
866 print initscript_name,'deleted'
868 print 'deletion went wrong - probably did not exist'
873 "create slices with PLCAPI"
874 return self.do_slices()
876 def clean_slices (self):
877 "delete slices with PLCAPI"
878 return self.do_slices("delete")
880 def do_slices (self, action="add"):
881 for slice in self.plc_spec['slices']:
882 site_spec = self.locate_site (slice['sitename'])
883 test_site = TestSite(self,site_spec)
884 test_slice=TestSlice(self,test_site,slice)
886 utils.header("Deleting slices in site %s"%test_site.name())
887 test_slice.delete_slice()
889 utils.pprint("Creating slice",slice)
890 test_slice.create_slice()
891 utils.header('Created Slice %s'%slice['slice_fields']['name'])
894 @slice_mapper_options
895 def check_slice(self):
896 "tries to ssh-enter the slice with the user key, to ensure slice creation"
900 def clear_known_hosts (self):
901 "remove test nodes entries from the local known_hosts file"
905 def start_node (self) :
906 "all nodes: start the qemu instance (also runs qemu-bridge-init start)"
909 def check_tcp (self):
910 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
911 specs = self.plc_spec['tcp_test']
916 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
917 if not s_test_sliver.run_tcp_server(port,timeout=10):
921 # idem for the client side
922 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
923 if not c_test_sliver.run_tcp_client(s_test_sliver.test_node.name(),port):
927 def plcsh_stress_test (self):
928 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
929 # install the stress-test in the plc image
930 location = "/usr/share/plc_api/plcsh_stress_test.py"
931 remote="/vservers/%s/%s"%(self.vservername,location)
932 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
934 command += " -- --check"
935 if self.options.size == 1:
937 return ( self.run_in_guest(command) == 0)
939 # populate runs the same utility without slightly different options
940 # in particular runs with --preserve (dont cleanup) and without --check
941 # also it gets run twice, once with the --foreign option for creating fake foreign entries
944 def install_sfa(self):
945 "yum install sfa, sfa-plc and sfa-client"
946 if self.options.personality == "linux32":
948 elif self.options.personality == "linux64":
951 raise Exception, "Unsupported personality %r"%self.options.personality
953 self.run_in_guest("yum -y install sfa")==0 and \
954 self.run_in_guest("yum -y install sfa-client")==0 and \
955 self.run_in_guest("yum -y install sfa-plc")==0 and \
956 self.run_in_guest("yum -y install sfa-sfatables")==0
958 def configure_sfa(self):
960 tmpname='%s.sfa-config-tty'%(self.name())
961 fileconf=open(tmpname,'w')
962 fileconf.write ('u\n')
963 for var in [ 'SFA_REGISTRY_ROOT_AUTH',
964 'SFA_REGISTRY_LEVEL1_AUTH',
969 'SFA_PLC_DB_PASSWORD']:
970 fileconf.write ('%s\n'%(self.plc_spec['sfa'][var]))
971 fileconf.write('w\n')
972 fileconf.write('q\n')
974 utils.system('cat %s'%tmpname)
975 self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
976 utils.system('rm %s'%tmpname)
979 def import_sfa(self):
981 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
982 self.run_in_guest('sfa-import-plc.py')
983 self.run_in_guest('cp /etc/sfa/authorities/%s/%s.pkey /etc/sfa/authorities/server.key'%(auth,auth))
988 self.run_in_guest('service sfa start')
992 "sfi client configuration"
994 if os.path.exists(dir_name):
995 utils.system('rm -rf %s'%dir_name)
996 utils.system('mkdir %s'%dir_name)
997 file_name=dir_name + os.sep + 'fake-pi1.pkey'
998 fileconf=open(file_name,'w')
999 fileconf.write (self.plc_spec['keys'][0]['private'])
1002 file_name=dir_name + os.sep + 'sfi_config'
1003 fileconf=open(file_name,'w')
1004 SFI_AUTH=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']+".main"
1005 fileconf.write ("SFI_AUTH='%s'"%SFI_AUTH)
1006 fileconf.write('\n')
1007 SFI_USER=SFI_AUTH+'.fake-pi1'
1008 fileconf.write ("SFI_USER='%s'"%SFI_USER)
1009 fileconf.write('\n')
1010 fileconf.write ("SFI_REGISTRY='http://localhost:12345/'")
1011 fileconf.write('\n')
1012 fileconf.write ("SFI_SM='http://localhost:12347/'")
1013 fileconf.write('\n')
1016 file_name=dir_name + os.sep + 'person.xml'
1017 fileconf=open(file_name,'w')
1018 for record in self.plc_spec['sfa']['sfa_person_xml']:
1019 person_record=record
1020 fileconf.write(person_record)
1021 fileconf.write('\n')
1024 file_name=dir_name + os.sep + 'slice.xml'
1025 fileconf=open(file_name,'w')
1026 for record in self.plc_spec['sfa']['sfa_slice_xml']:
1028 #slice_record=self.plc_spec['sfa']['sfa_slice_xml']
1029 fileconf.write(slice_record)
1030 fileconf.write('\n')
1033 file_name=dir_name + os.sep + 'slice.rspec'
1034 fileconf=open(file_name,'w')
1036 for (key, value) in self.plc_spec['sfa']['sfa_slice_rspec'].items():
1038 fileconf.write(slice_rspec)
1039 fileconf.write('\n')
1042 remote="/vservers/%s/%s"%(self.vservername,location)
1043 self.test_ssh.copy_abs(dir_name, remote, recursive=True)
1045 #utils.system('cat %s'%tmpname)
1046 utils.system('rm -rf %s'%dir_name)
1050 "run sfi.py add (on Registry) and sfi.py create (on SM) to form new objects"
1052 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1053 success=test_user_sfa.add_user()
1055 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1056 site_spec = self.locate_site (slice_spec['sitename'])
1057 test_site = TestSite(self,site_spec)
1058 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1059 success1=test_slice_sfa.add_slice()
1060 success2=test_slice_sfa.create_slice()
1061 return success and success1 and success2
1063 def update_sfa(self):
1064 "run sfi.py update (on Registry) and sfi.py create (on SM) on existing objects"
1066 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1067 success1=test_user_sfa.update_user()
1069 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1070 site_spec = self.locate_site (slice_spec['sitename'])
1071 test_site = TestSite(self,site_spec)
1072 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1073 success2=test_slice_sfa.update_slice()
1074 return success1 and success2
1077 "run sfi.py list and sfi.py show (both on Registry) and sfi.py slices and sfi.py resources (both on SM)"
1078 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1080 self.run_in_guest("sfi.py -d /root/.sfi/ list %s.main"%auth)==0 and \
1081 self.run_in_guest("sfi.py -d /root/.sfi/ show %s.main"%auth)==0 and \
1082 self.run_in_guest("sfi.py -d /root/.sfi/ slices")==0 and \
1083 self.run_in_guest("sfi.py -d /root/.sfi/ resources")==0
1085 @slice_mapper_options_sfa
1086 def check_slice_sfa(self):
1087 "tries to ssh-enter the SFA slice"
1090 def delete_sfa(self):
1091 "run sfi.py delete (on SM), sfi.py remove (on Registry)"
1093 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1094 success1=test_user_sfa.delete_user()
1095 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1096 site_spec = self.locate_site (slice_spec['sitename'])
1097 test_site = TestSite(self,site_spec)
1098 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1099 success2=test_slice_sfa.delete_slice()
1101 return success1 and success2
1105 self.run_in_guest('service sfa stop')
1108 def populate (self):
1109 "creates random entries in the PLCAPI"
1110 # install the stress-test in the plc image
1111 location = "/usr/share/plc_api/plcsh_stress_test.py"
1112 remote="/vservers/%s/%s"%(self.vservername,location)
1113 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1115 command += " -- --preserve --short-names"
1116 local = (self.run_in_guest(command) == 0);
1117 # second run with --foreign
1118 command += ' --foreign'
1119 remote = (self.run_in_guest(command) == 0);
1120 return ( local and remote)
1122 def gather_logs (self):
1123 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1124 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1125 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1126 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1127 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1128 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1130 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1131 self.gather_var_logs ()
1133 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1134 self.gather_pgsql_logs ()
1136 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1137 for site_spec in self.plc_spec['sites']:
1138 test_site = TestSite (self,site_spec)
1139 for node_spec in site_spec['nodes']:
1140 test_node=TestNode(self,test_site,node_spec)
1141 test_node.gather_qemu_logs()
1143 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1144 self.gather_nodes_var_logs()
1146 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1147 self.gather_slivers_var_logs()
1150 def gather_slivers_var_logs(self):
1151 for test_sliver in self.all_sliver_objs():
1152 remote = test_sliver.tar_var_logs()
1153 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
1154 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
1155 utils.system(command)
1158 def gather_var_logs (self):
1159 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
1160 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1161 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
1162 utils.system(command)
1163 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd"%self.name()
1164 utils.system(command)
1166 def gather_pgsql_logs (self):
1167 utils.system("mkdir -p logs/myplc.pgsql-log.%s"%self.name())
1168 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1169 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -"%self.name()
1170 utils.system(command)
1172 def gather_nodes_var_logs (self):
1173 for site_spec in self.plc_spec['sites']:
1174 test_site = TestSite (self,site_spec)
1175 for node_spec in site_spec['nodes']:
1176 test_node=TestNode(self,test_site,node_spec)
1177 test_ssh = TestSsh (test_node.name(),key="/etc/planetlab/root_ssh_key.rsa")
1178 to_plc = self.actual_command_in_guest ( test_ssh.actual_command("tar -C /var/log -cf - ."))
1179 command = to_plc + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
1180 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
1181 utils.system(command)
1184 # returns the filename to use for sql dump/restore, using options.dbname if set
1185 def dbfile (self, database):
1186 # uses options.dbname if it is found
1188 name=self.options.dbname
1189 if not isinstance(name,StringTypes):
1192 t=datetime.datetime.now()
1195 return "/root/%s-%s.sql"%(database,name)
1198 dump=self.dbfile("planetab4")
1199 self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
1200 utils.header('Dumped planetlab4 database in %s'%dump)
1203 def db_restore(self):
1204 dump=self.dbfile("planetab4")
1205 ##stop httpd service
1206 self.run_in_guest('service httpd stop')
1207 # xxx - need another wrapper
1208 self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
1209 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
1210 self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
1211 ##starting httpd service
1212 self.run_in_guest('service httpd start')
1214 utils.header('Database restored from ' + dump)
1217 def standby_1(): pass
1219 def standby_2(): pass
1221 def standby_3(): pass
1223 def standby_4(): pass
1225 def standby_5(): pass
1227 def standby_6(): pass
1229 def standby_7(): pass
1231 def standby_8(): pass
1233 def standby_9(): pass
1235 def standby_10(): pass
1237 def standby_11(): pass
1239 def standby_12(): pass
1241 def standby_13(): pass
1243 def standby_14(): pass
1245 def standby_15(): pass
1247 def standby_16(): pass
1249 def standby_17(): pass
1251 def standby_18(): pass
1253 def standby_19(): pass
1255 def standby_20(): pass