7 from types import StringTypes
11 from TestSite import TestSite
12 from TestNode import TestNode
13 from TestUser import TestUser
14 from TestKey import TestKey
15 from TestSlice import TestSlice
16 from TestSliver import TestSliver
17 from TestBox import TestBox
18 from TestSsh import TestSsh
19 from TestApiserver import TestApiserver
20 from TestSliceSfa import TestSliceSfa
21 from TestUserSfa import TestUserSfa
23 # step methods must take (self) and return a boolean (options is a member of the class)
25 def standby(minutes,dry_run):
26 utils.header('Entering StandBy for %d mn'%minutes)
30 time.sleep(60*minutes)
33 def standby_generic (func):
35 minutes=int(func.__name__.split("_")[1])
36 return standby(minutes,self.options.dry_run)
39 def node_mapper (method):
42 node_method = TestNode.__dict__[method.__name__]
43 for site_spec in self.plc_spec['sites']:
44 test_site = TestSite (self,site_spec)
45 for node_spec in site_spec['nodes']:
46 test_node = TestNode (self,test_site,node_spec)
47 if not node_method(test_node): overall=False
49 # restore the doc text
50 actual.__doc__=method.__doc__
53 def slice_mapper_options (method):
56 slice_method = TestSlice.__dict__[method.__name__]
57 for slice_spec in self.plc_spec['slices']:
58 site_spec = self.locate_site (slice_spec['sitename'])
59 test_site = TestSite(self,site_spec)
60 test_slice=TestSlice(self,test_site,slice_spec)
61 if not slice_method(test_slice,self.options): overall=False
63 # restore the doc text
64 actual.__doc__=method.__doc__
67 def slice_mapper_options_sfa (method):
71 slice_method = TestSliceSfa.__dict__[method.__name__]
72 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
73 site_spec = self.locate_site (slice_spec['sitename'])
74 test_site = TestSite(self,site_spec)
75 test_slice=TestSliceSfa(test_plc,test_site,slice_spec)
76 if not slice_method(test_slice,self.options): overall=False
78 # restore the doc text
79 actual.__doc__=method.__doc__
87 'display', 'resources_pre', SEP,
88 'delete_vs','create_vs','install', 'configure', 'start', SEP,
89 'fetch_keys', 'store_keys', 'clear_known_hosts', SEP,
90 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', SEP,
91 'reinstall_node', 'init_node','bootcd', 'configure_qemu', 'export_qemu',
92 'kill_all_qemus', 'start_node', SEP,
93 # better use of time: do this now that the nodes are taking off
94 'plcsh_stress_test', SEP,
95 'install_sfa', 'configure_sfa', 'import_sfa', 'start_sfa', SEP,
96 'setup_sfa', 'add_sfa', 'update_sfa', 'view_sfa', SEP,
97 'nodes_ssh_debug', 'nodes_ssh_boot', 'check_slice', 'check_initscripts', SEP,
98 # optionally run sfa later; takes longer, but checks more about nm
99 # 'install_sfa', 'configure_sfa', 'import_sfa', 'start_sfa', SEP,
100 # 'setup_sfa', 'add_sfa', 'update_sfa', 'view_sfa', SEP,
101 'check_slice_sfa', 'delete_sfa', 'stop_sfa', SEP,
102 'check_tcp', 'check_hooks', SEP,
103 'force_gather_logs', 'force_resources_post',
106 'show_boxes', 'resources_list','resources_release','resources_release_plc','resources_release_qemu',SEP,
107 'stop', 'vs_start', SEP,
108 'clean_initscripts', 'clean_nodegroups','clean_all_sites', SEP,
109 'clean_sites', 'clean_nodes', 'clean_slices', 'clean_keys', SEP,
111 'list_all_qemus', 'list_qemus', 'kill_qemus', SEP,
112 'db_dump' , 'db_restore', SEP,
113 'standby_1 through 20',
117 def printable_steps (list):
118 return " ".join(list).replace(" "+SEP+" "," \\\n")
120 def valid_step (step):
123 def __init__ (self,plc_spec,options):
124 self.plc_spec=plc_spec
126 self.test_ssh=TestSsh(self.plc_spec['hostname'],self.options.buildname)
128 self.vserverip=plc_spec['vserverip']
129 self.vservername=plc_spec['vservername']
130 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
133 raise Exception,'chroot-based myplc testing is deprecated'
134 self.apiserver=TestApiserver(self.url,options.dry_run)
137 name=self.plc_spec['name']
138 return "%s.%s"%(name,self.vservername)
141 return self.plc_spec['hostname']
144 return self.test_ssh.is_local()
146 # define the API methods on this object through xmlrpc
147 # would help, but not strictly necessary
151 def actual_command_in_guest (self,command):
152 return self.test_ssh.actual_command(self.host_to_guest(command))
154 def start_guest (self):
155 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host()))
157 def run_in_guest (self,command):
158 return utils.system(self.actual_command_in_guest(command))
160 def run_in_host (self,command):
161 return self.test_ssh.run_in_buildname(command)
163 #command gets run in the vserver
164 def host_to_guest(self,command):
165 return "vserver %s exec %s"%(self.vservername,command)
167 #command gets run in the vserver
168 def start_guest_in_host(self):
169 return "vserver %s start"%(self.vservername)
172 def run_in_guest_piped (self,local,remote):
173 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
175 def auth_root (self):
176 return {'Username':self.plc_spec['PLC_ROOT_USER'],
177 'AuthMethod':'password',
178 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
179 'Role' : self.plc_spec['role']
181 def locate_site (self,sitename):
182 for site in self.plc_spec['sites']:
183 if site['site_fields']['name'] == sitename:
185 if site['site_fields']['login_base'] == sitename:
187 raise Exception,"Cannot locate site %s"%sitename
189 def locate_node (self,nodename):
190 for site in self.plc_spec['sites']:
191 for node in site['nodes']:
192 if node['name'] == nodename:
194 raise Exception,"Cannot locate node %s"%nodename
196 def locate_hostname (self,hostname):
197 for site in self.plc_spec['sites']:
198 for node in site['nodes']:
199 if node['node_fields']['hostname'] == hostname:
201 raise Exception,"Cannot locate hostname %s"%hostname
203 def locate_key (self,keyname):
204 for key in self.plc_spec['keys']:
205 if key['name'] == keyname:
207 raise Exception,"Cannot locate key %s"%keyname
209 def locate_slice (self, slicename):
210 for slice in self.plc_spec['slices']:
211 if slice['slice_fields']['name'] == slicename:
213 raise Exception,"Cannot locate slice %s"%slicename
215 def all_sliver_objs (self):
217 for slice_spec in self.plc_spec['slices']:
218 slicename = slice_spec['slice_fields']['name']
219 for nodename in slice_spec['nodenames']:
220 result.append(self.locate_sliver_obj (nodename,slicename))
223 def locate_sliver_obj (self,nodename,slicename):
224 (site,node) = self.locate_node(nodename)
225 slice = self.locate_slice (slicename)
227 test_site = TestSite (self, site)
228 test_node = TestNode (self, test_site,node)
229 # xxx the slice site is assumed to be the node site - mhh - probably harmless
230 test_slice = TestSlice (self, test_site, slice)
231 return TestSliver (self, test_node, test_slice)
233 def locate_first_node(self):
234 nodename=self.plc_spec['slices'][0]['nodenames'][0]
235 (site,node) = self.locate_node(nodename)
236 test_site = TestSite (self, site)
237 test_node = TestNode (self, test_site,node)
240 def locate_first_sliver (self):
241 slice_spec=self.plc_spec['slices'][0]
242 slicename=slice_spec['slice_fields']['name']
243 nodename=slice_spec['nodenames'][0]
244 return self.locate_sliver_obj(nodename,slicename)
246 # all different hostboxes used in this plc
247 def gather_hostBoxes(self):
248 # maps on sites and nodes, return [ (host_box,test_node) ]
250 for site_spec in self.plc_spec['sites']:
251 test_site = TestSite (self,site_spec)
252 for node_spec in site_spec['nodes']:
253 test_node = TestNode (self, test_site, node_spec)
254 if not test_node.is_real():
255 tuples.append( (test_node.host_box(),test_node) )
256 # transform into a dict { 'host_box' -> [ test_node .. ] }
258 for (box,node) in tuples:
259 if not result.has_key(box):
262 result[box].append(node)
265 # a step for checking this stuff
266 def show_boxes (self):
267 'print summary of nodes location'
268 for (box,nodes) in self.gather_hostBoxes().iteritems():
269 print box,":"," + ".join( [ node.name() for node in nodes ] )
272 # make this a valid step
273 def kill_all_qemus(self):
274 'kill all qemu instances on the qemu boxes involved by this setup'
275 # this is the brute force version, kill all qemus on that host box
276 for (box,nodes) in self.gather_hostBoxes().iteritems():
277 # pass the first nodename, as we don't push template-qemu on testboxes
278 nodedir=nodes[0].nodedir()
279 TestBox(box,self.options.buildname).kill_all_qemus(nodedir)
282 # make this a valid step
283 def list_all_qemus(self):
284 'list all qemu instances on the qemu boxes involved by this setup'
285 for (box,nodes) in self.gather_hostBoxes().iteritems():
286 # this is the brute force version, kill all qemus on that host box
287 TestBox(box,self.options.buildname).list_all_qemus()
290 # kill only the right qemus
291 def list_qemus(self):
292 'list qemu instances for our nodes'
293 for (box,nodes) in self.gather_hostBoxes().iteritems():
294 # the fine-grain version
299 # kill only the right qemus
300 def kill_qemus(self):
301 'kill the qemu instances for our nodes'
302 for (box,nodes) in self.gather_hostBoxes().iteritems():
303 # the fine-grain version
308 #################### display config
310 "show test configuration after localization"
311 self.display_pass (1)
312 self.display_pass (2)
316 always_display_keys=['PLC_WWW_HOST','nodes','sites',]
317 def display_pass (self,passno):
318 for (key,val) in self.plc_spec.iteritems():
319 if not self.options.verbose and key not in TestPlc.always_display_keys: continue
323 self.display_site_spec(site)
324 for node in site['nodes']:
325 self.display_node_spec(node)
326 elif key=='initscripts':
327 for initscript in val:
328 self.display_initscript_spec (initscript)
331 self.display_slice_spec (slice)
334 self.display_key_spec (key)
336 if key not in ['sites','initscripts','slices','keys', 'sfa']:
337 print '+ ',key,':',val
339 def display_site_spec (self,site):
340 print '+ ======== site',site['site_fields']['name']
341 for (k,v) in site.iteritems():
342 if not self.options.verbose and k not in TestPlc.always_display_keys: continue
345 print '+ ','nodes : ',
347 print node['node_fields']['hostname'],'',
353 print user['name'],'',
355 elif k == 'site_fields':
356 print '+ login_base',':',v['login_base']
357 elif k == 'address_fields':
361 PrettyPrinter(indent=8,depth=2).pprint(v)
363 def display_initscript_spec (self,initscript):
364 print '+ ======== initscript',initscript['initscript_fields']['name']
366 def display_key_spec (self,key):
367 print '+ ======== key',key['name']
369 def display_slice_spec (self,slice):
370 print '+ ======== slice',slice['slice_fields']['name']
371 for (k,v) in slice.iteritems():
384 elif k=='slice_fields':
385 print '+ fields',':',
386 print 'max_nodes=',v['max_nodes'],
391 def display_node_spec (self,node):
392 print "+ node",node['name'],"host_box=",node['host_box'],
393 print "hostname=",node['node_fields']['hostname'],
394 print "ip=",node['interface_fields']['ip']
397 # another entry point for just showing the boxes involved
398 def display_mapping (self):
399 TestPlc.display_mapping_plc(self.plc_spec)
403 def display_mapping_plc (plc_spec):
404 print '+ MyPLC',plc_spec['name']
405 print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['hostname'],plc_spec['vservername'])
406 print '+\tIP = %s/%s'%(plc_spec['PLC_API_HOST'],plc_spec['vserverip'])
407 for site_spec in plc_spec['sites']:
408 for node_spec in site_spec['nodes']:
409 TestPlc.display_mapping_node(node_spec)
412 def display_mapping_node (node_spec):
413 print '+ NODE %s'%(node_spec['name'])
414 print '+\tqemu box %s'%node_spec['host_box']
415 print '+\thostname=%s'%node_spec['node_fields']['hostname']
417 def resources_pre (self):
418 "run site-dependant pre-test script as defined in LocalTestResources"
419 from LocalTestResources import local_resources
420 return local_resources.step_pre(self)
422 def resources_post (self):
423 "run site-dependant post-test script as defined in LocalTestResources"
424 from LocalTestResources import local_resources
425 return local_resources.step_post(self)
427 def resources_list (self):
428 "run site-dependant list script as defined in LocalTestResources"
429 from LocalTestResources import local_resources
430 return local_resources.step_list(self)
432 def resources_release (self):
433 "run site-dependant release script as defined in LocalTestResources"
434 from LocalTestResources import local_resources
435 return local_resources.step_release(self)
437 def resources_release_plc (self):
438 "run site-dependant release script as defined in LocalTestResources"
439 from LocalTestResources import local_resources
440 return local_resources.step_release_plc(self)
442 def resources_release_qemu (self):
443 "run site-dependant release script as defined in LocalTestResources"
444 from LocalTestResources import local_resources
445 return local_resources.step_release_qemu(self)
448 "vserver delete the test myplc"
449 self.run_in_host("vserver --silent %s delete"%self.vservername)
453 def create_vs (self):
454 "vserver creation (no install done)"
456 # a full path for the local calls
457 build_dir=os.path.dirname(sys.argv[0])
458 # sometimes this is empty - set to "." in such a case
459 if not build_dir: build_dir="."
460 build_dir += "/build"
462 # use a standard name - will be relative to remote buildname
465 # historically the build was being fetched by the tests
466 # if we've got a build/ locally, we don't need to mess with the build url
467 if os.path.isdir('build'):
468 self.test_ssh.mkdir(build_dir)
469 self.test_ssh.copy_abs('build',build_dir,recursive=True)
471 # use old svn strategy run checkout in any case - would do an update if already exists
472 print "WARNING: The tests module no longer has the ability to pull the build module from svn"
473 print "Please extract a build module under 'build'"
475 build_checkout = "svn checkout %s %s"%(self.options.build_url,build_dir)
476 if self.run_in_host(build_checkout) != 0:
478 # the repo url is taken from arch-rpms-url
479 # with the last step (i386) removed
480 repo_url = self.options.arch_rpms_url
481 for level in [ 'arch' ]:
482 repo_url = os.path.dirname(repo_url)
483 # pass the vbuild-nightly options to vtest-init-vserver
485 test_env_options += " -p %s"%self.options.personality
486 test_env_options += " -d %s"%self.options.pldistro
487 test_env_options += " -f %s"%self.options.fcdistro
488 script="vtest-init-vserver.sh"
489 vserver_name = self.vservername
490 vserver_options="--netdev eth0 --interface %s"%self.vserverip
492 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
493 vserver_options += " --hostname %s"%vserver_hostname
495 print "Cannot reverse lookup %s"%self.vserverip
496 print "This is considered fatal, as this might pollute the test results"
498 create_vserver="%(build_dir)s/%(script)s %(test_env_options)s %(vserver_name)s %(repo_url)s -- %(vserver_options)s"%locals()
499 return self.run_in_host(create_vserver) == 0
503 "yum install myplc, noderepo, and the plain bootstrapfs"
505 # workaround for getting pgsql8.2 on centos5
506 if self.options.fcdistro == "centos5":
507 self.run_in_guest("rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-3.noarch.rpm")
509 if self.options.personality == "linux32":
511 elif self.options.personality == "linux64":
514 raise Exception, "Unsupported personality %r"%self.options.personality
516 nodefamily="%s-%s-%s"%(self.options.pldistro,self.options.fcdistro,arch)
518 # try to install slicerepo - not fatal yet
519 self.run_in_guest("yum -y install slicerepo-%s"%nodefamily)
522 self.run_in_guest("yum -y install myplc")==0 and \
523 self.run_in_guest("yum -y install noderepo-%s"%nodefamily)==0 and \
524 self.run_in_guest("yum -y install bootstrapfs-%s-plain"%nodefamily)==0
529 tmpname='%s.plc-config-tty'%(self.name())
530 fileconf=open(tmpname,'w')
531 for var in [ 'PLC_NAME',
535 'PLC_MAIL_SUPPORT_ADDRESS',
538 # Above line was added for integrating SFA Testing
544 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
545 fileconf.write('w\n')
546 fileconf.write('q\n')
548 utils.system('cat %s'%tmpname)
549 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
550 utils.system('rm %s'%tmpname)
555 self.run_in_guest('service plc start')
560 self.run_in_guest('service plc stop')
564 "start the PLC vserver"
568 # stores the keys from the config for further use
569 def store_keys(self):
570 "stores test users ssh keys in keys/"
571 for key_spec in self.plc_spec['keys']:
572 TestKey(self,key_spec).store_key()
575 def clean_keys(self):
576 "removes keys cached in keys/"
577 utils.system("rm -rf %s/keys/"%os.path(sys.argv[0]))
579 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
580 # for later direct access to the nodes
581 def fetch_keys(self):
582 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
584 if not os.path.isdir(dir):
586 vservername=self.vservername
588 prefix = 'debug_ssh_key'
589 for ext in [ 'pub', 'rsa' ] :
590 src="/vservers/%(vservername)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
591 dst="keys/%(vservername)s-debug.%(ext)s"%locals()
592 if self.test_ssh.fetch(src,dst) != 0: overall=False
596 "create sites with PLCAPI"
597 return self.do_sites()
599 def clean_sites (self):
600 "delete sites with PLCAPI"
601 return self.do_sites(action="delete")
603 def do_sites (self,action="add"):
604 for site_spec in self.plc_spec['sites']:
605 test_site = TestSite (self,site_spec)
606 if (action != "add"):
607 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
608 test_site.delete_site()
609 # deleted with the site
610 #test_site.delete_users()
613 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
614 test_site.create_site()
615 test_site.create_users()
618 def clean_all_sites (self):
619 "Delete all sites in PLC, and related objects"
620 print 'auth_root',self.auth_root()
621 site_ids = [s['site_id'] for s in self.apiserver.GetSites(self.auth_root(), {}, ['site_id'])]
622 for site_id in site_ids:
623 print 'Deleting site_id',site_id
624 self.apiserver.DeleteSite(self.auth_root(),site_id)
627 "create nodes with PLCAPI"
628 return self.do_nodes()
629 def clean_nodes (self):
630 "delete nodes with PLCAPI"
631 return self.do_nodes(action="delete")
633 def do_nodes (self,action="add"):
634 for site_spec in self.plc_spec['sites']:
635 test_site = TestSite (self,site_spec)
637 utils.header("Deleting nodes in site %s"%test_site.name())
638 for node_spec in site_spec['nodes']:
639 test_node=TestNode(self,test_site,node_spec)
640 utils.header("Deleting %s"%test_node.name())
641 test_node.delete_node()
643 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
644 for node_spec in site_spec['nodes']:
645 utils.pprint('Creating node %s'%node_spec,node_spec)
646 test_node = TestNode (self,test_site,node_spec)
647 test_node.create_node ()
650 def nodegroups (self):
651 "create nodegroups with PLCAPI"
652 return self.do_nodegroups("add")
653 def clean_nodegroups (self):
654 "delete nodegroups with PLCAPI"
655 return self.do_nodegroups("delete")
657 # create nodegroups if needed, and populate
658 def do_nodegroups (self, action="add"):
659 # 1st pass to scan contents
661 for site_spec in self.plc_spec['sites']:
662 test_site = TestSite (self,site_spec)
663 for node_spec in site_spec['nodes']:
664 test_node=TestNode (self,test_site,node_spec)
665 if node_spec.has_key('nodegroups'):
666 nodegroupnames=node_spec['nodegroups']
667 if isinstance(nodegroupnames,StringTypes):
668 nodegroupnames = [ nodegroupnames ]
669 for nodegroupname in nodegroupnames:
670 if not groups_dict.has_key(nodegroupname):
671 groups_dict[nodegroupname]=[]
672 groups_dict[nodegroupname].append(test_node.name())
673 auth=self.auth_root()
675 for (nodegroupname,group_nodes) in groups_dict.iteritems():
677 print 'nodegroups:','dealing with nodegroup',nodegroupname,'on nodes',group_nodes
678 # first, check if the nodetagtype is here
679 tag_types = self.apiserver.GetTagTypes(auth,{'tagname':nodegroupname})
681 tag_type_id = tag_types[0]['tag_type_id']
683 tag_type_id = self.apiserver.AddTagType(auth,
684 {'tagname':nodegroupname,
685 'description': 'for nodegroup %s'%nodegroupname,
688 print 'located tag (type)',nodegroupname,'as',tag_type_id
690 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
692 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
693 print 'created nodegroup',nodegroupname,'from tagname',nodegroupname,'and value','yes'
694 # set node tag on all nodes, value='yes'
695 for nodename in group_nodes:
697 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
699 traceback.print_exc()
700 print 'node',nodename,'seems to already have tag',nodegroupname
703 expect_yes = self.apiserver.GetNodeTags(auth,
704 {'hostname':nodename,
705 'tagname':nodegroupname},
706 ['value'])[0]['value']
707 if expect_yes != "yes":
708 print 'Mismatch node tag on node',nodename,'got',expect_yes
711 if not self.options.dry_run:
712 print 'Cannot find tag',nodegroupname,'on node',nodename
716 print 'cleaning nodegroup',nodegroupname
717 self.apiserver.DeleteNodeGroup(auth,nodegroupname)
719 traceback.print_exc()
723 # return a list of tuples (nodename,qemuname)
724 def all_node_infos (self) :
726 for site_spec in self.plc_spec['sites']:
727 node_infos += [ (node_spec['node_fields']['hostname'],node_spec['host_box']) \
728 for node_spec in site_spec['nodes'] ]
731 def all_nodenames (self): return [ x[0] for x in self.all_node_infos() ]
733 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
734 def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period=15):
735 if self.options.dry_run:
739 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
740 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
741 # the nodes that haven't checked yet - start with a full list and shrink over time
742 tocheck = self.all_hostnames()
743 utils.header("checking nodes %r"%tocheck)
744 # create a dict hostname -> status
745 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
748 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
750 for array in tocheck_status:
751 hostname=array['hostname']
752 boot_state=array['boot_state']
753 if boot_state == target_boot_state:
754 utils.header ("%s has reached the %s state"%(hostname,target_boot_state))
756 # if it's a real node, never mind
757 (site_spec,node_spec)=self.locate_hostname(hostname)
758 if TestNode.is_real_model(node_spec['node_fields']['model']):
759 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
761 boot_state = target_boot_state
762 elif datetime.datetime.now() > graceout:
763 utils.header ("%s still in '%s' state"%(hostname,boot_state))
764 graceout=datetime.datetime.now()+datetime.timedelta(1)
765 status[hostname] = boot_state
767 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != target_boot_state ]
770 if datetime.datetime.now() > timeout:
771 for hostname in tocheck:
772 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
774 # otherwise, sleep for a while
776 # only useful in empty plcs
779 def nodes_booted(self):
780 return self.nodes_check_boot_state('boot',timeout_minutes=30,silent_minutes=20)
782 def check_nodes_ssh(self,debug,timeout_minutes,silent_minutes,period=15):
784 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
785 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
786 vservername=self.vservername
789 local_key = "keys/%(vservername)s-debug.rsa"%locals()
792 local_key = "keys/key1.rsa"
793 node_infos = self.all_node_infos()
794 utils.header("checking ssh access (expected in %s mode) to nodes:"%message)
795 for (nodename,qemuname) in node_infos:
796 utils.header("hostname=%s -- qemubox=%s"%(nodename,qemuname))
797 utils.header("max timeout is %d minutes, silent for %d minutes (period is %s)"%\
798 (timeout_minutes,silent_minutes,period))
800 for node_info in node_infos:
801 (hostname,qemuname) = node_info
802 # try to run 'hostname' in the node
803 command = TestSsh (hostname,key=local_key).actual_command("hostname;uname -a")
804 # don't spam logs - show the command only after the grace period
805 success = utils.system ( command, silent=datetime.datetime.now() < graceout)
807 utils.header('Successfully entered root@%s (%s)'%(hostname,message))
809 node_infos.remove(node_info)
811 # we will have tried real nodes once, in case they're up - but if not, just skip
812 (site_spec,node_spec)=self.locate_hostname(hostname)
813 if TestNode.is_real_model(node_spec['node_fields']['model']):
814 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
815 node_infos.remove(node_info)
818 if datetime.datetime.now() > timeout:
819 for (hostname,qemuname) in node_infos:
820 utils.header("FAILURE to ssh into %s (on %s)"%(hostname,qemuname))
822 # otherwise, sleep for a while
824 # only useful in empty plcs
827 def nodes_ssh_debug(self):
828 "Tries to ssh into nodes in debug mode with the debug ssh key"
829 return self.check_nodes_ssh(debug=True,timeout_minutes=30,silent_minutes=5)
831 def nodes_ssh_boot(self):
832 "Tries to ssh into nodes in production mode with the root ssh key"
833 return self.check_nodes_ssh(debug=False,timeout_minutes=30,silent_minutes=15)
836 def init_node (self):
837 "all nodes : init a clean local directory for holding node-dep stuff like iso image..."
841 "all nodes: invoke GetBootMedium and store result locally"
844 def configure_qemu (self):
845 "all nodes: compute qemu config qemu.conf and store it locally"
848 def reinstall_node (self):
849 "all nodes: mark PLCAPI boot_state as reinstall"
852 def export_qemu (self):
853 "all nodes: push local node-dep directory on the qemu box"
856 ### check hooks : invoke scripts from hooks/{node,slice}
857 def check_hooks_node (self):
858 return self.locate_first_node().check_hooks()
859 def check_hooks_sliver (self) :
860 return self.locate_first_sliver().check_hooks()
862 def check_hooks (self):
863 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
864 return self.check_hooks_node() and self.check_hooks_sliver()
867 def do_check_initscripts(self):
869 for slice_spec in self.plc_spec['slices']:
870 if not slice_spec.has_key('initscriptname'):
872 initscript=slice_spec['initscriptname']
873 for nodename in slice_spec['nodenames']:
874 (site,node) = self.locate_node (nodename)
875 # xxx - passing the wrong site - probably harmless
876 test_site = TestSite (self,site)
877 test_slice = TestSlice (self,test_site,slice_spec)
878 test_node = TestNode (self,test_site,node)
879 test_sliver = TestSliver (self, test_node, test_slice)
880 if not test_sliver.check_initscript(initscript):
884 def check_initscripts(self):
885 "check that the initscripts have triggered"
886 return self.do_check_initscripts()
888 def initscripts (self):
889 "create initscripts with PLCAPI"
890 for initscript in self.plc_spec['initscripts']:
891 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
892 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
895 def clean_initscripts (self):
896 "delete initscripts with PLCAPI"
897 for initscript in self.plc_spec['initscripts']:
898 initscript_name = initscript['initscript_fields']['name']
899 print('Attempting to delete %s in plc %s'%(initscript_name,self.plc_spec['name']))
901 self.apiserver.DeleteInitScript(self.auth_root(),initscript_name)
902 print initscript_name,'deleted'
904 print 'deletion went wrong - probably did not exist'
909 "create slices with PLCAPI"
910 return self.do_slices()
912 def clean_slices (self):
913 "delete slices with PLCAPI"
914 return self.do_slices("delete")
916 def do_slices (self, action="add"):
917 for slice in self.plc_spec['slices']:
918 site_spec = self.locate_site (slice['sitename'])
919 test_site = TestSite(self,site_spec)
920 test_slice=TestSlice(self,test_site,slice)
922 utils.header("Deleting slices in site %s"%test_site.name())
923 test_slice.delete_slice()
925 utils.pprint("Creating slice",slice)
926 test_slice.create_slice()
927 utils.header('Created Slice %s'%slice['slice_fields']['name'])
930 @slice_mapper_options
931 def check_slice(self):
932 "tries to ssh-enter the slice with the user key, to ensure slice creation"
936 def clear_known_hosts (self):
937 "remove test nodes entries from the local known_hosts file"
941 def start_node (self) :
942 "all nodes: start the qemu instance (also runs qemu-bridge-init start)"
945 def check_tcp (self):
946 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
947 specs = self.plc_spec['tcp_test']
952 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
953 if not s_test_sliver.run_tcp_server(port,timeout=10):
957 # idem for the client side
958 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
959 if not c_test_sliver.run_tcp_client(s_test_sliver.test_node.name(),port):
963 def plcsh_stress_test (self):
964 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
965 # install the stress-test in the plc image
966 location = "/usr/share/plc_api/plcsh_stress_test.py"
967 remote="/vservers/%s/%s"%(self.vservername,location)
968 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
970 command += " -- --check"
971 if self.options.size == 1:
973 return ( self.run_in_guest(command) == 0)
975 # populate runs the same utility without slightly different options
976 # in particular runs with --preserve (dont cleanup) and without --check
977 # also it gets run twice, once with the --foreign option for creating fake foreign entries
980 def install_sfa(self):
981 "yum install sfa, sfa-plc and sfa-client"
982 if self.options.personality == "linux32":
984 elif self.options.personality == "linux64":
987 raise Exception, "Unsupported personality %r"%self.options.personality
988 return self.run_in_guest("yum -y install sfa sfa-client sfa-plc sfa-sfatables")==0
991 def configure_sfa(self):
993 tmpname='%s.sfa-config-tty'%(self.name())
994 fileconf=open(tmpname,'w')
995 for var in [ 'SFA_REGISTRY_ROOT_AUTH',
996 'SFA_REGISTRY_LEVEL1_AUTH',
998 'SFA_AGGREGATE_HOST',
1004 'SFA_PLC_DB_PASSWORD',
1006 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec['sfa'][var]))
1007 fileconf.write('w\n')
1008 fileconf.write('R\n')
1009 fileconf.write('q\n')
1011 utils.system('cat %s'%tmpname)
1012 self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
1013 utils.system('rm %s'%tmpname)
1016 def import_sfa(self):
1018 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1019 return self.run_in_guest('sfa-import-plc.py')==0
1020 # not needed anymore
1021 # self.run_in_guest('cp /etc/sfa/authorities/%s/%s.pkey /etc/sfa/authorities/server.key'%(auth,auth))
1023 def start_sfa(self):
1025 return self.run_in_guest('service sfa start')==0
1027 def setup_sfa(self):
1028 "sfi client configuration"
1030 if os.path.exists(dir_name):
1031 utils.system('rm -rf %s'%dir_name)
1032 utils.system('mkdir %s'%dir_name)
1033 file_name=dir_name + os.sep + 'fake-pi1.pkey'
1034 fileconf=open(file_name,'w')
1035 fileconf.write (self.plc_spec['keys'][0]['private'])
1038 file_name=dir_name + os.sep + 'sfi_config'
1039 fileconf=open(file_name,'w')
1040 SFI_AUTH=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']+".main"
1041 fileconf.write ("SFI_AUTH='%s'"%SFI_AUTH)
1042 fileconf.write('\n')
1043 SFI_USER=SFI_AUTH+'.fake-pi1'
1044 fileconf.write ("SFI_USER='%s'"%SFI_USER)
1045 fileconf.write('\n')
1046 SFI_REGISTRY='http://' + self.plc_spec['sfa']['SFA_PLC_DB_HOST'] + ':12345/'
1047 fileconf.write ("SFI_REGISTRY='%s'"%SFI_REGISTRY)
1048 fileconf.write('\n')
1049 SFI_SM='http://' + self.plc_spec['sfa']['SFA_PLC_DB_HOST'] + ':12347/'
1050 fileconf.write ("SFI_SM='%s'"%SFI_SM)
1051 fileconf.write('\n')
1054 file_name=dir_name + os.sep + 'person.xml'
1055 fileconf=open(file_name,'w')
1056 for record in self.plc_spec['sfa']['sfa_person_xml']:
1057 person_record=record
1058 fileconf.write(person_record)
1059 fileconf.write('\n')
1062 file_name=dir_name + os.sep + 'slice.xml'
1063 fileconf=open(file_name,'w')
1064 for record in self.plc_spec['sfa']['sfa_slice_xml']:
1066 #slice_record=self.plc_spec['sfa']['sfa_slice_xml']
1067 fileconf.write(slice_record)
1068 fileconf.write('\n')
1071 file_name=dir_name + os.sep + 'slice.rspec'
1072 fileconf=open(file_name,'w')
1074 for (key, value) in self.plc_spec['sfa']['sfa_slice_rspec'].items():
1076 fileconf.write(slice_rspec)
1077 fileconf.write('\n')
1080 remote="/vservers/%s/%s"%(self.vservername,location)
1081 self.test_ssh.copy_abs(dir_name, remote, recursive=True)
1083 #utils.system('cat %s'%tmpname)
1084 utils.system('rm -rf %s'%dir_name)
1088 "run sfi.py add (on Registry) and sfi.py create (on SM) to form new objects"
1090 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1091 success=test_user_sfa.add_user()
1093 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1094 site_spec = self.locate_site (slice_spec['sitename'])
1095 test_site = TestSite(self,site_spec)
1096 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1097 success1=test_slice_sfa.add_slice()
1098 success2=test_slice_sfa.create_slice()
1099 return success and success1 and success2
1101 def update_sfa(self):
1102 "run sfi.py update (on Registry) and sfi.py create (on SM) on existing objects"
1104 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1105 success1=test_user_sfa.update_user()
1107 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1108 site_spec = self.locate_site (slice_spec['sitename'])
1109 test_site = TestSite(self,site_spec)
1110 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1111 success2=test_slice_sfa.update_slice()
1112 return success1 and success2
1115 "run sfi.py list and sfi.py show (both on Registry) and sfi.py slices and sfi.py resources (both on SM)"
1116 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1118 self.run_in_guest("sfi.py -d /root/.sfi/ list %s.main"%auth)==0 and \
1119 self.run_in_guest("sfi.py -d /root/.sfi/ show %s.main"%auth)==0 and \
1120 self.run_in_guest("sfi.py -d /root/.sfi/ slices")==0 and \
1121 self.run_in_guest("sfi.py -d /root/.sfi/ resources -o resources")==0
1123 @slice_mapper_options_sfa
1124 def check_slice_sfa(self):
1125 "tries to ssh-enter the SFA slice"
1128 def delete_sfa(self):
1129 "run sfi.py delete (on SM), sfi.py remove (on Registry)"
1131 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1132 success1=test_user_sfa.delete_user()
1133 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1134 site_spec = self.locate_site (slice_spec['sitename'])
1135 test_site = TestSite(self,site_spec)
1136 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1137 success2=test_slice_sfa.delete_slice()
1139 return success1 and success2
1143 return self.run_in_guest('service sfa stop')==0
1145 def populate (self):
1146 "creates random entries in the PLCAPI"
1147 # install the stress-test in the plc image
1148 location = "/usr/share/plc_api/plcsh_stress_test.py"
1149 remote="/vservers/%s/%s"%(self.vservername,location)
1150 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1152 command += " -- --preserve --short-names"
1153 local = (self.run_in_guest(command) == 0);
1154 # second run with --foreign
1155 command += ' --foreign'
1156 remote = (self.run_in_guest(command) == 0);
1157 return ( local and remote)
1159 def gather_logs (self):
1160 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1161 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1162 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1163 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1164 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1165 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1167 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1168 self.gather_var_logs ()
1170 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1171 self.gather_pgsql_logs ()
1173 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1174 for site_spec in self.plc_spec['sites']:
1175 test_site = TestSite (self,site_spec)
1176 for node_spec in site_spec['nodes']:
1177 test_node=TestNode(self,test_site,node_spec)
1178 test_node.gather_qemu_logs()
1180 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1181 self.gather_nodes_var_logs()
1183 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1184 self.gather_slivers_var_logs()
1187 def gather_slivers_var_logs(self):
1188 for test_sliver in self.all_sliver_objs():
1189 remote = test_sliver.tar_var_logs()
1190 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
1191 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
1192 utils.system(command)
1195 def gather_var_logs (self):
1196 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
1197 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1198 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
1199 utils.system(command)
1200 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd"%self.name()
1201 utils.system(command)
1203 def gather_pgsql_logs (self):
1204 utils.system("mkdir -p logs/myplc.pgsql-log.%s"%self.name())
1205 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1206 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -"%self.name()
1207 utils.system(command)
1209 def gather_nodes_var_logs (self):
1210 for site_spec in self.plc_spec['sites']:
1211 test_site = TestSite (self,site_spec)
1212 for node_spec in site_spec['nodes']:
1213 test_node=TestNode(self,test_site,node_spec)
1214 test_ssh = TestSsh (test_node.name(),key="keys/key1.rsa")
1215 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1216 command = command + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
1217 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
1218 utils.system(command)
1221 # returns the filename to use for sql dump/restore, using options.dbname if set
1222 def dbfile (self, database):
1223 # uses options.dbname if it is found
1225 name=self.options.dbname
1226 if not isinstance(name,StringTypes):
1229 t=datetime.datetime.now()
1232 return "/root/%s-%s.sql"%(database,name)
1235 'dump the planetlab5 DB in /root in the PLC - filename has time'
1236 dump=self.dbfile("planetab5")
1237 self.run_in_guest('pg_dump -U pgsqluser planetlab5 -f '+ dump)
1238 utils.header('Dumped planetlab5 database in %s'%dump)
1241 def db_restore(self):
1242 'restore the planetlab5 DB - looks broken, but run -n might help'
1243 dump=self.dbfile("planetab5")
1244 ##stop httpd service
1245 self.run_in_guest('service httpd stop')
1246 # xxx - need another wrapper
1247 self.run_in_guest_piped('echo drop database planetlab5','psql --user=pgsqluser template1')
1248 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab5')
1249 self.run_in_guest('psql -U pgsqluser planetlab5 -f '+dump)
1250 ##starting httpd service
1251 self.run_in_guest('service httpd start')
1253 utils.header('Database restored from ' + dump)
1256 def standby_1(): pass
1258 def standby_2(): pass
1260 def standby_3(): pass
1262 def standby_4(): pass
1264 def standby_5(): pass
1266 def standby_6(): pass
1268 def standby_7(): pass
1270 def standby_8(): pass
1272 def standby_9(): pass
1274 def standby_10(): pass
1276 def standby_11(): pass
1278 def standby_12(): pass
1280 def standby_13(): pass
1282 def standby_14(): pass
1284 def standby_15(): pass
1286 def standby_16(): pass
1288 def standby_17(): pass
1290 def standby_18(): pass
1292 def standby_19(): pass
1294 def standby_20(): pass