7 from types import StringTypes
11 from TestSite import TestSite
12 from TestNode import TestNode
13 from TestUser import TestUser
14 from TestKey import TestKey
15 from TestSlice import TestSlice
16 from TestSliver import TestSliver
17 from TestBox import TestBox
18 from TestSsh import TestSsh
19 from TestApiserver import TestApiserver
20 from TestSliceSfa import TestSliceSfa
21 from TestUserSfa import TestUserSfa
23 # step methods must take (self) and return a boolean (options is a member of the class)
25 def standby(minutes,dry_run):
26 utils.header('Entering StandBy for %d mn'%minutes)
30 time.sleep(60*minutes)
33 def standby_generic (func):
35 minutes=int(func.__name__.split("_")[1])
36 return standby(minutes,self.options.dry_run)
39 def node_mapper (method):
42 node_method = TestNode.__dict__[method.__name__]
43 for site_spec in self.plc_spec['sites']:
44 test_site = TestSite (self,site_spec)
45 for node_spec in site_spec['nodes']:
46 test_node = TestNode (self,test_site,node_spec)
47 if not node_method(test_node): overall=False
49 # restore the doc text
50 actual.__doc__=method.__doc__
53 def slice_mapper_options (method):
56 slice_method = TestSlice.__dict__[method.__name__]
57 for slice_spec in self.plc_spec['slices']:
58 site_spec = self.locate_site (slice_spec['sitename'])
59 test_site = TestSite(self,site_spec)
60 test_slice=TestSlice(self,test_site,slice_spec)
61 if not slice_method(test_slice,self.options): overall=False
63 # restore the doc text
64 actual.__doc__=method.__doc__
67 def slice_mapper_options_sfa (method):
71 slice_method = TestSliceSfa.__dict__[method.__name__]
72 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
73 site_spec = self.locate_site (slice_spec['sitename'])
74 test_site = TestSite(self,site_spec)
75 test_slice=TestSliceSfa(test_plc,test_site,slice_spec)
76 if not slice_method(test_slice,self.options): overall=False
78 # restore the doc text
79 actual.__doc__=method.__doc__
87 'display', 'resources_pre', SEP,
88 'delete_vs','create_vs','install', 'configure', 'start', SEP,
89 'fetch_keys', 'store_keys', 'clear_known_hosts', SEP,
90 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', 'leases', SEP,
91 'reinstall_node', 'init_node','bootcd', 'configure_qemu', 'export_qemu',
92 'kill_all_qemus', 'start_node', SEP,
93 # better use of time: do this now that the nodes are taking off
94 'plcsh_stress_test', SEP,
95 'install_sfa', 'configure_sfa', 'import_sfa', 'start_sfa', SEP,
96 'setup_sfa', 'add_sfa', 'update_sfa', 'view_sfa', SEP,
97 'nodes_ssh_debug', 'nodes_ssh_boot', 'check_slice', 'check_initscripts', SEP,
98 # optionally run sfa later; takes longer, but checks more about nm
99 # 'install_sfa', 'configure_sfa', 'import_sfa', 'start_sfa', SEP,
100 # 'setup_sfa', 'add_sfa', 'update_sfa', 'view_sfa', SEP,
101 'check_slice_sfa', 'delete_sfa', 'stop_sfa', SEP,
102 'check_tcp', 'check_hooks', SEP,
103 'force_gather_logs', 'force_resources_post',
106 'show_boxes', 'resources_list','resources_release','resources_release_plc','resources_release_qemu',SEP,
107 'stop', 'vs_start', SEP,
108 'clean_initscripts', 'clean_nodegroups','clean_all_sites', SEP,
109 'clean_sites', 'clean_nodes', 'clean_slices', 'clean_keys', SEP,
110 'clean_leases', 'list_leases', SEP,
112 'list_all_qemus', 'list_qemus', 'kill_qemus', SEP,
113 'db_dump' , 'db_restore', SEP,
114 'standby_1 through 20',
118 def printable_steps (list):
119 return " ".join(list).replace(" "+SEP+" "," \\\n")
121 def valid_step (step):
124 # turn off the sfa-related steps when build has skipped SFA
125 # this is originally for centos5 as recent SFAs won't build on this platformb
127 def check_whether_build_has_sfa (rpms_url):
128 retcod=os.system ("curl --silent %s/ | grep -q sfa"%rpms_url)
129 # full builds are expected to return with 0 here
131 TestPlc.default_steps = [ step for step in TestPlc.default_steps
132 if step.find('sfa') < 0 ]
134 def __init__ (self,plc_spec,options):
135 self.plc_spec=plc_spec
137 self.test_ssh=TestSsh(self.plc_spec['hostname'],self.options.buildname)
139 self.vserverip=plc_spec['vserverip']
140 self.vservername=plc_spec['vservername']
141 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
144 raise Exception,'chroot-based myplc testing is deprecated'
145 self.apiserver=TestApiserver(self.url,options.dry_run)
148 name=self.plc_spec['name']
149 return "%s.%s"%(name,self.vservername)
152 return self.plc_spec['hostname']
155 return self.test_ssh.is_local()
157 # define the API methods on this object through xmlrpc
158 # would help, but not strictly necessary
162 def actual_command_in_guest (self,command):
163 return self.test_ssh.actual_command(self.host_to_guest(command))
165 def start_guest (self):
166 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host()))
168 def run_in_guest (self,command):
169 return utils.system(self.actual_command_in_guest(command))
171 def run_in_host (self,command):
172 return self.test_ssh.run_in_buildname(command)
174 #command gets run in the vserver
175 def host_to_guest(self,command):
176 return "vserver %s exec %s"%(self.vservername,command)
178 #command gets run in the vserver
179 def start_guest_in_host(self):
180 return "vserver %s start"%(self.vservername)
183 def run_in_guest_piped (self,local,remote):
184 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
186 def auth_root (self):
187 return {'Username':self.plc_spec['PLC_ROOT_USER'],
188 'AuthMethod':'password',
189 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
190 'Role' : self.plc_spec['role']
192 def locate_site (self,sitename):
193 for site in self.plc_spec['sites']:
194 if site['site_fields']['name'] == sitename:
196 if site['site_fields']['login_base'] == sitename:
198 raise Exception,"Cannot locate site %s"%sitename
200 def locate_node (self,nodename):
201 for site in self.plc_spec['sites']:
202 for node in site['nodes']:
203 if node['name'] == nodename:
205 raise Exception,"Cannot locate node %s"%nodename
207 def locate_hostname (self,hostname):
208 for site in self.plc_spec['sites']:
209 for node in site['nodes']:
210 if node['node_fields']['hostname'] == hostname:
212 raise Exception,"Cannot locate hostname %s"%hostname
214 def locate_key (self,keyname):
215 for key in self.plc_spec['keys']:
216 if key['name'] == keyname:
218 raise Exception,"Cannot locate key %s"%keyname
220 def locate_slice (self, slicename):
221 for slice in self.plc_spec['slices']:
222 if slice['slice_fields']['name'] == slicename:
224 raise Exception,"Cannot locate slice %s"%slicename
226 def all_sliver_objs (self):
228 for slice_spec in self.plc_spec['slices']:
229 slicename = slice_spec['slice_fields']['name']
230 for nodename in slice_spec['nodenames']:
231 result.append(self.locate_sliver_obj (nodename,slicename))
234 def locate_sliver_obj (self,nodename,slicename):
235 (site,node) = self.locate_node(nodename)
236 slice = self.locate_slice (slicename)
238 test_site = TestSite (self, site)
239 test_node = TestNode (self, test_site,node)
240 # xxx the slice site is assumed to be the node site - mhh - probably harmless
241 test_slice = TestSlice (self, test_site, slice)
242 return TestSliver (self, test_node, test_slice)
244 def locate_first_node(self):
245 nodename=self.plc_spec['slices'][0]['nodenames'][0]
246 (site,node) = self.locate_node(nodename)
247 test_site = TestSite (self, site)
248 test_node = TestNode (self, test_site,node)
251 def locate_first_sliver (self):
252 slice_spec=self.plc_spec['slices'][0]
253 slicename=slice_spec['slice_fields']['name']
254 nodename=slice_spec['nodenames'][0]
255 return self.locate_sliver_obj(nodename,slicename)
257 # all different hostboxes used in this plc
258 def gather_hostBoxes(self):
259 # maps on sites and nodes, return [ (host_box,test_node) ]
261 for site_spec in self.plc_spec['sites']:
262 test_site = TestSite (self,site_spec)
263 for node_spec in site_spec['nodes']:
264 test_node = TestNode (self, test_site, node_spec)
265 if not test_node.is_real():
266 tuples.append( (test_node.host_box(),test_node) )
267 # transform into a dict { 'host_box' -> [ test_node .. ] }
269 for (box,node) in tuples:
270 if not result.has_key(box):
273 result[box].append(node)
276 # a step for checking this stuff
277 def show_boxes (self):
278 'print summary of nodes location'
279 for (box,nodes) in self.gather_hostBoxes().iteritems():
280 print box,":"," + ".join( [ node.name() for node in nodes ] )
283 # make this a valid step
284 def kill_all_qemus(self):
285 'kill all qemu instances on the qemu boxes involved by this setup'
286 # this is the brute force version, kill all qemus on that host box
287 for (box,nodes) in self.gather_hostBoxes().iteritems():
288 # pass the first nodename, as we don't push template-qemu on testboxes
289 nodedir=nodes[0].nodedir()
290 TestBox(box,self.options.buildname).kill_all_qemus(nodedir)
293 # make this a valid step
294 def list_all_qemus(self):
295 'list all qemu instances on the qemu boxes involved by this setup'
296 for (box,nodes) in self.gather_hostBoxes().iteritems():
297 # this is the brute force version, kill all qemus on that host box
298 TestBox(box,self.options.buildname).list_all_qemus()
301 # kill only the right qemus
302 def list_qemus(self):
303 'list qemu instances for our nodes'
304 for (box,nodes) in self.gather_hostBoxes().iteritems():
305 # the fine-grain version
310 # kill only the right qemus
311 def kill_qemus(self):
312 'kill the qemu instances for our nodes'
313 for (box,nodes) in self.gather_hostBoxes().iteritems():
314 # the fine-grain version
319 #################### display config
321 "show test configuration after localization"
322 self.display_pass (1)
323 self.display_pass (2)
327 always_display_keys=['PLC_WWW_HOST','nodes','sites',]
328 def display_pass (self,passno):
329 for (key,val) in self.plc_spec.iteritems():
330 if not self.options.verbose and key not in TestPlc.always_display_keys: continue
334 self.display_site_spec(site)
335 for node in site['nodes']:
336 self.display_node_spec(node)
337 elif key=='initscripts':
338 for initscript in val:
339 self.display_initscript_spec (initscript)
342 self.display_slice_spec (slice)
345 self.display_key_spec (key)
347 if key not in ['sites','initscripts','slices','keys', 'sfa']:
348 print '+ ',key,':',val
350 def display_site_spec (self,site):
351 print '+ ======== site',site['site_fields']['name']
352 for (k,v) in site.iteritems():
353 if not self.options.verbose and k not in TestPlc.always_display_keys: continue
356 print '+ ','nodes : ',
358 print node['node_fields']['hostname'],'',
364 print user['name'],'',
366 elif k == 'site_fields':
367 print '+ login_base',':',v['login_base']
368 elif k == 'address_fields':
374 def display_initscript_spec (self,initscript):
375 print '+ ======== initscript',initscript['initscript_fields']['name']
377 def display_key_spec (self,key):
378 print '+ ======== key',key['name']
380 def display_slice_spec (self,slice):
381 print '+ ======== slice',slice['slice_fields']['name']
382 for (k,v) in slice.iteritems():
395 elif k=='slice_fields':
396 print '+ fields',':',
397 print 'max_nodes=',v['max_nodes'],
402 def display_node_spec (self,node):
403 print "+ node=%s host_box=%s"%(node['name'],node['host_box']),
404 print "hostname=",node['node_fields']['hostname'],
405 print "ip=",node['interface_fields']['ip']
406 if self.options.verbose:
407 utils.pprint("node details",node,depth=3)
409 # another entry point for just showing the boxes involved
410 def display_mapping (self):
411 TestPlc.display_mapping_plc(self.plc_spec)
415 def display_mapping_plc (plc_spec):
416 print '+ MyPLC',plc_spec['name']
417 print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['hostname'],plc_spec['vservername'])
418 print '+\tIP = %s/%s'%(plc_spec['PLC_API_HOST'],plc_spec['vserverip'])
419 for site_spec in plc_spec['sites']:
420 for node_spec in site_spec['nodes']:
421 TestPlc.display_mapping_node(node_spec)
424 def display_mapping_node (node_spec):
425 print '+ NODE %s'%(node_spec['name'])
426 print '+\tqemu box %s'%node_spec['host_box']
427 print '+\thostname=%s'%node_spec['node_fields']['hostname']
429 def resources_pre (self):
430 "run site-dependant pre-test script as defined in LocalTestResources"
431 from LocalTestResources import local_resources
432 return local_resources.step_pre(self)
434 def resources_post (self):
435 "run site-dependant post-test script as defined in LocalTestResources"
436 from LocalTestResources import local_resources
437 return local_resources.step_post(self)
439 def resources_list (self):
440 "run site-dependant list script as defined in LocalTestResources"
441 from LocalTestResources import local_resources
442 return local_resources.step_list(self)
444 def resources_release (self):
445 "run site-dependant release script as defined in LocalTestResources"
446 from LocalTestResources import local_resources
447 return local_resources.step_release(self)
449 def resources_release_plc (self):
450 "run site-dependant release script as defined in LocalTestResources"
451 from LocalTestResources import local_resources
452 return local_resources.step_release_plc(self)
454 def resources_release_qemu (self):
455 "run site-dependant release script as defined in LocalTestResources"
456 from LocalTestResources import local_resources
457 return local_resources.step_release_qemu(self)
460 "vserver delete the test myplc"
461 self.run_in_host("vserver --silent %s delete"%self.vservername)
465 # historically the build was being fetched by the tests
466 # now the build pushes itself as a subdir of the tests workdir
467 # so that the tests do not have to worry about extracting the build (svn, git, or whatever)
468 def create_vs (self):
469 "vserver creation (no install done)"
470 # push the local build/ dir to the testplc box
472 # a full path for the local calls
473 build_dir=os.path.dirname(sys.argv[0])
474 # sometimes this is empty - set to "." in such a case
475 if not build_dir: build_dir="."
476 build_dir += "/build"
478 # use a standard name - will be relative to remote buildname
480 # remove for safety; do *not* mkdir first, otherwise we end up with build/build/
481 self.test_ssh.rmdir(build_dir)
482 self.test_ssh.copy(build_dir,recursive=True)
483 # the repo url is taken from arch-rpms-url
484 # with the last step (i386) removed
485 repo_url = self.options.arch_rpms_url
486 for level in [ 'arch' ]:
487 repo_url = os.path.dirname(repo_url)
488 # pass the vbuild-nightly options to vtest-init-vserver
490 test_env_options += " -p %s"%self.options.personality
491 test_env_options += " -d %s"%self.options.pldistro
492 test_env_options += " -f %s"%self.options.fcdistro
493 script="vtest-init-vserver.sh"
494 vserver_name = self.vservername
495 vserver_options="--netdev eth0 --interface %s"%self.vserverip
497 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
498 vserver_options += " --hostname %s"%vserver_hostname
500 print "Cannot reverse lookup %s"%self.vserverip
501 print "This is considered fatal, as this might pollute the test results"
503 create_vserver="%(build_dir)s/%(script)s %(test_env_options)s %(vserver_name)s %(repo_url)s -- %(vserver_options)s"%locals()
504 return self.run_in_host(create_vserver) == 0
508 "yum install myplc, noderepo, and the plain bootstrapfs"
510 # workaround for getting pgsql8.2 on centos5
511 if self.options.fcdistro == "centos5":
512 self.run_in_guest("rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-3.noarch.rpm")
514 if self.options.personality == "linux32":
516 elif self.options.personality == "linux64":
519 raise Exception, "Unsupported personality %r"%self.options.personality
521 nodefamily="%s-%s-%s"%(self.options.pldistro,self.options.fcdistro,arch)
523 # try to install slicerepo - not fatal yet
524 self.run_in_guest("yum -y install slicerepo-%s"%nodefamily)
527 self.run_in_guest("yum -y install myplc")==0 and \
528 self.run_in_guest("yum -y install noderepo-%s"%nodefamily)==0 and \
529 self.run_in_guest("yum -y install bootstrapfs-%s-plain"%nodefamily)==0
534 tmpname='%s.plc-config-tty'%(self.name())
535 fileconf=open(tmpname,'w')
536 for var in [ 'PLC_NAME',
540 'PLC_MAIL_SUPPORT_ADDRESS',
543 # Above line was added for integrating SFA Testing
549 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
550 fileconf.write('w\n')
551 fileconf.write('q\n')
553 utils.system('cat %s'%tmpname)
554 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
555 utils.system('rm %s'%tmpname)
560 self.run_in_guest('service plc start')
565 self.run_in_guest('service plc stop')
569 "start the PLC vserver"
573 # stores the keys from the config for further use
574 def store_keys(self):
575 "stores test users ssh keys in keys/"
576 for key_spec in self.plc_spec['keys']:
577 TestKey(self,key_spec).store_key()
580 def clean_keys(self):
581 "removes keys cached in keys/"
582 utils.system("rm -rf %s/keys/"%os.path(sys.argv[0]))
584 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
585 # for later direct access to the nodes
586 def fetch_keys(self):
587 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
589 if not os.path.isdir(dir):
591 vservername=self.vservername
593 prefix = 'debug_ssh_key'
594 for ext in [ 'pub', 'rsa' ] :
595 src="/vservers/%(vservername)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
596 dst="keys/%(vservername)s-debug.%(ext)s"%locals()
597 if self.test_ssh.fetch(src,dst) != 0: overall=False
601 "create sites with PLCAPI"
602 return self.do_sites()
604 def clean_sites (self):
605 "delete sites with PLCAPI"
606 return self.do_sites(action="delete")
608 def do_sites (self,action="add"):
609 for site_spec in self.plc_spec['sites']:
610 test_site = TestSite (self,site_spec)
611 if (action != "add"):
612 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
613 test_site.delete_site()
614 # deleted with the site
615 #test_site.delete_users()
618 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
619 test_site.create_site()
620 test_site.create_users()
623 def clean_all_sites (self):
624 "Delete all sites in PLC, and related objects"
625 print 'auth_root',self.auth_root()
626 site_ids = [s['site_id'] for s in self.apiserver.GetSites(self.auth_root(), {}, ['site_id'])]
627 for site_id in site_ids:
628 print 'Deleting site_id',site_id
629 self.apiserver.DeleteSite(self.auth_root(),site_id)
632 "create nodes with PLCAPI"
633 return self.do_nodes()
634 def clean_nodes (self):
635 "delete nodes with PLCAPI"
636 return self.do_nodes(action="delete")
638 def do_nodes (self,action="add"):
639 for site_spec in self.plc_spec['sites']:
640 test_site = TestSite (self,site_spec)
642 utils.header("Deleting nodes in site %s"%test_site.name())
643 for node_spec in site_spec['nodes']:
644 test_node=TestNode(self,test_site,node_spec)
645 utils.header("Deleting %s"%test_node.name())
646 test_node.delete_node()
648 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
649 for node_spec in site_spec['nodes']:
650 utils.pprint('Creating node %s'%node_spec,node_spec)
651 test_node = TestNode (self,test_site,node_spec)
652 test_node.create_node ()
655 def nodegroups (self):
656 "create nodegroups with PLCAPI"
657 return self.do_nodegroups("add")
658 def clean_nodegroups (self):
659 "delete nodegroups with PLCAPI"
660 return self.do_nodegroups("delete")
664 def translate_timestamp (start,timestamp):
665 if timestamp < TestPlc.YEAR: return start+timestamp
666 else: return timestamp
669 def timestamp_printable (timestamp):
670 return time.strftime('%m-%d %H:%M:%S UTC',time.gmtime(timestamp))
673 "create leases (on reservable nodes only, use e.g. run -c default -c resa)"
675 grain=self.apiserver.GetLeaseGranularity(self.auth_root())
676 round_time=(now/grain)*grain
677 start=round_time+grain
678 # find out all nodes that are reservable
679 nodes=self.all_reservable_nodenames()
681 utils.header ("No reservable node found - proceeding without leases")
684 # attach them to the leases as specified in plc_specs
685 # this is where the 'leases' field gets interpreted as relative of absolute
686 for lease_spec in self.plc_spec['leases']:
687 # skip the ones that come with a null slice id
688 if not lease_spec['slice']: continue
689 lease_spec['t_from']=TestPlc.translate_timestamp(start,lease_spec['t_from'])
690 lease_spec['t_until']=TestPlc.translate_timestamp(start,lease_spec['t_until'])
691 lease_addition=self.apiserver.AddLeases(self.auth_root(),nodes,
692 lease_spec['slice'],lease_spec['t_from'],lease_spec['t_until'])
693 if lease_addition['errors']:
694 utils.header("Cannot create leases, %s"%lease_addition['errors'])
697 utils.header('Leases on nodes %r for %s from %d (%s) until %d (%s)'%\
698 (nodes,lease_spec['slice'],
699 lease_spec['t_from'],TestPlc.timestamp_printable(lease_spec['t_from']),
700 lease_spec['t_until'],TestPlc.timestamp_printable(lease_spec['t_until'])))
704 def clean_leases (self):
705 "remove all leases in the myplc side"
706 lease_ids= [ l['lease_id'] for l in self.apiserver.GetLeases(self.auth_root())]
707 utils.header("Cleaning leases %r"%lease_ids)
708 self.apiserver.DeleteLeases(self.auth_root(),lease_ids)
711 def list_leases (self):
712 "list all leases known to the myplc"
713 leases = self.apiserver.GetLeases(self.auth_root())
715 utils.header("%s %s from %s until %s"%(l['hostname'],l['name'],
716 TestPlc.timestamp_printable(l['t_from']),
717 TestPlc.timestamp_printable(l['t_until'])))
720 # create nodegroups if needed, and populate
721 def do_nodegroups (self, action="add"):
722 # 1st pass to scan contents
724 for site_spec in self.plc_spec['sites']:
725 test_site = TestSite (self,site_spec)
726 for node_spec in site_spec['nodes']:
727 test_node=TestNode (self,test_site,node_spec)
728 if node_spec.has_key('nodegroups'):
729 nodegroupnames=node_spec['nodegroups']
730 if isinstance(nodegroupnames,StringTypes):
731 nodegroupnames = [ nodegroupnames ]
732 for nodegroupname in nodegroupnames:
733 if not groups_dict.has_key(nodegroupname):
734 groups_dict[nodegroupname]=[]
735 groups_dict[nodegroupname].append(test_node.name())
736 auth=self.auth_root()
738 for (nodegroupname,group_nodes) in groups_dict.iteritems():
740 print 'nodegroups:','dealing with nodegroup',nodegroupname,'on nodes',group_nodes
741 # first, check if the nodetagtype is here
742 tag_types = self.apiserver.GetTagTypes(auth,{'tagname':nodegroupname})
744 tag_type_id = tag_types[0]['tag_type_id']
746 tag_type_id = self.apiserver.AddTagType(auth,
747 {'tagname':nodegroupname,
748 'description': 'for nodegroup %s'%nodegroupname,
751 print 'located tag (type)',nodegroupname,'as',tag_type_id
753 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
755 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
756 print 'created nodegroup',nodegroupname,'from tagname',nodegroupname,'and value','yes'
757 # set node tag on all nodes, value='yes'
758 for nodename in group_nodes:
760 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
762 traceback.print_exc()
763 print 'node',nodename,'seems to already have tag',nodegroupname
766 expect_yes = self.apiserver.GetNodeTags(auth,
767 {'hostname':nodename,
768 'tagname':nodegroupname},
769 ['value'])[0]['value']
770 if expect_yes != "yes":
771 print 'Mismatch node tag on node',nodename,'got',expect_yes
774 if not self.options.dry_run:
775 print 'Cannot find tag',nodegroupname,'on node',nodename
779 print 'cleaning nodegroup',nodegroupname
780 self.apiserver.DeleteNodeGroup(auth,nodegroupname)
782 traceback.print_exc()
786 # return a list of tuples (nodename,qemuname)
787 def all_node_infos (self) :
789 for site_spec in self.plc_spec['sites']:
790 node_infos += [ (node_spec['node_fields']['hostname'],node_spec['host_box']) \
791 for node_spec in site_spec['nodes'] ]
794 def all_nodenames (self): return [ x[0] for x in self.all_node_infos() ]
795 def all_reservable_nodenames (self):
797 for site_spec in self.plc_spec['sites']:
798 for node_spec in site_spec['nodes']:
799 node_fields=node_spec['node_fields']
800 if 'node_type' in node_fields and node_fields['node_type']=='reservable':
801 res.append(node_fields['hostname'])
804 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
805 def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period=15):
806 if self.options.dry_run:
810 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
811 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
812 # the nodes that haven't checked yet - start with a full list and shrink over time
813 tocheck = self.all_hostnames()
814 utils.header("checking nodes %r"%tocheck)
815 # create a dict hostname -> status
816 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
819 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
821 for array in tocheck_status:
822 hostname=array['hostname']
823 boot_state=array['boot_state']
824 if boot_state == target_boot_state:
825 utils.header ("%s has reached the %s state"%(hostname,target_boot_state))
827 # if it's a real node, never mind
828 (site_spec,node_spec)=self.locate_hostname(hostname)
829 if TestNode.is_real_model(node_spec['node_fields']['model']):
830 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
832 boot_state = target_boot_state
833 elif datetime.datetime.now() > graceout:
834 utils.header ("%s still in '%s' state"%(hostname,boot_state))
835 graceout=datetime.datetime.now()+datetime.timedelta(1)
836 status[hostname] = boot_state
838 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != target_boot_state ]
841 if datetime.datetime.now() > timeout:
842 for hostname in tocheck:
843 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
845 # otherwise, sleep for a while
847 # only useful in empty plcs
850 def nodes_booted(self):
851 return self.nodes_check_boot_state('boot',timeout_minutes=30,silent_minutes=20)
853 def check_nodes_ssh(self,debug,timeout_minutes,silent_minutes,period=15):
855 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
856 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
857 vservername=self.vservername
860 local_key = "keys/%(vservername)s-debug.rsa"%locals()
863 local_key = "keys/key1.rsa"
864 node_infos = self.all_node_infos()
865 utils.header("checking ssh access (expected in %s mode) to nodes:"%message)
866 for (nodename,qemuname) in node_infos:
867 utils.header("hostname=%s -- qemubox=%s"%(nodename,qemuname))
868 utils.header("max timeout is %d minutes, silent for %d minutes (period is %s)"%\
869 (timeout_minutes,silent_minutes,period))
871 for node_info in node_infos:
872 (hostname,qemuname) = node_info
873 # try to run 'hostname' in the node
874 command = TestSsh (hostname,key=local_key).actual_command("hostname;uname -a")
875 # don't spam logs - show the command only after the grace period
876 success = utils.system ( command, silent=datetime.datetime.now() < graceout)
878 utils.header('Successfully entered root@%s (%s)'%(hostname,message))
880 node_infos.remove(node_info)
882 # we will have tried real nodes once, in case they're up - but if not, just skip
883 (site_spec,node_spec)=self.locate_hostname(hostname)
884 if TestNode.is_real_model(node_spec['node_fields']['model']):
885 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
886 node_infos.remove(node_info)
889 if datetime.datetime.now() > timeout:
890 for (hostname,qemuname) in node_infos:
891 utils.header("FAILURE to ssh into %s (on %s)"%(hostname,qemuname))
893 # otherwise, sleep for a while
895 # only useful in empty plcs
898 def nodes_ssh_debug(self):
899 "Tries to ssh into nodes in debug mode with the debug ssh key"
900 return self.check_nodes_ssh(debug=True,timeout_minutes=30,silent_minutes=5)
902 def nodes_ssh_boot(self):
903 "Tries to ssh into nodes in production mode with the root ssh key"
904 return self.check_nodes_ssh(debug=False,timeout_minutes=30,silent_minutes=15)
907 def init_node (self):
908 "all nodes : init a clean local directory for holding node-dep stuff like iso image..."
912 "all nodes: invoke GetBootMedium and store result locally"
915 def configure_qemu (self):
916 "all nodes: compute qemu config qemu.conf and store it locally"
919 def reinstall_node (self):
920 "all nodes: mark PLCAPI boot_state as reinstall"
923 def export_qemu (self):
924 "all nodes: push local node-dep directory on the qemu box"
927 ### check hooks : invoke scripts from hooks/{node,slice}
928 def check_hooks_node (self):
929 return self.locate_first_node().check_hooks()
930 def check_hooks_sliver (self) :
931 return self.locate_first_sliver().check_hooks()
933 def check_hooks (self):
934 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
935 return self.check_hooks_node() and self.check_hooks_sliver()
938 def do_check_initscripts(self):
940 for slice_spec in self.plc_spec['slices']:
941 if not slice_spec.has_key('initscriptname'):
943 initscript=slice_spec['initscriptname']
944 for nodename in slice_spec['nodenames']:
945 (site,node) = self.locate_node (nodename)
946 # xxx - passing the wrong site - probably harmless
947 test_site = TestSite (self,site)
948 test_slice = TestSlice (self,test_site,slice_spec)
949 test_node = TestNode (self,test_site,node)
950 test_sliver = TestSliver (self, test_node, test_slice)
951 if not test_sliver.check_initscript(initscript):
955 def check_initscripts(self):
956 "check that the initscripts have triggered"
957 return self.do_check_initscripts()
959 def initscripts (self):
960 "create initscripts with PLCAPI"
961 for initscript in self.plc_spec['initscripts']:
962 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
963 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
966 def clean_initscripts (self):
967 "delete initscripts with PLCAPI"
968 for initscript in self.plc_spec['initscripts']:
969 initscript_name = initscript['initscript_fields']['name']
970 print('Attempting to delete %s in plc %s'%(initscript_name,self.plc_spec['name']))
972 self.apiserver.DeleteInitScript(self.auth_root(),initscript_name)
973 print initscript_name,'deleted'
975 print 'deletion went wrong - probably did not exist'
980 "create slices with PLCAPI"
981 return self.do_slices()
983 def clean_slices (self):
984 "delete slices with PLCAPI"
985 return self.do_slices("delete")
987 def do_slices (self, action="add"):
988 for slice in self.plc_spec['slices']:
989 site_spec = self.locate_site (slice['sitename'])
990 test_site = TestSite(self,site_spec)
991 test_slice=TestSlice(self,test_site,slice)
993 utils.header("Deleting slices in site %s"%test_site.name())
994 test_slice.delete_slice()
996 utils.pprint("Creating slice",slice)
997 test_slice.create_slice()
998 utils.header('Created Slice %s'%slice['slice_fields']['name'])
1001 @slice_mapper_options
1002 def check_slice(self):
1003 "tries to ssh-enter the slice with the user key, to ensure slice creation"
1007 def clear_known_hosts (self):
1008 "remove test nodes entries from the local known_hosts file"
1012 def start_node (self) :
1013 "all nodes: start the qemu instance (also runs qemu-bridge-init start)"
1016 def check_tcp (self):
1017 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
1018 specs = self.plc_spec['tcp_test']
1023 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
1024 if not s_test_sliver.run_tcp_server(port,timeout=10):
1028 # idem for the client side
1029 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
1030 if not c_test_sliver.run_tcp_client(s_test_sliver.test_node.name(),port):
1034 def plcsh_stress_test (self):
1035 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
1036 # install the stress-test in the plc image
1037 location = "/usr/share/plc_api/plcsh_stress_test.py"
1038 remote="/vservers/%s/%s"%(self.vservername,location)
1039 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1041 command += " -- --check"
1042 if self.options.size == 1:
1043 command += " --tiny"
1044 return ( self.run_in_guest(command) == 0)
1046 # populate runs the same utility without slightly different options
1047 # in particular runs with --preserve (dont cleanup) and without --check
1048 # also it gets run twice, once with the --foreign option for creating fake foreign entries
1051 def install_sfa(self):
1052 "yum install sfa, sfa-plc and sfa-client"
1053 if self.options.personality == "linux32":
1055 elif self.options.personality == "linux64":
1058 raise Exception, "Unsupported personality %r"%self.options.personality
1059 return self.run_in_guest("yum -y install sfa sfa-client sfa-plc sfa-sfatables")==0
1062 def configure_sfa(self):
1063 "run sfa-config-tty"
1064 tmpname='%s.sfa-config-tty'%(self.name())
1065 fileconf=open(tmpname,'w')
1066 for var in [ 'SFA_REGISTRY_ROOT_AUTH',
1067 'SFA_REGISTRY_LEVEL1_AUTH',
1068 'SFA_REGISTRY_HOST',
1069 'SFA_AGGREGATE_HOST',
1075 'SFA_PLC_DB_PASSWORD',
1077 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec['sfa'][var]))
1078 fileconf.write('w\n')
1079 fileconf.write('R\n')
1080 fileconf.write('q\n')
1082 utils.system('cat %s'%tmpname)
1083 self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
1084 utils.system('rm %s'%tmpname)
1087 def import_sfa(self):
1089 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1090 return self.run_in_guest('sfa-import-plc.py')==0
1091 # not needed anymore
1092 # self.run_in_guest('cp /etc/sfa/authorities/%s/%s.pkey /etc/sfa/authorities/server.key'%(auth,auth))
1094 def start_sfa(self):
1096 return self.run_in_guest('service sfa start')==0
1098 def setup_sfa(self):
1099 "sfi client configuration"
1101 if os.path.exists(dir_name):
1102 utils.system('rm -rf %s'%dir_name)
1103 utils.system('mkdir %s'%dir_name)
1104 file_name=dir_name + os.sep + 'fake-pi1.pkey'
1105 fileconf=open(file_name,'w')
1106 fileconf.write (self.plc_spec['keys'][0]['private'])
1109 file_name=dir_name + os.sep + 'sfi_config'
1110 fileconf=open(file_name,'w')
1111 SFI_AUTH=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']+".main"
1112 fileconf.write ("SFI_AUTH='%s'"%SFI_AUTH)
1113 fileconf.write('\n')
1114 SFI_USER=SFI_AUTH+'.fake-pi1'
1115 fileconf.write ("SFI_USER='%s'"%SFI_USER)
1116 fileconf.write('\n')
1117 SFI_REGISTRY='http://' + self.plc_spec['sfa']['SFA_PLC_DB_HOST'] + ':12345/'
1118 fileconf.write ("SFI_REGISTRY='%s'"%SFI_REGISTRY)
1119 fileconf.write('\n')
1120 SFI_SM='http://' + self.plc_spec['sfa']['SFA_PLC_DB_HOST'] + ':12347/'
1121 fileconf.write ("SFI_SM='%s'"%SFI_SM)
1122 fileconf.write('\n')
1125 file_name=dir_name + os.sep + 'person.xml'
1126 fileconf=open(file_name,'w')
1127 for record in self.plc_spec['sfa']['sfa_person_xml']:
1128 person_record=record
1129 fileconf.write(person_record)
1130 fileconf.write('\n')
1133 file_name=dir_name + os.sep + 'slice.xml'
1134 fileconf=open(file_name,'w')
1135 for record in self.plc_spec['sfa']['sfa_slice_xml']:
1137 #slice_record=self.plc_spec['sfa']['sfa_slice_xml']
1138 fileconf.write(slice_record)
1139 fileconf.write('\n')
1142 file_name=dir_name + os.sep + 'slice.rspec'
1143 fileconf=open(file_name,'w')
1145 for (key, value) in self.plc_spec['sfa']['sfa_slice_rspec'].items():
1147 fileconf.write(slice_rspec)
1148 fileconf.write('\n')
1151 remote="/vservers/%s/%s"%(self.vservername,location)
1152 self.test_ssh.copy_abs(dir_name, remote, recursive=True)
1154 #utils.system('cat %s'%tmpname)
1155 utils.system('rm -rf %s'%dir_name)
1159 "run sfi.py add (on Registry) and sfi.py create (on SM) to form new objects"
1161 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1162 success=test_user_sfa.add_user()
1164 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1165 site_spec = self.locate_site (slice_spec['sitename'])
1166 test_site = TestSite(self,site_spec)
1167 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1168 success1=test_slice_sfa.add_slice()
1169 success2=test_slice_sfa.create_slice()
1170 return success and success1 and success2
1172 def update_sfa(self):
1173 "run sfi.py update (on Registry) and sfi.py create (on SM) on existing objects"
1175 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1176 success1=test_user_sfa.update_user()
1178 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1179 site_spec = self.locate_site (slice_spec['sitename'])
1180 test_site = TestSite(self,site_spec)
1181 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1182 success2=test_slice_sfa.update_slice()
1183 return success1 and success2
1186 "run sfi.py list and sfi.py show (both on Registry) and sfi.py slices and sfi.py resources (both on SM)"
1187 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1189 self.run_in_guest("sfi.py -d /root/.sfi/ list %s.main"%auth)==0 and \
1190 self.run_in_guest("sfi.py -d /root/.sfi/ show %s.main"%auth)==0 and \
1191 self.run_in_guest("sfi.py -d /root/.sfi/ slices")==0 and \
1192 self.run_in_guest("sfi.py -d /root/.sfi/ resources -o resources")==0
1194 @slice_mapper_options_sfa
1195 def check_slice_sfa(self):
1196 "tries to ssh-enter the SFA slice"
1199 def delete_sfa(self):
1200 "run sfi.py delete (on SM), sfi.py remove (on Registry)"
1202 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1203 success1=test_user_sfa.delete_user()
1204 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1205 site_spec = self.locate_site (slice_spec['sitename'])
1206 test_site = TestSite(self,site_spec)
1207 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1208 success2=test_slice_sfa.delete_slice()
1210 return success1 and success2
1214 return self.run_in_guest('service sfa stop')==0
1216 def populate (self):
1217 "creates random entries in the PLCAPI"
1218 # install the stress-test in the plc image
1219 location = "/usr/share/plc_api/plcsh_stress_test.py"
1220 remote="/vservers/%s/%s"%(self.vservername,location)
1221 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1223 command += " -- --preserve --short-names"
1224 local = (self.run_in_guest(command) == 0);
1225 # second run with --foreign
1226 command += ' --foreign'
1227 remote = (self.run_in_guest(command) == 0);
1228 return ( local and remote)
1230 def gather_logs (self):
1231 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1232 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1233 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1234 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1235 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1236 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1238 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1239 self.gather_var_logs ()
1241 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1242 self.gather_pgsql_logs ()
1244 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1245 for site_spec in self.plc_spec['sites']:
1246 test_site = TestSite (self,site_spec)
1247 for node_spec in site_spec['nodes']:
1248 test_node=TestNode(self,test_site,node_spec)
1249 test_node.gather_qemu_logs()
1251 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1252 self.gather_nodes_var_logs()
1254 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1255 self.gather_slivers_var_logs()
1258 def gather_slivers_var_logs(self):
1259 for test_sliver in self.all_sliver_objs():
1260 remote = test_sliver.tar_var_logs()
1261 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
1262 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
1263 utils.system(command)
1266 def gather_var_logs (self):
1267 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
1268 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1269 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
1270 utils.system(command)
1271 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd"%self.name()
1272 utils.system(command)
1274 def gather_pgsql_logs (self):
1275 utils.system("mkdir -p logs/myplc.pgsql-log.%s"%self.name())
1276 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1277 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -"%self.name()
1278 utils.system(command)
1280 def gather_nodes_var_logs (self):
1281 for site_spec in self.plc_spec['sites']:
1282 test_site = TestSite (self,site_spec)
1283 for node_spec in site_spec['nodes']:
1284 test_node=TestNode(self,test_site,node_spec)
1285 test_ssh = TestSsh (test_node.name(),key="keys/key1.rsa")
1286 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1287 command = command + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
1288 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
1289 utils.system(command)
1292 # returns the filename to use for sql dump/restore, using options.dbname if set
1293 def dbfile (self, database):
1294 # uses options.dbname if it is found
1296 name=self.options.dbname
1297 if not isinstance(name,StringTypes):
1300 t=datetime.datetime.now()
1303 return "/root/%s-%s.sql"%(database,name)
1306 'dump the planetlab5 DB in /root in the PLC - filename has time'
1307 dump=self.dbfile("planetab5")
1308 self.run_in_guest('pg_dump -U pgsqluser planetlab5 -f '+ dump)
1309 utils.header('Dumped planetlab5 database in %s'%dump)
1312 def db_restore(self):
1313 'restore the planetlab5 DB - looks broken, but run -n might help'
1314 dump=self.dbfile("planetab5")
1315 ##stop httpd service
1316 self.run_in_guest('service httpd stop')
1317 # xxx - need another wrapper
1318 self.run_in_guest_piped('echo drop database planetlab5','psql --user=pgsqluser template1')
1319 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab5')
1320 self.run_in_guest('psql -U pgsqluser planetlab5 -f '+dump)
1321 ##starting httpd service
1322 self.run_in_guest('service httpd start')
1324 utils.header('Database restored from ' + dump)
1327 def standby_1(): pass
1329 def standby_2(): pass
1331 def standby_3(): pass
1333 def standby_4(): pass
1335 def standby_5(): pass
1337 def standby_6(): pass
1339 def standby_7(): pass
1341 def standby_8(): pass
1343 def standby_9(): pass
1345 def standby_10(): pass
1347 def standby_11(): pass
1349 def standby_12(): pass
1351 def standby_13(): pass
1353 def standby_14(): pass
1355 def standby_15(): pass
1357 def standby_16(): pass
1359 def standby_17(): pass
1361 def standby_18(): pass
1363 def standby_19(): pass
1365 def standby_20(): pass