7 from types import StringTypes
11 from TestSite import TestSite
12 from TestNode import TestNode
13 from TestUser import TestUser
14 from TestKey import TestKey
15 from TestSlice import TestSlice
16 from TestSliver import TestSliver
17 from TestBox import TestBox
18 from TestSsh import TestSsh
19 from TestApiserver import TestApiserver
20 from TestSliceSfa import TestSliceSfa
21 from TestUserSfa import TestUserSfa
23 # step methods must take (self) and return a boolean (options is a member of the class)
25 def standby(minutes,dry_run):
26 utils.header('Entering StandBy for %d mn'%minutes)
30 time.sleep(60*minutes)
33 def standby_generic (func):
35 minutes=int(func.__name__.split("_")[1])
36 return standby(minutes,self.options.dry_run)
39 def node_mapper (method):
42 node_method = TestNode.__dict__[method.__name__]
43 for site_spec in self.plc_spec['sites']:
44 test_site = TestSite (self,site_spec)
45 for node_spec in site_spec['nodes']:
46 test_node = TestNode (self,test_site,node_spec)
47 if not node_method(test_node): overall=False
49 # restore the doc text
50 actual.__doc__=method.__doc__
53 def slice_mapper_options (method):
56 slice_method = TestSlice.__dict__[method.__name__]
57 for slice_spec in self.plc_spec['slices']:
58 site_spec = self.locate_site (slice_spec['sitename'])
59 test_site = TestSite(self,site_spec)
60 test_slice=TestSlice(self,test_site,slice_spec)
61 if not slice_method(test_slice,self.options): overall=False
63 # restore the doc text
64 actual.__doc__=method.__doc__
67 def slice_mapper_options_sfa (method):
71 slice_method = TestSliceSfa.__dict__[method.__name__]
72 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
73 site_spec = self.locate_site (slice_spec['sitename'])
74 test_site = TestSite(self,site_spec)
75 test_slice=TestSliceSfa(test_plc,test_site,slice_spec)
76 if not slice_method(test_slice,self.options): overall=False
78 # restore the doc text
79 actual.__doc__=method.__doc__
88 'display', 'resources_pre', SEP,
89 'delete_vs','create_vs','install', 'configure', 'start', SEP,
90 'fetch_keys', 'store_keys', 'clear_known_hosts', SEP,
91 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', 'leases', SEP,
92 'reinstall_node', 'init_node','bootcd', 'configure_qemu', 'export_qemu',
93 'kill_all_qemus', 'start_node', SEP,
94 # better use of time: do this now that the nodes are taking off
95 'plcsh_stress_test', SEP,
96 'install_sfa', 'configure_sfa', 'import_sfa', 'start_sfa', SEPSFA,
97 'setup_sfa', 'add_sfa', 'update_sfa', 'view_sfa', SEPSFA,
98 'nodes_ssh_debug', 'nodes_ssh_boot', 'check_slice', 'check_initscripts', SEPSFA,
99 'check_slice_sfa', 'delete_sfa', 'stop_sfa', SEPSFA,
100 'check_tcp', 'check_hooks', SEP,
101 'force_gather_logs', 'force_resources_post', SEP,
104 'show_boxes', 'resources_list','resources_release','resources_release_plc','resources_release_qemu',SEP,
105 'stop', 'vs_start', SEP,
106 'clean_initscripts', 'clean_nodegroups','clean_all_sites', SEP,
107 'clean_sites', 'clean_nodes', 'clean_slices', 'clean_keys', SEP,
108 'clean_leases', 'list_leases', SEP,
110 'list_all_qemus', 'list_qemus', 'kill_qemus', SEP,
111 'db_dump' , 'db_restore', SEP,
112 'standby_1 through 20',SEP,
116 def printable_steps (list):
117 single_line=" ".join(list)+" "
118 return single_line.replace(" "+SEP+" "," \\\n").replace(" "+SEPSFA+" "," \\\n")
120 def valid_step (step):
121 return step != SEP and step != SEPSFA
123 # turn off the sfa-related steps when build has skipped SFA
124 # this is originally for centos5 as recent SFAs won't build on this platformb
126 def check_whether_build_has_sfa (rpms_url):
127 retcod=os.system ("curl --silent %s/ | grep -q sfa"%rpms_url)
128 # full builds are expected to return with 0 here
130 # move all steps containing 'sfa' from default_steps to other_steps
131 sfa_steps= [ step for step in TestPlc.default_steps if step.find('sfa')>=0 ]
132 TestPlc.other_steps += sfa_steps
133 for step in sfa_steps: TestPlc.default_steps.remove(step)
135 def __init__ (self,plc_spec,options):
136 self.plc_spec=plc_spec
138 self.test_ssh=TestSsh(self.plc_spec['hostname'],self.options.buildname)
140 self.vserverip=plc_spec['vserverip']
141 self.vservername=plc_spec['vservername']
142 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
145 raise Exception,'chroot-based myplc testing is deprecated'
146 self.apiserver=TestApiserver(self.url,options.dry_run)
149 name=self.plc_spec['name']
150 return "%s.%s"%(name,self.vservername)
153 return self.plc_spec['hostname']
156 return self.test_ssh.is_local()
158 # define the API methods on this object through xmlrpc
159 # would help, but not strictly necessary
163 def actual_command_in_guest (self,command):
164 return self.test_ssh.actual_command(self.host_to_guest(command))
166 def start_guest (self):
167 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host()))
169 def run_in_guest (self,command):
170 return utils.system(self.actual_command_in_guest(command))
172 def run_in_host (self,command):
173 return self.test_ssh.run_in_buildname(command)
175 #command gets run in the vserver
176 def host_to_guest(self,command):
177 return "vserver %s exec %s"%(self.vservername,command)
179 #command gets run in the vserver
180 def start_guest_in_host(self):
181 return "vserver %s start"%(self.vservername)
184 def run_in_guest_piped (self,local,remote):
185 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
187 def auth_root (self):
188 return {'Username':self.plc_spec['PLC_ROOT_USER'],
189 'AuthMethod':'password',
190 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
191 'Role' : self.plc_spec['role']
193 def locate_site (self,sitename):
194 for site in self.plc_spec['sites']:
195 if site['site_fields']['name'] == sitename:
197 if site['site_fields']['login_base'] == sitename:
199 raise Exception,"Cannot locate site %s"%sitename
201 def locate_node (self,nodename):
202 for site in self.plc_spec['sites']:
203 for node in site['nodes']:
204 if node['name'] == nodename:
206 raise Exception,"Cannot locate node %s"%nodename
208 def locate_hostname (self,hostname):
209 for site in self.plc_spec['sites']:
210 for node in site['nodes']:
211 if node['node_fields']['hostname'] == hostname:
213 raise Exception,"Cannot locate hostname %s"%hostname
215 def locate_key (self,keyname):
216 for key in self.plc_spec['keys']:
217 if key['name'] == keyname:
219 raise Exception,"Cannot locate key %s"%keyname
221 def locate_slice (self, slicename):
222 for slice in self.plc_spec['slices']:
223 if slice['slice_fields']['name'] == slicename:
225 raise Exception,"Cannot locate slice %s"%slicename
227 def all_sliver_objs (self):
229 for slice_spec in self.plc_spec['slices']:
230 slicename = slice_spec['slice_fields']['name']
231 for nodename in slice_spec['nodenames']:
232 result.append(self.locate_sliver_obj (nodename,slicename))
235 def locate_sliver_obj (self,nodename,slicename):
236 (site,node) = self.locate_node(nodename)
237 slice = self.locate_slice (slicename)
239 test_site = TestSite (self, site)
240 test_node = TestNode (self, test_site,node)
241 # xxx the slice site is assumed to be the node site - mhh - probably harmless
242 test_slice = TestSlice (self, test_site, slice)
243 return TestSliver (self, test_node, test_slice)
245 def locate_first_node(self):
246 nodename=self.plc_spec['slices'][0]['nodenames'][0]
247 (site,node) = self.locate_node(nodename)
248 test_site = TestSite (self, site)
249 test_node = TestNode (self, test_site,node)
252 def locate_first_sliver (self):
253 slice_spec=self.plc_spec['slices'][0]
254 slicename=slice_spec['slice_fields']['name']
255 nodename=slice_spec['nodenames'][0]
256 return self.locate_sliver_obj(nodename,slicename)
258 # all different hostboxes used in this plc
259 def gather_hostBoxes(self):
260 # maps on sites and nodes, return [ (host_box,test_node) ]
262 for site_spec in self.plc_spec['sites']:
263 test_site = TestSite (self,site_spec)
264 for node_spec in site_spec['nodes']:
265 test_node = TestNode (self, test_site, node_spec)
266 if not test_node.is_real():
267 tuples.append( (test_node.host_box(),test_node) )
268 # transform into a dict { 'host_box' -> [ test_node .. ] }
270 for (box,node) in tuples:
271 if not result.has_key(box):
274 result[box].append(node)
277 # a step for checking this stuff
278 def show_boxes (self):
279 'print summary of nodes location'
280 for (box,nodes) in self.gather_hostBoxes().iteritems():
281 print box,":"," + ".join( [ node.name() for node in nodes ] )
284 # make this a valid step
285 def kill_all_qemus(self):
286 'kill all qemu instances on the qemu boxes involved by this setup'
287 # this is the brute force version, kill all qemus on that host box
288 for (box,nodes) in self.gather_hostBoxes().iteritems():
289 # pass the first nodename, as we don't push template-qemu on testboxes
290 nodedir=nodes[0].nodedir()
291 TestBox(box,self.options.buildname).kill_all_qemus(nodedir)
294 # make this a valid step
295 def list_all_qemus(self):
296 'list all qemu instances on the qemu boxes involved by this setup'
297 for (box,nodes) in self.gather_hostBoxes().iteritems():
298 # this is the brute force version, kill all qemus on that host box
299 TestBox(box,self.options.buildname).list_all_qemus()
302 # kill only the right qemus
303 def list_qemus(self):
304 'list qemu instances for our nodes'
305 for (box,nodes) in self.gather_hostBoxes().iteritems():
306 # the fine-grain version
311 # kill only the right qemus
312 def kill_qemus(self):
313 'kill the qemu instances for our nodes'
314 for (box,nodes) in self.gather_hostBoxes().iteritems():
315 # the fine-grain version
320 #################### display config
322 "show test configuration after localization"
323 self.display_pass (1)
324 self.display_pass (2)
328 always_display_keys=['PLC_WWW_HOST','nodes','sites',]
329 def display_pass (self,passno):
330 for (key,val) in self.plc_spec.iteritems():
331 if not self.options.verbose and key not in TestPlc.always_display_keys: continue
335 self.display_site_spec(site)
336 for node in site['nodes']:
337 self.display_node_spec(node)
338 elif key=='initscripts':
339 for initscript in val:
340 self.display_initscript_spec (initscript)
343 self.display_slice_spec (slice)
346 self.display_key_spec (key)
348 if key not in ['sites','initscripts','slices','keys', 'sfa']:
349 print '+ ',key,':',val
351 def display_site_spec (self,site):
352 print '+ ======== site',site['site_fields']['name']
353 for (k,v) in site.iteritems():
354 if not self.options.verbose and k not in TestPlc.always_display_keys: continue
357 print '+ ','nodes : ',
359 print node['node_fields']['hostname'],'',
365 print user['name'],'',
367 elif k == 'site_fields':
368 print '+ login_base',':',v['login_base']
369 elif k == 'address_fields':
375 def display_initscript_spec (self,initscript):
376 print '+ ======== initscript',initscript['initscript_fields']['name']
378 def display_key_spec (self,key):
379 print '+ ======== key',key['name']
381 def display_slice_spec (self,slice):
382 print '+ ======== slice',slice['slice_fields']['name']
383 for (k,v) in slice.iteritems():
396 elif k=='slice_fields':
397 print '+ fields',':',
398 print 'max_nodes=',v['max_nodes'],
403 def display_node_spec (self,node):
404 print "+ node=%s host_box=%s"%(node['name'],node['host_box']),
405 print "hostname=",node['node_fields']['hostname'],
406 print "ip=",node['interface_fields']['ip']
407 if self.options.verbose:
408 utils.pprint("node details",node,depth=3)
410 # another entry point for just showing the boxes involved
411 def display_mapping (self):
412 TestPlc.display_mapping_plc(self.plc_spec)
416 def display_mapping_plc (plc_spec):
417 print '+ MyPLC',plc_spec['name']
418 print '+\tvserver address = root@%s:/vservers/%s'%(plc_spec['hostname'],plc_spec['vservername'])
419 print '+\tIP = %s/%s'%(plc_spec['PLC_API_HOST'],plc_spec['vserverip'])
420 for site_spec in plc_spec['sites']:
421 for node_spec in site_spec['nodes']:
422 TestPlc.display_mapping_node(node_spec)
425 def display_mapping_node (node_spec):
426 print '+ NODE %s'%(node_spec['name'])
427 print '+\tqemu box %s'%node_spec['host_box']
428 print '+\thostname=%s'%node_spec['node_fields']['hostname']
430 def resources_pre (self):
431 "run site-dependant pre-test script as defined in LocalTestResources"
432 from LocalTestResources import local_resources
433 return local_resources.step_pre(self)
435 def resources_post (self):
436 "run site-dependant post-test script as defined in LocalTestResources"
437 from LocalTestResources import local_resources
438 return local_resources.step_post(self)
440 def resources_list (self):
441 "run site-dependant list script as defined in LocalTestResources"
442 from LocalTestResources import local_resources
443 return local_resources.step_list(self)
445 def resources_release (self):
446 "run site-dependant release script as defined in LocalTestResources"
447 from LocalTestResources import local_resources
448 return local_resources.step_release(self)
450 def resources_release_plc (self):
451 "run site-dependant release script as defined in LocalTestResources"
452 from LocalTestResources import local_resources
453 return local_resources.step_release_plc(self)
455 def resources_release_qemu (self):
456 "run site-dependant release script as defined in LocalTestResources"
457 from LocalTestResources import local_resources
458 return local_resources.step_release_qemu(self)
461 "vserver delete the test myplc"
462 self.run_in_host("vserver --silent %s delete"%self.vservername)
466 # historically the build was being fetched by the tests
467 # now the build pushes itself as a subdir of the tests workdir
468 # so that the tests do not have to worry about extracting the build (svn, git, or whatever)
469 def create_vs (self):
470 "vserver creation (no install done)"
471 # push the local build/ dir to the testplc box
473 # a full path for the local calls
474 build_dir=os.path.dirname(sys.argv[0])
475 # sometimes this is empty - set to "." in such a case
476 if not build_dir: build_dir="."
477 build_dir += "/build"
479 # use a standard name - will be relative to remote buildname
481 # remove for safety; do *not* mkdir first, otherwise we end up with build/build/
482 self.test_ssh.rmdir(build_dir)
483 self.test_ssh.copy(build_dir,recursive=True)
484 # the repo url is taken from arch-rpms-url
485 # with the last step (i386) removed
486 repo_url = self.options.arch_rpms_url
487 for level in [ 'arch' ]:
488 repo_url = os.path.dirname(repo_url)
489 # pass the vbuild-nightly options to vtest-init-vserver
491 test_env_options += " -p %s"%self.options.personality
492 test_env_options += " -d %s"%self.options.pldistro
493 test_env_options += " -f %s"%self.options.fcdistro
494 script="vtest-init-vserver.sh"
495 vserver_name = self.vservername
496 vserver_options="--netdev eth0 --interface %s"%self.vserverip
498 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
499 vserver_options += " --hostname %s"%vserver_hostname
501 print "Cannot reverse lookup %s"%self.vserverip
502 print "This is considered fatal, as this might pollute the test results"
504 create_vserver="%(build_dir)s/%(script)s %(test_env_options)s %(vserver_name)s %(repo_url)s -- %(vserver_options)s"%locals()
505 return self.run_in_host(create_vserver) == 0
509 "yum install myplc, noderepo, and the plain bootstrapfs"
511 # workaround for getting pgsql8.2 on centos5
512 if self.options.fcdistro == "centos5":
513 self.run_in_guest("rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-3.noarch.rpm")
515 if self.options.personality == "linux32":
517 elif self.options.personality == "linux64":
520 raise Exception, "Unsupported personality %r"%self.options.personality
522 nodefamily="%s-%s-%s"%(self.options.pldistro,self.options.fcdistro,arch)
524 # try to install slicerepo - not fatal yet
525 self.run_in_guest("yum -y install slicerepo-%s"%nodefamily)
528 self.run_in_guest("yum -y install myplc")==0 and \
529 self.run_in_guest("yum -y install noderepo-%s"%nodefamily)==0 and \
530 self.run_in_guest("yum -y install bootstrapfs-%s-plain"%nodefamily)==0
535 tmpname='%s.plc-config-tty'%(self.name())
536 fileconf=open(tmpname,'w')
537 for var in [ 'PLC_NAME',
541 'PLC_MAIL_SUPPORT_ADDRESS',
544 # Above line was added for integrating SFA Testing
550 'PLC_RESERVATION_GRANULARITY',
553 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
554 fileconf.write('w\n')
555 fileconf.write('q\n')
557 utils.system('cat %s'%tmpname)
558 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
559 utils.system('rm %s'%tmpname)
564 self.run_in_guest('service plc start')
569 self.run_in_guest('service plc stop')
573 "start the PLC vserver"
577 # stores the keys from the config for further use
578 def store_keys(self):
579 "stores test users ssh keys in keys/"
580 for key_spec in self.plc_spec['keys']:
581 TestKey(self,key_spec).store_key()
584 def clean_keys(self):
585 "removes keys cached in keys/"
586 utils.system("rm -rf %s/keys/"%os.path(sys.argv[0]))
588 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
589 # for later direct access to the nodes
590 def fetch_keys(self):
591 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
593 if not os.path.isdir(dir):
595 vservername=self.vservername
597 prefix = 'debug_ssh_key'
598 for ext in [ 'pub', 'rsa' ] :
599 src="/vservers/%(vservername)s/etc/planetlab/%(prefix)s.%(ext)s"%locals()
600 dst="keys/%(vservername)s-debug.%(ext)s"%locals()
601 if self.test_ssh.fetch(src,dst) != 0: overall=False
605 "create sites with PLCAPI"
606 return self.do_sites()
608 def clean_sites (self):
609 "delete sites with PLCAPI"
610 return self.do_sites(action="delete")
612 def do_sites (self,action="add"):
613 for site_spec in self.plc_spec['sites']:
614 test_site = TestSite (self,site_spec)
615 if (action != "add"):
616 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
617 test_site.delete_site()
618 # deleted with the site
619 #test_site.delete_users()
622 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
623 test_site.create_site()
624 test_site.create_users()
627 def clean_all_sites (self):
628 "Delete all sites in PLC, and related objects"
629 print 'auth_root',self.auth_root()
630 site_ids = [s['site_id'] for s in self.apiserver.GetSites(self.auth_root(), {}, ['site_id'])]
631 for site_id in site_ids:
632 print 'Deleting site_id',site_id
633 self.apiserver.DeleteSite(self.auth_root(),site_id)
636 "create nodes with PLCAPI"
637 return self.do_nodes()
638 def clean_nodes (self):
639 "delete nodes with PLCAPI"
640 return self.do_nodes(action="delete")
642 def do_nodes (self,action="add"):
643 for site_spec in self.plc_spec['sites']:
644 test_site = TestSite (self,site_spec)
646 utils.header("Deleting nodes in site %s"%test_site.name())
647 for node_spec in site_spec['nodes']:
648 test_node=TestNode(self,test_site,node_spec)
649 utils.header("Deleting %s"%test_node.name())
650 test_node.delete_node()
652 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
653 for node_spec in site_spec['nodes']:
654 utils.pprint('Creating node %s'%node_spec,node_spec)
655 test_node = TestNode (self,test_site,node_spec)
656 test_node.create_node ()
659 def nodegroups (self):
660 "create nodegroups with PLCAPI"
661 return self.do_nodegroups("add")
662 def clean_nodegroups (self):
663 "delete nodegroups with PLCAPI"
664 return self.do_nodegroups("delete")
668 def translate_timestamp (start,timestamp):
669 if timestamp < TestPlc.YEAR: return start+timestamp
670 else: return timestamp
673 def timestamp_printable (timestamp):
674 return time.strftime('%m-%d %H:%M:%S UTC',time.gmtime(timestamp))
677 "create leases (on reservable nodes only, use e.g. run -c default -c resa)"
679 grain=self.apiserver.GetLeaseGranularity(self.auth_root())
680 round_time=(now/grain)*grain
681 start=round_time+grain
682 # find out all nodes that are reservable
683 nodes=self.all_reservable_nodenames()
685 utils.header ("No reservable node found - proceeding without leases")
688 # attach them to the leases as specified in plc_specs
689 # this is where the 'leases' field gets interpreted as relative of absolute
690 for lease_spec in self.plc_spec['leases']:
691 # skip the ones that come with a null slice id
692 if not lease_spec['slice']: continue
693 lease_spec['t_from']=TestPlc.translate_timestamp(start,lease_spec['t_from'])
694 lease_spec['t_until']=TestPlc.translate_timestamp(start,lease_spec['t_until'])
695 lease_addition=self.apiserver.AddLeases(self.auth_root(),nodes,
696 lease_spec['slice'],lease_spec['t_from'],lease_spec['t_until'])
697 if lease_addition['errors']:
698 utils.header("Cannot create leases, %s"%lease_addition['errors'])
701 utils.header('Leases on nodes %r for %s from %d (%s) until %d (%s)'%\
702 (nodes,lease_spec['slice'],
703 lease_spec['t_from'],TestPlc.timestamp_printable(lease_spec['t_from']),
704 lease_spec['t_until'],TestPlc.timestamp_printable(lease_spec['t_until'])))
708 def clean_leases (self):
709 "remove all leases in the myplc side"
710 lease_ids= [ l['lease_id'] for l in self.apiserver.GetLeases(self.auth_root())]
711 utils.header("Cleaning leases %r"%lease_ids)
712 self.apiserver.DeleteLeases(self.auth_root(),lease_ids)
715 def list_leases (self):
716 "list all leases known to the myplc"
717 leases = self.apiserver.GetLeases(self.auth_root())
720 current=l['t_until']>=now
721 if self.options.verbose or current:
722 utils.header("%s %s from %s until %s"%(l['hostname'],l['name'],
723 TestPlc.timestamp_printable(l['t_from']),
724 TestPlc.timestamp_printable(l['t_until'])))
727 # create nodegroups if needed, and populate
728 def do_nodegroups (self, action="add"):
729 # 1st pass to scan contents
731 for site_spec in self.plc_spec['sites']:
732 test_site = TestSite (self,site_spec)
733 for node_spec in site_spec['nodes']:
734 test_node=TestNode (self,test_site,node_spec)
735 if node_spec.has_key('nodegroups'):
736 nodegroupnames=node_spec['nodegroups']
737 if isinstance(nodegroupnames,StringTypes):
738 nodegroupnames = [ nodegroupnames ]
739 for nodegroupname in nodegroupnames:
740 if not groups_dict.has_key(nodegroupname):
741 groups_dict[nodegroupname]=[]
742 groups_dict[nodegroupname].append(test_node.name())
743 auth=self.auth_root()
745 for (nodegroupname,group_nodes) in groups_dict.iteritems():
747 print 'nodegroups:','dealing with nodegroup',nodegroupname,'on nodes',group_nodes
748 # first, check if the nodetagtype is here
749 tag_types = self.apiserver.GetTagTypes(auth,{'tagname':nodegroupname})
751 tag_type_id = tag_types[0]['tag_type_id']
753 tag_type_id = self.apiserver.AddTagType(auth,
754 {'tagname':nodegroupname,
755 'description': 'for nodegroup %s'%nodegroupname,
758 print 'located tag (type)',nodegroupname,'as',tag_type_id
760 nodegroups = self.apiserver.GetNodeGroups (auth, {'groupname':nodegroupname})
762 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
763 print 'created nodegroup',nodegroupname,'from tagname',nodegroupname,'and value','yes'
764 # set node tag on all nodes, value='yes'
765 for nodename in group_nodes:
767 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
769 traceback.print_exc()
770 print 'node',nodename,'seems to already have tag',nodegroupname
773 expect_yes = self.apiserver.GetNodeTags(auth,
774 {'hostname':nodename,
775 'tagname':nodegroupname},
776 ['value'])[0]['value']
777 if expect_yes != "yes":
778 print 'Mismatch node tag on node',nodename,'got',expect_yes
781 if not self.options.dry_run:
782 print 'Cannot find tag',nodegroupname,'on node',nodename
786 print 'cleaning nodegroup',nodegroupname
787 self.apiserver.DeleteNodeGroup(auth,nodegroupname)
789 traceback.print_exc()
793 # return a list of tuples (nodename,qemuname)
794 def all_node_infos (self) :
796 for site_spec in self.plc_spec['sites']:
797 node_infos += [ (node_spec['node_fields']['hostname'],node_spec['host_box']) \
798 for node_spec in site_spec['nodes'] ]
801 def all_nodenames (self): return [ x[0] for x in self.all_node_infos() ]
802 def all_reservable_nodenames (self):
804 for site_spec in self.plc_spec['sites']:
805 for node_spec in site_spec['nodes']:
806 node_fields=node_spec['node_fields']
807 if 'node_type' in node_fields and node_fields['node_type']=='reservable':
808 res.append(node_fields['hostname'])
811 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
812 def nodes_check_boot_state (self, target_boot_state, timeout_minutes, silent_minutes,period=15):
813 if self.options.dry_run:
817 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
818 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
819 # the nodes that haven't checked yet - start with a full list and shrink over time
820 tocheck = self.all_hostnames()
821 utils.header("checking nodes %r"%tocheck)
822 # create a dict hostname -> status
823 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
826 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
828 for array in tocheck_status:
829 hostname=array['hostname']
830 boot_state=array['boot_state']
831 if boot_state == target_boot_state:
832 utils.header ("%s has reached the %s state"%(hostname,target_boot_state))
834 # if it's a real node, never mind
835 (site_spec,node_spec)=self.locate_hostname(hostname)
836 if TestNode.is_real_model(node_spec['node_fields']['model']):
837 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
839 boot_state = target_boot_state
840 elif datetime.datetime.now() > graceout:
841 utils.header ("%s still in '%s' state"%(hostname,boot_state))
842 graceout=datetime.datetime.now()+datetime.timedelta(1)
843 status[hostname] = boot_state
845 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != target_boot_state ]
848 if datetime.datetime.now() > timeout:
849 for hostname in tocheck:
850 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
852 # otherwise, sleep for a while
854 # only useful in empty plcs
857 def nodes_booted(self):
858 return self.nodes_check_boot_state('boot',timeout_minutes=30,silent_minutes=20)
860 def check_nodes_ssh(self,debug,timeout_minutes,silent_minutes,period=15):
862 timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes)
863 graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes)
864 vservername=self.vservername
867 local_key = "keys/%(vservername)s-debug.rsa"%locals()
870 local_key = "keys/key1.rsa"
871 node_infos = self.all_node_infos()
872 utils.header("checking ssh access (expected in %s mode) to nodes:"%message)
873 for (nodename,qemuname) in node_infos:
874 utils.header("hostname=%s -- qemubox=%s"%(nodename,qemuname))
875 utils.header("max timeout is %d minutes, silent for %d minutes (period is %s)"%\
876 (timeout_minutes,silent_minutes,period))
878 for node_info in node_infos:
879 (hostname,qemuname) = node_info
880 # try to run 'hostname' in the node
881 command = TestSsh (hostname,key=local_key).actual_command("hostname;uname -a")
882 # don't spam logs - show the command only after the grace period
883 success = utils.system ( command, silent=datetime.datetime.now() < graceout)
885 utils.header('Successfully entered root@%s (%s)'%(hostname,message))
887 node_infos.remove(node_info)
889 # we will have tried real nodes once, in case they're up - but if not, just skip
890 (site_spec,node_spec)=self.locate_hostname(hostname)
891 if TestNode.is_real_model(node_spec['node_fields']['model']):
892 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
893 node_infos.remove(node_info)
896 if datetime.datetime.now() > timeout:
897 for (hostname,qemuname) in node_infos:
898 utils.header("FAILURE to ssh into %s (on %s)"%(hostname,qemuname))
900 # otherwise, sleep for a while
902 # only useful in empty plcs
905 def nodes_ssh_debug(self):
906 "Tries to ssh into nodes in debug mode with the debug ssh key"
907 return self.check_nodes_ssh(debug=True,timeout_minutes=10,silent_minutes=5)
909 def nodes_ssh_boot(self):
910 "Tries to ssh into nodes in production mode with the root ssh key"
911 return self.check_nodes_ssh(debug=False,timeout_minutes=40,silent_minutes=15)
914 def init_node (self):
915 "all nodes : init a clean local directory for holding node-dep stuff like iso image..."
919 "all nodes: invoke GetBootMedium and store result locally"
922 def configure_qemu (self):
923 "all nodes: compute qemu config qemu.conf and store it locally"
926 def reinstall_node (self):
927 "all nodes: mark PLCAPI boot_state as reinstall"
930 def export_qemu (self):
931 "all nodes: push local node-dep directory on the qemu box"
934 ### check hooks : invoke scripts from hooks/{node,slice}
935 def check_hooks_node (self):
936 return self.locate_first_node().check_hooks()
937 def check_hooks_sliver (self) :
938 return self.locate_first_sliver().check_hooks()
940 def check_hooks (self):
941 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
942 return self.check_hooks_node() and self.check_hooks_sliver()
945 def do_check_initscripts(self):
947 for slice_spec in self.plc_spec['slices']:
948 if not slice_spec.has_key('initscriptname'):
950 initscript=slice_spec['initscriptname']
951 for nodename in slice_spec['nodenames']:
952 (site,node) = self.locate_node (nodename)
953 # xxx - passing the wrong site - probably harmless
954 test_site = TestSite (self,site)
955 test_slice = TestSlice (self,test_site,slice_spec)
956 test_node = TestNode (self,test_site,node)
957 test_sliver = TestSliver (self, test_node, test_slice)
958 if not test_sliver.check_initscript(initscript):
962 def check_initscripts(self):
963 "check that the initscripts have triggered"
964 return self.do_check_initscripts()
966 def initscripts (self):
967 "create initscripts with PLCAPI"
968 for initscript in self.plc_spec['initscripts']:
969 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
970 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
973 def clean_initscripts (self):
974 "delete initscripts with PLCAPI"
975 for initscript in self.plc_spec['initscripts']:
976 initscript_name = initscript['initscript_fields']['name']
977 print('Attempting to delete %s in plc %s'%(initscript_name,self.plc_spec['name']))
979 self.apiserver.DeleteInitScript(self.auth_root(),initscript_name)
980 print initscript_name,'deleted'
982 print 'deletion went wrong - probably did not exist'
987 "create slices with PLCAPI"
988 return self.do_slices()
990 def clean_slices (self):
991 "delete slices with PLCAPI"
992 return self.do_slices("delete")
994 def do_slices (self, action="add"):
995 for slice in self.plc_spec['slices']:
996 site_spec = self.locate_site (slice['sitename'])
997 test_site = TestSite(self,site_spec)
998 test_slice=TestSlice(self,test_site,slice)
1000 utils.header("Deleting slices in site %s"%test_site.name())
1001 test_slice.delete_slice()
1003 utils.pprint("Creating slice",slice)
1004 test_slice.create_slice()
1005 utils.header('Created Slice %s'%slice['slice_fields']['name'])
1008 @slice_mapper_options
1009 def check_slice(self):
1010 "tries to ssh-enter the slice with the user key, to ensure slice creation"
1014 def clear_known_hosts (self):
1015 "remove test nodes entries from the local known_hosts file"
1019 def start_node (self) :
1020 "all nodes: start the qemu instance (also runs qemu-bridge-init start)"
1023 def check_tcp (self):
1024 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
1025 specs = self.plc_spec['tcp_test']
1030 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
1031 if not s_test_sliver.run_tcp_server(port,timeout=10):
1035 # idem for the client side
1036 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
1037 if not c_test_sliver.run_tcp_client(s_test_sliver.test_node.name(),port):
1041 def plcsh_stress_test (self):
1042 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
1043 # install the stress-test in the plc image
1044 location = "/usr/share/plc_api/plcsh_stress_test.py"
1045 remote="/vservers/%s/%s"%(self.vservername,location)
1046 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1048 command += " -- --check"
1049 if self.options.size == 1:
1050 command += " --tiny"
1051 return ( self.run_in_guest(command) == 0)
1053 # populate runs the same utility without slightly different options
1054 # in particular runs with --preserve (dont cleanup) and without --check
1055 # also it gets run twice, once with the --foreign option for creating fake foreign entries
1058 def install_sfa(self):
1059 "yum install sfa, sfa-plc and sfa-client"
1060 if self.options.personality == "linux32":
1062 elif self.options.personality == "linux64":
1065 raise Exception, "Unsupported personality %r"%self.options.personality
1066 return self.run_in_guest("yum -y install sfa sfa-client sfa-plc sfa-sfatables")==0
1069 def configure_sfa(self):
1070 "run sfa-config-tty"
1071 tmpname='%s.sfa-config-tty'%(self.name())
1072 fileconf=open(tmpname,'w')
1073 for var in [ 'SFA_REGISTRY_ROOT_AUTH',
1074 'SFA_REGISTRY_LEVEL1_AUTH',
1075 'SFA_REGISTRY_HOST',
1076 'SFA_AGGREGATE_HOST',
1082 'SFA_PLC_DB_PASSWORD',
1084 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec['sfa'][var]))
1085 fileconf.write('w\n')
1086 fileconf.write('R\n')
1087 fileconf.write('q\n')
1089 utils.system('cat %s'%tmpname)
1090 self.run_in_guest_piped('cat %s'%tmpname,'sfa-config-tty')
1091 utils.system('rm %s'%tmpname)
1094 def import_sfa(self):
1096 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1097 return self.run_in_guest('sfa-import-plc.py')==0
1098 # not needed anymore
1099 # self.run_in_guest('cp /etc/sfa/authorities/%s/%s.pkey /etc/sfa/authorities/server.key'%(auth,auth))
1101 def start_sfa(self):
1103 return self.run_in_guest('service sfa start')==0
1105 def setup_sfa(self):
1106 "sfi client configuration"
1108 if os.path.exists(dir_name):
1109 utils.system('rm -rf %s'%dir_name)
1110 utils.system('mkdir %s'%dir_name)
1111 file_name=dir_name + os.sep + 'fake-pi1.pkey'
1112 fileconf=open(file_name,'w')
1113 fileconf.write (self.plc_spec['keys'][0]['private'])
1116 file_name=dir_name + os.sep + 'sfi_config'
1117 fileconf=open(file_name,'w')
1118 SFI_AUTH=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']+".main"
1119 fileconf.write ("SFI_AUTH='%s'"%SFI_AUTH)
1120 fileconf.write('\n')
1121 SFI_USER=SFI_AUTH+'.fake-pi1'
1122 fileconf.write ("SFI_USER='%s'"%SFI_USER)
1123 fileconf.write('\n')
1124 SFI_REGISTRY='http://' + self.plc_spec['sfa']['SFA_PLC_DB_HOST'] + ':12345/'
1125 fileconf.write ("SFI_REGISTRY='%s'"%SFI_REGISTRY)
1126 fileconf.write('\n')
1127 SFI_SM='http://' + self.plc_spec['sfa']['SFA_PLC_DB_HOST'] + ':12347/'
1128 fileconf.write ("SFI_SM='%s'"%SFI_SM)
1129 fileconf.write('\n')
1132 file_name=dir_name + os.sep + 'person.xml'
1133 fileconf=open(file_name,'w')
1134 for record in self.plc_spec['sfa']['sfa_person_xml']:
1135 person_record=record
1136 fileconf.write(person_record)
1137 fileconf.write('\n')
1140 file_name=dir_name + os.sep + 'slice.xml'
1141 fileconf=open(file_name,'w')
1142 for record in self.plc_spec['sfa']['sfa_slice_xml']:
1144 #slice_record=self.plc_spec['sfa']['sfa_slice_xml']
1145 fileconf.write(slice_record)
1146 fileconf.write('\n')
1149 file_name=dir_name + os.sep + 'slice.rspec'
1150 fileconf=open(file_name,'w')
1152 for (key, value) in self.plc_spec['sfa']['sfa_slice_rspec'].items():
1154 fileconf.write(slice_rspec)
1155 fileconf.write('\n')
1158 remote="/vservers/%s/%s"%(self.vservername,location)
1159 self.test_ssh.copy_abs(dir_name, remote, recursive=True)
1161 #utils.system('cat %s'%tmpname)
1162 utils.system('rm -rf %s'%dir_name)
1166 "run sfi.py add (on Registry) and sfi.py create (on SM) to form new objects"
1168 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1169 success=test_user_sfa.add_user()
1171 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1172 site_spec = self.locate_site (slice_spec['sitename'])
1173 test_site = TestSite(self,site_spec)
1174 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1175 success1=test_slice_sfa.add_slice()
1176 success2=test_slice_sfa.create_slice()
1177 return success and success1 and success2
1179 def update_sfa(self):
1180 "run sfi.py update (on Registry) and sfi.py create (on SM) on existing objects"
1182 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1183 success1=test_user_sfa.update_user()
1185 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1186 site_spec = self.locate_site (slice_spec['sitename'])
1187 test_site = TestSite(self,site_spec)
1188 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1189 success2=test_slice_sfa.update_slice()
1190 return success1 and success2
1193 "run sfi.py list and sfi.py show (both on Registry) and sfi.py slices and sfi.py resources (both on SM)"
1194 auth=self.plc_spec['sfa']['SFA_REGISTRY_ROOT_AUTH']
1196 self.run_in_guest("sfi.py -d /root/.sfi/ list %s.main"%auth)==0 and \
1197 self.run_in_guest("sfi.py -d /root/.sfi/ show %s.main"%auth)==0 and \
1198 self.run_in_guest("sfi.py -d /root/.sfi/ slices")==0 and \
1199 self.run_in_guest("sfi.py -d /root/.sfi/ resources -o resources")==0
1201 @slice_mapper_options_sfa
1202 def check_slice_sfa(self):
1203 "tries to ssh-enter the SFA slice"
1206 def delete_sfa(self):
1207 "run sfi.py delete (on SM), sfi.py remove (on Registry)"
1209 test_user_sfa=TestUserSfa(test_plc,self.plc_spec['sfa'])
1210 success1=test_user_sfa.delete_user()
1211 for slice_spec in self.plc_spec['sfa']['slices_sfa']:
1212 site_spec = self.locate_site (slice_spec['sitename'])
1213 test_site = TestSite(self,site_spec)
1214 test_slice_sfa=TestSliceSfa(test_plc,test_site,slice_spec)
1215 success2=test_slice_sfa.delete_slice()
1217 return success1 and success2
1221 return self.run_in_guest('service sfa stop')==0
1223 def populate (self):
1224 "creates random entries in the PLCAPI"
1225 # install the stress-test in the plc image
1226 location = "/usr/share/plc_api/plcsh_stress_test.py"
1227 remote="/vservers/%s/%s"%(self.vservername,location)
1228 self.test_ssh.copy_abs("plcsh_stress_test.py",remote)
1230 command += " -- --preserve --short-names"
1231 local = (self.run_in_guest(command) == 0);
1232 # second run with --foreign
1233 command += ' --foreign'
1234 remote = (self.run_in_guest(command) == 0);
1235 return ( local and remote)
1237 def gather_logs (self):
1238 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1239 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1240 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1241 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1242 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1243 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1245 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1246 self.gather_var_logs ()
1248 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1249 self.gather_pgsql_logs ()
1251 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1252 for site_spec in self.plc_spec['sites']:
1253 test_site = TestSite (self,site_spec)
1254 for node_spec in site_spec['nodes']:
1255 test_node=TestNode(self,test_site,node_spec)
1256 test_node.gather_qemu_logs()
1258 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1259 self.gather_nodes_var_logs()
1261 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1262 self.gather_slivers_var_logs()
1265 def gather_slivers_var_logs(self):
1266 for test_sliver in self.all_sliver_objs():
1267 remote = test_sliver.tar_var_logs()
1268 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
1269 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
1270 utils.system(command)
1273 def gather_var_logs (self):
1274 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
1275 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1276 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
1277 utils.system(command)
1278 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd"%self.name()
1279 utils.system(command)
1281 def gather_pgsql_logs (self):
1282 utils.system("mkdir -p logs/myplc.pgsql-log.%s"%self.name())
1283 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1284 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -"%self.name()
1285 utils.system(command)
1287 def gather_nodes_var_logs (self):
1288 for site_spec in self.plc_spec['sites']:
1289 test_site = TestSite (self,site_spec)
1290 for node_spec in site_spec['nodes']:
1291 test_node=TestNode(self,test_site,node_spec)
1292 test_ssh = TestSsh (test_node.name(),key="keys/key1.rsa")
1293 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1294 command = command + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
1295 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
1296 utils.system(command)
1299 # returns the filename to use for sql dump/restore, using options.dbname if set
1300 def dbfile (self, database):
1301 # uses options.dbname if it is found
1303 name=self.options.dbname
1304 if not isinstance(name,StringTypes):
1307 t=datetime.datetime.now()
1310 return "/root/%s-%s.sql"%(database,name)
1313 'dump the planetlab5 DB in /root in the PLC - filename has time'
1314 dump=self.dbfile("planetab5")
1315 self.run_in_guest('pg_dump -U pgsqluser planetlab5 -f '+ dump)
1316 utils.header('Dumped planetlab5 database in %s'%dump)
1319 def db_restore(self):
1320 'restore the planetlab5 DB - looks broken, but run -n might help'
1321 dump=self.dbfile("planetab5")
1322 ##stop httpd service
1323 self.run_in_guest('service httpd stop')
1324 # xxx - need another wrapper
1325 self.run_in_guest_piped('echo drop database planetlab5','psql --user=pgsqluser template1')
1326 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab5')
1327 self.run_in_guest('psql -U pgsqluser planetlab5 -f '+dump)
1328 ##starting httpd service
1329 self.run_in_guest('service httpd start')
1331 utils.header('Database restored from ' + dump)
1334 def standby_1(): pass
1336 def standby_2(): pass
1338 def standby_3(): pass
1340 def standby_4(): pass
1342 def standby_5(): pass
1344 def standby_6(): pass
1346 def standby_7(): pass
1348 def standby_8(): pass
1350 def standby_9(): pass
1352 def standby_10(): pass
1354 def standby_11(): pass
1356 def standby_12(): pass
1358 def standby_13(): pass
1360 def standby_14(): pass
1362 def standby_15(): pass
1364 def standby_16(): pass
1366 def standby_17(): pass
1368 def standby_18(): pass
1370 def standby_19(): pass
1372 def standby_20(): pass