7 from types import StringTypes
11 from TestSite import TestSite
12 from TestNode import TestNode
13 from TestUser import TestUser
14 from TestKey import TestKey
15 from TestSlice import TestSlice
16 from TestSliver import TestSliver
17 from TestBox import TestBox
18 from TestSsh import TestSsh
19 from TestApiserver import TestApiserver
21 # step methods must take (self) and return a boolean (options is a member of the class)
23 def standby(minutes,dry_run):
24 utils.header('Entering StandBy for %d mn'%minutes)
28 time.sleep(60*minutes)
31 def standby_generic (func):
33 minutes=int(func.__name__.split("_")[1])
34 return standby(minutes,self.options.dry_run)
37 def node_mapper (method):
40 node_method = TestNode.__dict__[method.__name__]
41 for site_spec in self.plc_spec['sites']:
42 test_site = TestSite (self,site_spec)
43 for node_spec in site_spec['nodes']:
44 test_node = TestNode (self,test_site,node_spec)
45 if not node_method(test_node): overall=False
49 def slice_mapper_options (method):
52 slice_method = TestSlice.__dict__[method.__name__]
53 for slice_spec in self.plc_spec['slices']:
54 site_spec = self.locate_site (slice_spec['sitename'])
55 test_site = TestSite(self,site_spec)
56 test_slice=TestSlice(self,test_site,slice_spec)
57 if not slice_method(test_slice,self.options): overall=False
65 default_steps = ['uninstall','install','install_rpm',
66 'configure', 'start', SEP,
67 'store_keys', 'clear_known_hosts', 'initscripts', SEP,
68 'sites', 'nodes', 'slices', 'nodegroups', SEP,
69 'init_node','bootcd', 'configure_qemu', 'export_qemu',
70 'kill_all_qemus', 'reinstall_node','start_node', SEP,
71 'nodes_booted', 'nodes_ssh', 'check_slice',
72 'check_initscripts', 'check_tcp',SEP,
73 'force_gather_logs', 'force_kill_qemus', 'force_record_tracker','force_free_tracker' ]
74 other_steps = [ 'stop_all_vservers','fresh_install', 'cache_rpm', 'stop', SEP,
75 'clean_sites', 'clean_nodes', 'clean_slices', 'clean_keys', SEP,
76 'show_boxes', 'list_all_qemus', 'list_qemus', SEP,
77 'db_dump' , 'db_restore', ' cleanup_tracker',
78 'standby_1 through 20'
82 def printable_steps (list):
83 return " ".join(list).replace(" "+SEP+" "," \\\n")
85 def valid_step (step):
88 def __init__ (self,plc_spec,options):
89 self.plc_spec=plc_spec
91 self.test_ssh=TestSsh(self.plc_spec['hostname'],self.options.buildname)
93 self.vserverip=plc_spec['vserverip']
94 self.vservername=plc_spec['vservername']
95 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
98 raise Exception,'chroot-based myplc testing is deprecated'
99 self.apiserver=TestApiserver(self.url,options.dry_run)
102 name=self.plc_spec['name']
103 return "%s.%s"%(name,self.vservername)
106 return self.plc_spec['hostname']
109 return self.test_ssh.is_local()
111 # define the API methods on this object through xmlrpc
112 # would help, but not strictly necessary
116 def actual_command_in_guest (self,command):
117 return self.test_ssh.actual_command(self.host_to_guest(command))
119 def run_in_guest (self,command):
120 return utils.system(self.actual_command_in_guest(command))
122 def run_in_host (self,command):
123 return self.test_ssh.run_in_buildname(command)
125 #command gets run in the vserver
126 def host_to_guest(self,command):
127 return "vserver %s exec %s"%(self.vservername,command)
130 def run_in_guest_piped (self,local,remote):
131 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
133 def auth_root (self):
134 return {'Username':self.plc_spec['PLC_ROOT_USER'],
135 'AuthMethod':'password',
136 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
137 'Role' : self.plc_spec['role']
139 def locate_site (self,sitename):
140 for site in self.plc_spec['sites']:
141 if site['site_fields']['name'] == sitename:
143 if site['site_fields']['login_base'] == sitename:
145 raise Exception,"Cannot locate site %s"%sitename
147 def locate_node (self,nodename):
148 for site in self.plc_spec['sites']:
149 for node in site['nodes']:
150 if node['name'] == nodename:
152 raise Exception,"Cannot locate node %s"%nodename
154 def locate_hostname (self,hostname):
155 for site in self.plc_spec['sites']:
156 for node in site['nodes']:
157 if node['node_fields']['hostname'] == hostname:
159 raise Exception,"Cannot locate hostname %s"%hostname
161 def locate_key (self,keyname):
162 for key in self.plc_spec['keys']:
163 if key['name'] == keyname:
165 raise Exception,"Cannot locate key %s"%keyname
167 def locate_slice (self, slicename):
168 for slice in self.plc_spec['slices']:
169 if slice['slice_fields']['name'] == slicename:
171 raise Exception,"Cannot locate slice %s"%slicename
173 # all different hostboxes used in this plc
174 def gather_hostBoxes(self):
175 # maps on sites and nodes, return [ (host_box,test_node) ]
177 for site_spec in self.plc_spec['sites']:
178 test_site = TestSite (self,site_spec)
179 for node_spec in site_spec['nodes']:
180 test_node = TestNode (self, test_site, node_spec)
181 if not test_node.is_real():
182 tuples.append( (test_node.host_box(),test_node) )
183 # transform into a dict { 'host_box' -> [ test_node .. ] }
185 for (box,node) in tuples:
186 if not result.has_key(box):
189 result[box].append(node)
192 # a step for checking this stuff
193 def show_boxes (self):
194 for (box,nodes) in self.gather_hostBoxes().iteritems():
195 print box,":"," + ".join( [ node.name() for node in nodes ] )
198 # make this a valid step
199 def kill_all_qemus(self):
200 # this is the brute force version, kill all qemus on that host box
201 for (box,nodes) in self.gather_hostBoxes().iteritems():
202 # pass the first nodename, as we don't push template-qemu on testboxes
203 nodedir=nodes[0].nodedir()
204 TestBox(box,self.options.buildname).kill_all_qemus(nodedir)
207 # make this a valid step
208 def list_all_qemus(self):
209 for (box,nodes) in self.gather_hostBoxes().iteritems():
210 # this is the brute force version, kill all qemus on that host box
211 TestBox(box,self.options.buildname).list_all_qemus()
214 # kill only the right qemus
215 def list_qemus(self):
216 for (box,nodes) in self.gather_hostBoxes().iteritems():
217 # the fine-grain version
222 # kill only the right qemus
223 def kill_qemus(self):
224 for (box,nodes) in self.gather_hostBoxes().iteritems():
225 # the fine-grain version
231 ### utility methods for handling the pool of IP addresses allocated to plcs
233 # (*) running plcs are recorded in the file named ~/running-test-plcs
234 # (*) this file contains a line for each running plc, older first
235 # (*) each line contains the vserver name + the hostname of the (vserver) testbox where it sits
236 # (*) the free_tracker method performs a vserver stop on the oldest entry
237 # (*) the record_tracker method adds an entry at the bottom of the file
238 # (*) the cleanup_tracker method stops all known vservers and removes the tracker file
240 TRACKER_FILE="~/running-test-plcs"
242 def record_tracker (self):
243 command="echo %s %s >> %s"%(self.vservername,self.test_ssh.hostname,TestPlc.TRACKER_FILE)
244 (code,output) = utils.output_of (self.test_ssh.actual_command(command))
246 print "WARNING : COULD NOT record_tracker %s as a running plc on %s"%(self.vservername,self.test_ssh.hostname)
248 print "Recorded %s in running plcs on host %s"%(self.vservername,self.test_ssh.hostname)
251 def free_tracker (self):
252 command="head -1 %s"%TestPlc.TRACKER_FILE
253 (code,line) = utils.output_of(self.test_ssh.actual_command(command))
255 print "No entry found in %s on %s"%(TestPlc.TRACKER_FILE,self.test_ssh.hostname)
258 [vserver_to_stop,hostname] = line.split()
260 print "WARNING: free_tracker: Could not parse %s - skipped"%TestPlc.TRACKER_FILE
262 stop_command = "vserver --silent %s stop"%vserver_to_stop
263 utils.system(self.test_ssh.actual_command(stop_command))
264 x=TestPlc.TRACKER_FILE
265 flush_command = "tail --lines=+2 %s > %s.tmp ; mv %s.tmp %s"%(x,x,x,x)
266 utils.system(self.test_ssh.actual_command(flush_command))
269 # this should/could stop only the ones in TRACKER_FILE if that turns out to be reliable
270 def cleanup_tracker (self):
271 stop_all = "cd /vservers ; for i in * ; do vserver --silent $i stop ; done"
272 utils.system(self.test_ssh.actual_command(stop_all))
273 clean_tracker = "rm -f %s"%TestPlc.TRACKER_FILE
274 utils.system(self.test_ssh.actual_command(clean_tracker))
277 self.run_in_host("vserver --silent %s delete"%self.vservername)
282 # we need build dir for vtest-init-vserver
284 # a full path for the local calls
285 build_dir=os.path.dirname(sys.argv[0])+"/build"
287 # use a standard name - will be relative to remote buildname
289 # run checkout in any case - would do an update if already exists
290 build_checkout = "svn checkout %s %s"%(self.options.build_url,build_dir)
291 if self.run_in_host(build_checkout) != 0:
293 # the repo url is taken from arch-rpms-url
294 # with the last step (i386.) removed
295 repo_url = self.options.arch_rpms_url
296 for level in [ 'arch' ]:
297 repo_url = os.path.dirname(repo_url)
298 if self.options.arch == "i386":
299 personality_option="-p linux32"
301 personality_option="-p linux64"
302 script="vtest-init-vserver.sh"
303 vserver_name = self.vservername
304 vserver_options="--netdev eth0 --interface %s"%self.vserverip
306 vserver_hostname=socket.gethostbyaddr(self.vserverip)[0]
307 vserver_options += " --hostname %s"%vserver_hostname
310 create_vserver="%(build_dir)s/%(script)s %(personality_option)s %(vserver_name)s %(repo_url)s -- %(vserver_options)s"%locals()
311 return self.run_in_host(create_vserver) == 0
314 def install_rpm(self):
315 return self.run_in_guest("yum -y install myplc-native")==0
319 tmpname='%s.plc-config-tty'%(self.name())
320 fileconf=open(tmpname,'w')
321 for var in [ 'PLC_NAME',
325 'PLC_MAIL_SUPPORT_ADDRESS',
332 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
333 fileconf.write('w\n')
334 fileconf.write('q\n')
336 utils.system('cat %s'%tmpname)
337 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
338 utils.system('rm %s'%tmpname)
342 self.run_in_guest('service plc start')
346 self.run_in_guest('service plc stop')
349 # could use a TestKey class
350 def store_keys(self):
351 for key_spec in self.plc_spec['keys']:
352 TestKey(self,key_spec).store_key()
355 def clean_keys(self):
356 utils.system("rm -rf %s/keys/"%os.path(sys.argv[0]))
359 return self.do_sites()
361 def clean_sites (self):
362 return self.do_sites(action="delete")
364 def do_sites (self,action="add"):
365 for site_spec in self.plc_spec['sites']:
366 test_site = TestSite (self,site_spec)
367 if (action != "add"):
368 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
369 test_site.delete_site()
370 # deleted with the site
371 #test_site.delete_users()
374 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
375 test_site.create_site()
376 test_site.create_users()
380 return self.do_nodes()
381 def clean_nodes (self):
382 return self.do_nodes(action="delete")
384 def do_nodes (self,action="add"):
385 for site_spec in self.plc_spec['sites']:
386 test_site = TestSite (self,site_spec)
388 utils.header("Deleting nodes in site %s"%test_site.name())
389 for node_spec in site_spec['nodes']:
390 test_node=TestNode(self,test_site,node_spec)
391 utils.header("Deleting %s"%test_node.name())
392 test_node.delete_node()
394 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
395 for node_spec in site_spec['nodes']:
396 utils.pprint('Creating node %s'%node_spec,node_spec)
397 test_node = TestNode (self,test_site,node_spec)
398 test_node.create_node ()
401 # create nodegroups if needed, and populate
402 # no need for a clean_nodegroups if we are careful enough
403 def nodegroups (self):
404 # 1st pass to scan contents
406 for site_spec in self.plc_spec['sites']:
407 test_site = TestSite (self,site_spec)
408 for node_spec in site_spec['nodes']:
409 test_node=TestNode (self,test_site,node_spec)
410 if node_spec.has_key('nodegroups'):
411 nodegroupnames=node_spec['nodegroups']
412 if isinstance(nodegroupnames,StringTypes):
413 nodegroupnames = [ nodegroupnames ]
414 for nodegroupname in nodegroupnames:
415 if not groups_dict.has_key(nodegroupname):
416 groups_dict[nodegroupname]=[]
417 groups_dict[nodegroupname].append(test_node.name())
418 auth=self.auth_root()
419 for (nodegroupname,group_nodes) in groups_dict.iteritems():
421 self.apiserver.GetNodeGroups(auth,{'name':nodegroupname})[0]
423 self.apiserver.AddNodeGroup(auth,{'name':nodegroupname})
424 for node in group_nodes:
425 self.apiserver.AddNodeToNodeGroup(auth,node,nodegroupname)
428 def all_hostnames (self) :
430 for site_spec in self.plc_spec['sites']:
431 hostnames += [ node_spec['node_fields']['hostname'] \
432 for node_spec in site_spec['nodes'] ]
435 # gracetime : during the first <gracetime> minutes nothing gets printed
436 def do_nodes_booted (self, minutes, gracetime,period=30):
437 if self.options.dry_run:
441 timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
442 graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
443 # the nodes that haven't checked yet - start with a full list and shrink over time
444 tocheck = self.all_hostnames()
445 utils.header("checking nodes %r"%tocheck)
446 # create a dict hostname -> status
447 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
450 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
452 for array in tocheck_status:
453 hostname=array['hostname']
454 boot_state=array['boot_state']
455 if boot_state == 'boot':
456 utils.header ("%s has reached the 'boot' state"%hostname)
458 # if it's a real node, never mind
459 (site_spec,node_spec)=self.locate_hostname(hostname)
460 if TestNode.is_real_model(node_spec['node_fields']['model']):
461 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
464 elif datetime.datetime.now() > graceout:
465 utils.header ("%s still in '%s' state"%(hostname,boot_state))
466 graceout=datetime.datetime.now()+datetime.timedelta(1)
467 status[hostname] = boot_state
469 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
472 if datetime.datetime.now() > timeout:
473 for hostname in tocheck:
474 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
476 # otherwise, sleep for a while
478 # only useful in empty plcs
481 def nodes_booted(self):
482 return self.do_nodes_booted(minutes=20,gracetime=15)
484 def do_nodes_ssh(self,minutes,gracetime,period=30):
486 timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
487 graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
488 tocheck = self.all_hostnames()
489 # self.scan_publicKeys(tocheck)
490 utils.header("checking Connectivity on nodes %r"%tocheck)
492 for hostname in tocheck:
493 # try to ssh in nodes
494 node_test_ssh = TestSsh (hostname,key="/etc/planetlab/root_ssh_key.rsa")
495 success=self.run_in_guest(node_test_ssh.actual_command("hostname"))==0
497 utils.header('The node %s is sshable -->'%hostname)
499 tocheck.remove(hostname)
501 # we will have tried real nodes once, in case they're up - but if not, just skip
502 (site_spec,node_spec)=self.locate_hostname(hostname)
503 if TestNode.is_real_model(node_spec['node_fields']['model']):
504 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
505 tocheck.remove(hostname)
506 elif datetime.datetime.now() > graceout:
507 utils.header("Could not ssh-enter root context on %s"%hostname)
510 if datetime.datetime.now() > timeout:
511 for hostname in tocheck:
512 utils.header("FAILURE to ssh into %s"%hostname)
514 # otherwise, sleep for a while
516 # only useful in empty plcs
520 return self.do_nodes_ssh(minutes=6,gracetime=4)
523 def init_node (self): pass
525 def bootcd (self): pass
527 def configure_qemu (self): pass
529 def reinstall_node (self): pass
531 def export_qemu (self): pass
533 def do_check_initscripts(self):
535 for slice_spec in self.plc_spec['slices']:
536 if not slice_spec.has_key('initscriptname'):
538 initscript=slice_spec['initscriptname']
539 for nodename in slice_spec['nodenames']:
540 (site,node) = self.locate_node (nodename)
541 # xxx - passing the wrong site - probably harmless
542 test_site = TestSite (self,site)
543 test_slice = TestSlice (self,test_site,slice_spec)
544 test_node = TestNode (self,test_site,node)
545 test_sliver = TestSliver (self, test_node, test_slice)
546 if not test_sliver.check_initscript(initscript):
550 def check_initscripts(self):
551 return self.do_check_initscripts()
553 def initscripts (self):
554 for initscript in self.plc_spec['initscripts']:
555 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
556 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
560 return self.do_slices()
562 def clean_slices (self):
563 return self.do_slices("delete")
565 def do_slices (self, action="add"):
566 for slice in self.plc_spec['slices']:
567 site_spec = self.locate_site (slice['sitename'])
568 test_site = TestSite(self,site_spec)
569 test_slice=TestSlice(self,test_site,slice)
571 utils.header("Deleting slices in site %s"%test_site.name())
572 test_slice.delete_slice()
574 utils.pprint("Creating slice",slice)
575 test_slice.create_slice()
576 utils.header('Created Slice %s'%slice['slice_fields']['name'])
579 @slice_mapper_options
580 def check_slice(self): pass
583 def clear_known_hosts (self): pass
586 def start_node (self) : pass
588 def all_sliver_objs (self):
590 for slice_spec in self.plc_spec['slices']:
591 slicename = slice_spec['slice_fields']['name']
592 for nodename in slice_spec['nodenames']:
593 result.append(self.locate_sliver_obj (nodename,slicename))
596 def locate_sliver_obj (self,nodename,slicename):
597 (site,node) = self.locate_node(nodename)
598 slice = self.locate_slice (slicename)
600 test_site = TestSite (self, site)
601 test_node = TestNode (self, test_site,node)
602 # xxx the slice site is assumed to be the node site - mhh - probably harmless
603 test_slice = TestSlice (self, test_site, slice)
604 return TestSliver (self, test_node, test_slice)
606 def check_tcp (self):
607 specs = self.plc_spec['tcp_test']
612 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
613 if not s_test_sliver.run_tcp_server(port,timeout=10):
617 # idem for the client side
618 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
619 if not c_test_sliver.run_tcp_client(s_test_sliver.test_node.name(),port):
624 def gather_logs (self):
625 # (1) get the plc's /var/log and store it locally in logs/myplc.var-log.<plcname>/*
626 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
627 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
628 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
630 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
631 self.gather_var_logs ()
633 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
634 for site_spec in self.plc_spec['sites']:
635 test_site = TestSite (self,site_spec)
636 for node_spec in site_spec['nodes']:
637 test_node=TestNode(self,test_site,node_spec)
638 test_node.gather_qemu_logs()
640 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
641 self.gather_nodes_var_logs()
643 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
644 self.gather_slivers_var_logs()
647 def gather_slivers_var_logs(self):
648 for test_sliver in self.all_sliver_objs():
649 remote = test_sliver.tar_var_logs()
650 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
651 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
652 utils.system(command)
655 def gather_var_logs (self):
656 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
657 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
658 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
659 utils.system(command)
661 def gather_nodes_var_logs (self):
662 for site_spec in self.plc_spec['sites']:
663 test_site = TestSite (self,site_spec)
664 for node_spec in site_spec['nodes']:
665 test_node=TestNode(self,test_site,node_spec)
666 test_ssh = TestSsh (test_node.name(),key="/etc/planetlab/root_ssh_key.rsa")
667 to_plc = self.actual_command_in_guest ( test_ssh.actual_command("tar -C /var/log -cf - ."))
668 command = to_plc + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
669 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
670 utils.system(command)
673 # returns the filename to use for sql dump/restore, using options.dbname if set
674 def dbfile (self, database):
675 # uses options.dbname if it is found
677 name=self.options.dbname
678 if not isinstance(name,StringTypes):
681 t=datetime.datetime.now()
684 return "/root/%s-%s.sql"%(database,name)
687 dump=self.dbfile("planetab4")
688 self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
689 utils.header('Dumped planetlab4 database in %s'%dump)
692 def db_restore(self):
693 dump=self.dbfile("planetab4")
695 self.run_in_guest('service httpd stop')
696 # xxx - need another wrapper
697 self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
698 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
699 self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
700 ##starting httpd service
701 self.run_in_guest('service httpd start')
703 utils.header('Database restored from ' + dump)
706 def standby_1(): pass
708 def standby_2(): pass
710 def standby_3(): pass
712 def standby_4(): pass
714 def standby_5(): pass
716 def standby_6(): pass
718 def standby_7(): pass
720 def standby_8(): pass
722 def standby_9(): pass
724 def standby_10(): pass
726 def standby_11(): pass
728 def standby_12(): pass
730 def standby_13(): pass
732 def standby_14(): pass
734 def standby_15(): pass
736 def standby_16(): pass
738 def standby_17(): pass
740 def standby_18(): pass
742 def standby_19(): pass
744 def standby_20(): pass