7 from types import StringTypes
10 from TestSite import TestSite
11 from TestNode import TestNode
12 from TestUser import TestUser
13 from TestKey import TestKey
14 from TestSlice import TestSlice
15 from TestSliver import TestSliver
16 from TestBox import TestBox
17 from TestSsh import TestSsh
18 from TestApiserver import TestApiserver
20 # step methods must take (self) and return a boolean (options is a member of the class)
22 def standby(minutes,dry_run):
23 utils.header('Entering StandBy for %d mn'%minutes)
27 time.sleep(60*minutes)
30 def standby_generic (func):
32 minutes=int(func.__name__.split("_")[1])
33 return standby(minutes,self.options.dry_run)
36 def node_mapper (method):
39 node_method = TestNode.__dict__[method.__name__]
40 for site_spec in self.plc_spec['sites']:
41 test_site = TestSite (self,site_spec)
42 for node_spec in site_spec['nodes']:
43 test_node = TestNode (self,test_site,node_spec)
44 if not node_method(test_node): overall=False
48 def slice_mapper_options (method):
51 slice_method = TestSlice.__dict__[method.__name__]
52 for slice_spec in self.plc_spec['slices']:
53 site_spec = self.locate_site (slice_spec['sitename'])
54 test_site = TestSite(self,site_spec)
55 test_slice=TestSlice(self,test_site,slice_spec)
56 if not slice_method(test_slice,self.options): overall=False
64 default_steps = ['uninstall','install','install_rpm',
65 'configure', 'start', SEP,
66 'store_keys', 'clear_known_hosts', 'initscripts', SEP,
67 'sites', 'nodes', 'slices', 'nodegroups', SEP,
68 'init_node','bootcd', 'configure_qemu', 'export_qemu',
69 'kill_all_qemus', 'reinstall_node','start_node', SEP,
70 'nodes_booted', 'nodes_ssh', 'check_slice',
71 'check_initscripts', 'check_tcp',SEP,
72 'force_gather_logs', 'force_kill_qemus', ]
73 other_steps = [ 'stop_all_vservers','fresh_install', 'cache_rpm', 'stop', SEP,
74 'clean_sites', 'clean_nodes', 'clean_slices', 'clean_keys', SEP,
75 'show_boxes', 'list_all_qemus', 'list_qemus', SEP,
76 'db_dump' , 'db_restore',
77 'standby_1 through 20'
81 def printable_steps (list):
82 return " ".join(list).replace(" "+SEP+" ","\n")
84 def valid_step (step):
87 def __init__ (self,plc_spec,options):
88 self.plc_spec=plc_spec
90 self.test_ssh=TestSsh(self.plc_spec['hostname'],self.options.buildname)
92 self.vserverip=plc_spec['vserverip']
93 self.vservername=plc_spec['vservername']
94 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
98 self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
99 # utils.header('Using API url %s'%self.url)
100 self.apiserver=TestApiserver(self.url,options.dry_run)
103 name=self.plc_spec['name']
105 return name+".vserver.%s"%self.vservername
107 return name+".chroot"
110 return self.plc_spec['hostname']
113 return self.test_ssh.is_local()
115 # define the API methods on this object through xmlrpc
116 # would help, but not strictly necessary
120 def actual_command_in_guest (self,command):
121 return self.test_ssh.actual_command(self.host_to_guest(command))
123 def run_in_guest (self,command):
124 return utils.system(self.actual_command_in_guest(command))
126 def run_in_host (self,command):
127 return self.test_ssh.run_in_buildname(command)
129 #command gets run in the chroot/vserver
130 def host_to_guest(self,command):
132 return "vserver %s exec %s"%(self.vservername,command)
134 return "chroot /plc/root %s"%TestSsh.backslash_shell_specials(command)
136 # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
137 def copy_in_guest (self, localfile, remotefile, in_data=False):
139 chroot_dest="/plc/data"
141 chroot_dest="/plc/root"
144 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
146 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
149 utils.system("scp %s %s:%s/%s"%(localfile,self.hostname(),chroot_dest,remotefile))
151 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.hostname(),self.vservername,remotefile))
155 def run_in_guest_piped (self,local,remote):
156 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
158 def auth_root (self):
159 return {'Username':self.plc_spec['PLC_ROOT_USER'],
160 'AuthMethod':'password',
161 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
162 'Role' : self.plc_spec['role']
164 def locate_site (self,sitename):
165 for site in self.plc_spec['sites']:
166 if site['site_fields']['name'] == sitename:
168 if site['site_fields']['login_base'] == sitename:
170 raise Exception,"Cannot locate site %s"%sitename
172 def locate_node (self,nodename):
173 for site in self.plc_spec['sites']:
174 for node in site['nodes']:
175 if node['name'] == nodename:
177 raise Exception,"Cannot locate node %s"%nodename
179 def locate_hostname (self,hostname):
180 for site in self.plc_spec['sites']:
181 for node in site['nodes']:
182 if node['node_fields']['hostname'] == hostname:
184 raise Exception,"Cannot locate hostname %s"%hostname
186 def locate_key (self,keyname):
187 for key in self.plc_spec['keys']:
188 if key['name'] == keyname:
190 raise Exception,"Cannot locate key %s"%keyname
192 def locate_slice (self, slicename):
193 for slice in self.plc_spec['slices']:
194 if slice['slice_fields']['name'] == slicename:
196 raise Exception,"Cannot locate slice %s"%slicename
198 # all different hostboxes used in this plc
199 def gather_hostBoxes(self):
200 # maps on sites and nodes, return [ (host_box,test_node) ]
202 for site_spec in self.plc_spec['sites']:
203 test_site = TestSite (self,site_spec)
204 for node_spec in site_spec['nodes']:
205 test_node = TestNode (self, test_site, node_spec)
206 if not test_node.is_real():
207 tuples.append( (test_node.host_box(),test_node) )
208 # transform into a dict { 'host_box' -> [ test_node .. ] }
210 for (box,node) in tuples:
211 if not result.has_key(box):
214 result[box].append(node)
217 # a step for checking this stuff
218 def show_boxes (self):
219 for (box,nodes) in self.gather_hostBoxes().iteritems():
220 print box,":"," + ".join( [ node.name() for node in nodes ] )
223 # make this a valid step
224 def kill_all_qemus(self):
225 # this is the brute force version, kill all qemus on that host box
226 for (box,nodes) in self.gather_hostBoxes().iteritems():
227 # pass the first nodename, as we don't push template-qemu on testboxes
228 nodedir=nodes[0].nodedir()
229 TestBox(box,self.options.buildname).kill_all_qemus(nodedir)
232 # make this a valid step
233 def list_all_qemus(self):
234 for (box,nodes) in self.gather_hostBoxes().iteritems():
235 # this is the brute force version, kill all qemus on that host box
236 TestBox(box,self.options.buildname).list_all_qemus()
239 # kill only the right qemus
240 def list_qemus(self):
241 for (box,nodes) in self.gather_hostBoxes().iteritems():
242 # the fine-grain version
247 # kill only the right qemus
248 def kill_qemus(self):
249 for (box,nodes) in self.gather_hostBoxes().iteritems():
250 # the fine-grain version
255 #################### step methods
258 def uninstall_chroot(self):
259 self.run_in_host('service plc safestop')
260 #####detecting the last myplc version installed and remove it
261 self.run_in_host('rpm -e myplc')
262 ##### Clean up the /plc directory
263 self.run_in_host('rm -rf /plc/data')
264 ##### stop any running vservers
265 self.run_in_host('for vserver in $(ls -d /vservers/* | sed -e s,/vservers/,,) ; do case $vserver in vtest*) echo Shutting down vserver $vserver ; vserver $vserver stop ;; esac ; done')
268 def uninstall_vserver(self):
269 self.run_in_host("vserver --silent %s delete"%self.vservername)
273 # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
274 # it sounds safer to have the former uninstalled too
275 # now the vserver method cannot be invoked for chroot instances as vservername is required
277 self.uninstall_vserver()
278 self.uninstall_chroot()
280 self.uninstall_chroot()
284 def install_chroot(self):
288 def install_vserver(self):
289 # we need build dir for vtest-init-vserver
291 # a full path for the local calls
292 build_dir=os.path.dirname(sys.argv[0])+"/build"
294 # use a standard name - will be relative to HOME
295 build_dir="options.buildname"
296 # run checkout in any case - would do an update if already exists
297 build_checkout = "svn checkout %s %s"%(self.options.build_url,build_dir)
298 if self.run_in_host(build_checkout) != 0:
300 # the repo url is taken from myplc-url
301 # with the last two steps (i386/myplc...) removed
302 repo_url = self.options.myplc_url
303 for level in [ 'rpmname','arch' ]:
304 repo_url = os.path.dirname(repo_url)
305 if self.options.arch == "i386":
306 personality="-p linux32"
308 personality="-p linux64"
309 create_vserver="%s/vtest-init-vserver.sh %s %s %s -- --interface eth0:%s"%\
310 (build_dir,personality,self.vservername,repo_url,self.vserverip)
311 return self.run_in_host(create_vserver) == 0
315 return self.install_vserver()
317 return self.install_chroot()
319 ### install_rpm - make this an optional step
321 url = self.options.myplc_url
322 rpm = os.path.basename(url)
323 cache_fetch="pwd;if [ -f %(rpm)s ] ; then echo Using cached rpm %(rpm)s ; else echo Fetching %(url)s ; curl -O %(url)s; fi"%locals()
324 return self.run_in_host(cache_fetch)==0
326 def install_rpm_chroot(self):
327 url = self.options.myplc_url
328 rpm = os.path.basename(url)
329 if not self.cache_rpm():
331 utils.header('Installing the : %s'%rpm)
332 return self.run_in_host('rpm -Uvh '+rpm)==0 and self.run_in_host('service plc mount')==0
334 def install_rpm_vserver(self):
335 return self.run_in_guest("yum -y install myplc-native")==0
337 def install_rpm(self):
339 return self.install_rpm_vserver()
341 return self.install_rpm_chroot()
345 tmpname='%s.plc-config-tty'%(self.name())
346 fileconf=open(tmpname,'w')
347 for var in [ 'PLC_NAME',
351 'PLC_MAIL_SUPPORT_ADDRESS',
358 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
359 fileconf.write('w\n')
360 fileconf.write('q\n')
362 utils.system('cat %s'%tmpname)
363 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
364 utils.system('rm %s'%tmpname)
367 # the chroot install is slightly different to this respect
370 self.run_in_guest('service plc start')
372 self.run_in_host('service plc start')
377 self.run_in_guest('service plc stop')
379 self.run_in_host('service plc stop')
382 # could use a TestKey class
383 def store_keys(self):
384 for key_spec in self.plc_spec['keys']:
385 TestKey(self,key_spec).store_key()
388 def clean_keys(self):
389 utils.system("rm -rf %s/keys/"%os.path(sys.argv[0]))
392 return self.do_sites()
394 def clean_sites (self):
395 return self.do_sites(action="delete")
397 def do_sites (self,action="add"):
398 for site_spec in self.plc_spec['sites']:
399 test_site = TestSite (self,site_spec)
400 if (action != "add"):
401 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
402 test_site.delete_site()
403 # deleted with the site
404 #test_site.delete_users()
407 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
408 test_site.create_site()
409 test_site.create_users()
413 return self.do_nodes()
414 def clean_nodes (self):
415 return self.do_nodes(action="delete")
417 def do_nodes (self,action="add"):
418 for site_spec in self.plc_spec['sites']:
419 test_site = TestSite (self,site_spec)
421 utils.header("Deleting nodes in site %s"%test_site.name())
422 for node_spec in site_spec['nodes']:
423 test_node=TestNode(self,test_site,node_spec)
424 utils.header("Deleting %s"%test_node.name())
425 test_node.delete_node()
427 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
428 for node_spec in site_spec['nodes']:
429 utils.pprint('Creating node %s'%node_spec,node_spec)
430 test_node = TestNode (self,test_site,node_spec)
431 test_node.create_node ()
434 # create nodegroups if needed, and populate
435 # no need for a clean_nodegroups if we are careful enough
436 def nodegroups (self):
437 # 1st pass to scan contents
439 for site_spec in self.plc_spec['sites']:
440 test_site = TestSite (self,site_spec)
441 for node_spec in site_spec['nodes']:
442 test_node=TestNode (self,test_site,node_spec)
443 if node_spec.has_key('nodegroups'):
444 nodegroupnames=node_spec['nodegroups']
445 if isinstance(nodegroupnames,StringTypes):
446 nodegroupnames = [ nodegroupnames ]
447 for nodegroupname in nodegroupnames:
448 if not groups_dict.has_key(nodegroupname):
449 groups_dict[nodegroupname]=[]
450 groups_dict[nodegroupname].append(test_node.name())
451 auth=self.auth_root()
452 for (nodegroupname,group_nodes) in groups_dict.iteritems():
454 self.apiserver.GetNodeGroups(auth,{'name':nodegroupname})[0]
456 self.apiserver.AddNodeGroup(auth,{'name':nodegroupname})
457 for node in group_nodes:
458 self.apiserver.AddNodeToNodeGroup(auth,node,nodegroupname)
461 def all_hostnames (self) :
463 for site_spec in self.plc_spec['sites']:
464 hostnames += [ node_spec['node_fields']['hostname'] \
465 for node_spec in site_spec['nodes'] ]
468 # gracetime : during the first <gracetime> minutes nothing gets printed
469 def do_nodes_booted (self, minutes, gracetime,period=30):
470 if self.options.dry_run:
474 timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
475 graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
476 # the nodes that haven't checked yet - start with a full list and shrink over time
477 tocheck = self.all_hostnames()
478 utils.header("checking nodes %r"%tocheck)
479 # create a dict hostname -> status
480 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
483 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
485 for array in tocheck_status:
486 hostname=array['hostname']
487 boot_state=array['boot_state']
488 if boot_state == 'boot':
489 utils.header ("%s has reached the 'boot' state"%hostname)
491 # if it's a real node, never mind
492 (site_spec,node_spec)=self.locate_hostname(hostname)
493 if TestNode.is_real_model(node_spec['node_fields']['model']):
494 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
497 elif datetime.datetime.now() > graceout:
498 utils.header ("%s still in '%s' state"%(hostname,boot_state))
499 graceout=datetime.datetime.now()+datetime.timedelta(1)
500 status[hostname] = boot_state
502 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
505 if datetime.datetime.now() > timeout:
506 for hostname in tocheck:
507 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
509 # otherwise, sleep for a while
511 # only useful in empty plcs
514 def nodes_booted(self):
515 return self.do_nodes_booted(minutes=20,gracetime=15)
517 def do_nodes_ssh(self,minutes,gracetime,period=30):
519 timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
520 graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
521 tocheck = self.all_hostnames()
522 # self.scan_publicKeys(tocheck)
523 utils.header("checking Connectivity on nodes %r"%tocheck)
525 for hostname in tocheck:
526 # try to ssh in nodes
527 node_test_ssh = TestSsh (hostname,key="/etc/planetlab/root_ssh_key.rsa")
528 success=self.run_in_guest(node_test_ssh.actual_command("hostname"))==0
530 utils.header('The node %s is sshable -->'%hostname)
532 tocheck.remove(hostname)
534 # we will have tried real nodes once, in case they're up - but if not, just skip
535 (site_spec,node_spec)=self.locate_hostname(hostname)
536 if TestNode.is_real_model(node_spec['node_fields']['model']):
537 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
538 tocheck.remove(hostname)
539 elif datetime.datetime.now() > graceout:
540 utils.header("Could not ssh-enter root context on %s"%hostname)
543 if datetime.datetime.now() > timeout:
544 for hostname in tocheck:
545 utils.header("FAILURE to ssh into %s"%hostname)
547 # otherwise, sleep for a while
549 # only useful in empty plcs
553 return self.do_nodes_ssh(minutes=6,gracetime=4)
556 def init_node (self): pass
558 def bootcd (self): pass
560 def configure_qemu (self): pass
562 def reinstall_node (self): pass
564 def export_qemu (self): pass
566 def do_check_initscripts(self):
568 for slice_spec in self.plc_spec['slices']:
569 if not slice_spec.has_key('initscriptname'):
571 initscript=slice_spec['initscriptname']
572 for nodename in slice_spec['nodenames']:
573 (site,node) = self.locate_node (nodename)
574 # xxx - passing the wrong site - probably harmless
575 test_site = TestSite (self,site)
576 test_slice = TestSlice (self,test_site,slice_spec)
577 test_node = TestNode (self,test_site,node)
578 test_sliver = TestSliver (self, test_node, test_slice)
579 if not test_sliver.check_initscript(initscript):
583 def check_initscripts(self):
584 return self.do_check_initscripts()
586 def initscripts (self):
587 for initscript in self.plc_spec['initscripts']:
588 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
589 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
593 return self.do_slices()
595 def clean_slices (self):
596 return self.do_slices("delete")
598 def do_slices (self, action="add"):
599 for slice in self.plc_spec['slices']:
600 site_spec = self.locate_site (slice['sitename'])
601 test_site = TestSite(self,site_spec)
602 test_slice=TestSlice(self,test_site,slice)
604 utils.header("Deleting slices in site %s"%test_site.name())
605 test_slice.delete_slice()
607 utils.pprint("Creating slice",slice)
608 test_slice.create_slice()
609 utils.header('Created Slice %s'%slice['slice_fields']['name'])
612 @slice_mapper_options
613 def check_slice(self): pass
616 def clear_known_hosts (self): pass
619 def start_node (self) : pass
621 def all_sliver_objs (self):
623 for slice_spec in self.plc_spec['slices']:
624 slicename = slice_spec['slice_fields']['name']
625 for nodename in slice_spec['nodenames']:
626 result.append(self.locate_sliver_obj (nodename,slicename))
629 def locate_sliver_obj (self,nodename,slicename):
630 (site,node) = self.locate_node(nodename)
631 slice = self.locate_slice (slicename)
633 test_site = TestSite (self, site)
634 test_node = TestNode (self, test_site,node)
635 # xxx the slice site is assumed to be the node site - mhh - probably harmless
636 test_slice = TestSlice (self, test_site, slice)
637 return TestSliver (self, test_node, test_slice)
639 def check_tcp (self):
640 specs = self.plc_spec['tcp_test']
645 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
646 if not s_test_sliver.run_tcp_server(port,timeout=10):
650 # idem for the client side
651 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
652 if not c_test_sliver.run_tcp_client(s_test_sliver.test_node.name(),port):
657 def gather_logs (self):
658 # (1) get the plc's /var/log and store it locally in logs/myplc.var-log.<plcname>/*
659 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
660 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
661 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
663 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
664 self.gather_var_logs ()
666 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
667 for site_spec in self.plc_spec['sites']:
668 test_site = TestSite (self,site_spec)
669 for node_spec in site_spec['nodes']:
670 test_node=TestNode(self,test_site,node_spec)
671 test_node.gather_qemu_logs()
673 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
674 self.gather_nodes_var_logs()
676 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
677 self.gather_slivers_var_logs()
680 def gather_slivers_var_logs(self):
681 for test_sliver in self.all_sliver_objs():
682 remote = test_sliver.tar_var_logs()
683 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
684 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
685 utils.system(command)
688 def gather_var_logs (self):
689 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
690 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
691 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
692 utils.system(command)
694 def gather_nodes_var_logs (self):
695 for site_spec in self.plc_spec['sites']:
696 test_site = TestSite (self,site_spec)
697 for node_spec in site_spec['nodes']:
698 test_node=TestNode(self,test_site,node_spec)
699 test_ssh = TestSsh (test_node.name(),key="/etc/planetlab/root_ssh_key.rsa")
700 to_plc = self.actual_command_in_guest ( test_ssh.actual_command("tar -C /var/log -cf - ."))
701 command = to_plc + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
702 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
703 utils.system(command)
706 # returns the filename to use for sql dump/restore, using options.dbname if set
707 def dbfile (self, database):
708 # uses options.dbname if it is found
710 name=self.options.dbname
711 if not isinstance(name,StringTypes):
714 t=datetime.datetime.now()
717 return "/root/%s-%s.sql"%(database,name)
720 dump=self.dbfile("planetab4")
721 self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
722 utils.header('Dumped planetlab4 database in %s'%dump)
725 def db_restore(self):
726 dump=self.dbfile("planetab4")
728 self.run_in_guest('service httpd stop')
729 # xxx - need another wrapper
730 self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
731 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
732 self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
733 ##starting httpd service
734 self.run_in_guest('service httpd start')
736 utils.header('Database restored from ' + dump)
739 def standby_1(): pass
741 def standby_2(): pass
743 def standby_3(): pass
745 def standby_4(): pass
747 def standby_5(): pass
749 def standby_6(): pass
751 def standby_7(): pass
753 def standby_8(): pass
755 def standby_9(): pass
757 def standby_10(): pass
759 def standby_11(): pass
761 def standby_12(): pass
763 def standby_13(): pass
765 def standby_14(): pass
767 def standby_15(): pass
769 def standby_16(): pass
771 def standby_17(): pass
773 def standby_18(): pass
775 def standby_19(): pass
777 def standby_20(): pass