8 from types import StringTypes
11 from TestSite import TestSite
12 from TestNode import TestNode
13 from TestUser import TestUser
14 from TestKey import TestKey
15 from TestSlice import TestSlice
16 from TestSliver import TestSliver
17 from TestBox import TestBox
18 from TestSsh import TestSsh
19 from TestApiserver import TestApiserver
21 # step methods must take (self) and return a boolean (options is a member of the class)
23 def standby(minutes,dry_run):
24 utils.header('Entering StandBy for %d mn'%minutes)
28 time.sleep(60*minutes)
31 def standby_generic (func):
33 minutes=int(func.__name__.split("_")[1])
34 return standby(minutes,self.options.dry_run)
37 def node_mapper (method):
40 node_method = TestNode.__dict__[method.__name__]
41 for site_spec in self.plc_spec['sites']:
42 test_site = TestSite (self,site_spec)
43 for node_spec in site_spec['nodes']:
44 test_node = TestNode (self,test_site,node_spec)
45 if not node_method(test_node): overall=False
49 def slice_mapper_options (method):
52 slice_method = TestSlice.__dict__[method.__name__]
53 for slice_spec in self.plc_spec['slices']:
54 site_spec = self.locate_site (slice_spec['sitename'])
55 test_site = TestSite(self,site_spec)
56 test_slice=TestSlice(self,test_site,slice_spec)
57 if not slice_method(test_slice,self.options): overall=False
65 default_steps = ['uninstall','install','install_rpm',
66 'configure', 'start', SEP,
67 'store_keys', 'clear_known_hosts', 'initscripts', SEP,
68 'sites', 'nodes', 'slices', 'nodegroups', SEP,
69 'init_node','bootcd', 'configure_qemu', 'export_qemu',
70 'kill_all_qemus', 'reinstall_node','start_node', SEP,
72 'nodes_booted', 'nodes_ssh', 'check_slice',
73 'check_initscripts', 'check_tcp',SEP,
74 'force_gather_logs', 'force_kill_qemus', ]
75 other_steps = [ 'stop_all_vservers','fresh_install', 'cache_rpm', 'stop', SEP,
76 'clean_sites', 'clean_nodes', 'clean_slices', 'clean_keys', SEP,
77 'show_boxes', 'list_all_qemus', 'list_qemus', SEP,
78 'db_dump' , 'db_restore',
79 'standby_1 through 20'
83 def printable_steps (list):
84 return " ".join(list).replace(" "+SEP+" ","\n")
86 def valid_step (step):
89 def __init__ (self,plc_spec,options):
90 self.plc_spec=plc_spec
92 self.test_ssh=TestSsh(self.plc_spec['hostname'],self.options.buildname)
94 self.vserverip=plc_spec['vserverip']
95 self.vservername=plc_spec['vservername']
96 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
100 self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
101 # utils.header('Using API url %s'%self.url)
102 self.apiserver=TestApiserver(self.url,options.dry_run)
105 name=self.plc_spec['name']
107 return name+".vserver.%s"%self.vservername
109 return name+".chroot"
112 return self.plc_spec['hostname']
115 return self.test_ssh.is_local()
117 # define the API methods on this object through xmlrpc
118 # would help, but not strictly necessary
122 def actual_command_in_guest (self,command):
123 return self.test_ssh.actual_command(self.host_to_guest(command))
125 def run_in_guest (self,command):
126 return utils.system(self.actual_command_in_guest(command))
128 def run_in_host (self,command):
129 return self.test_ssh.run_in_buildname(command)
131 #command gets run in the chroot/vserver
132 def host_to_guest(self,command):
134 return "vserver %s exec %s"%(self.vservername,command)
136 return "chroot /plc/root %s"%TestSsh.backslash_shell_specials(command)
138 # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
139 def copy_in_guest (self, localfile, remotefile, in_data=False):
141 chroot_dest="/plc/data"
143 chroot_dest="/plc/root"
146 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
148 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
151 utils.system("scp %s %s:%s/%s"%(localfile,self.hostname(),chroot_dest,remotefile))
153 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.hostname(),self.vservername,remotefile))
157 def run_in_guest_piped (self,local,remote):
158 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote)))
160 def auth_root (self):
161 return {'Username':self.plc_spec['PLC_ROOT_USER'],
162 'AuthMethod':'password',
163 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
164 'Role' : self.plc_spec['role']
166 def locate_site (self,sitename):
167 for site in self.plc_spec['sites']:
168 if site['site_fields']['name'] == sitename:
170 if site['site_fields']['login_base'] == sitename:
172 raise Exception,"Cannot locate site %s"%sitename
174 def locate_node (self,nodename):
175 for site in self.plc_spec['sites']:
176 for node in site['nodes']:
177 if node['name'] == nodename:
179 raise Exception,"Cannot locate node %s"%nodename
181 def locate_hostname (self,hostname):
182 for site in self.plc_spec['sites']:
183 for node in site['nodes']:
184 if node['node_fields']['hostname'] == hostname:
186 raise Exception,"Cannot locate hostname %s"%hostname
188 def locate_key (self,keyname):
189 for key in self.plc_spec['keys']:
190 if key['name'] == keyname:
192 raise Exception,"Cannot locate key %s"%keyname
194 def locate_slice (self, slicename):
195 for slice in self.plc_spec['slices']:
196 if slice['slice_fields']['name'] == slicename:
198 raise Exception,"Cannot locate slice %s"%slicename
200 # all different hostboxes used in this plc
201 def gather_hostBoxes(self):
202 # maps on sites and nodes, return [ (host_box,test_node) ]
204 for site_spec in self.plc_spec['sites']:
205 test_site = TestSite (self,site_spec)
206 for node_spec in site_spec['nodes']:
207 test_node = TestNode (self, test_site, node_spec)
208 if not test_node.is_real():
209 tuples.append( (test_node.host_box(),test_node) )
210 # transform into a dict { 'host_box' -> [ test_node .. ] }
212 for (box,node) in tuples:
213 if not result.has_key(box):
216 result[box].append(node)
219 # a step for checking this stuff
220 def show_boxes (self):
221 for (box,nodes) in self.gather_hostBoxes().iteritems():
222 print box,":"," + ".join( [ node.name() for node in nodes ] )
225 # make this a valid step
226 def kill_all_qemus(self):
227 # this is the brute force version, kill all qemus on that host box
228 for (box,nodes) in self.gather_hostBoxes().iteritems():
229 # pass the first nodename, as we don't push template-qemu on testboxes
230 nodedir=nodes[0].nodedir()
231 TestBox(box,self.options.buildname).kill_all_qemus(nodedir)
234 # make this a valid step
235 def list_all_qemus(self):
236 for (box,nodes) in self.gather_hostBoxes().iteritems():
237 # this is the brute force version, kill all qemus on that host box
238 TestBox(box,self.options.buildname).list_all_qemus()
241 # kill only the right qemus
242 def list_qemus(self):
243 for (box,nodes) in self.gather_hostBoxes().iteritems():
244 # the fine-grain version
249 # kill only the right qemus
250 def kill_qemus(self):
251 for (box,nodes) in self.gather_hostBoxes().iteritems():
252 # the fine-grain version
257 #################### step methods
260 def uninstall_chroot(self):
261 self.run_in_host('service plc safestop')
262 #####detecting the last myplc version installed and remove it
263 self.run_in_host('rpm -e myplc')
264 ##### Clean up the /plc directory
265 self.run_in_host('rm -rf /plc/data')
266 ##### stop any running vservers
267 self.run_in_host('for vserver in $(ls -d /vservers/* | sed -e s,/vservers/,,) ; do case $vserver in vtest*) echo Shutting down vserver $vserver ; vserver $vserver stop ;; esac ; done')
270 def uninstall_vserver(self):
271 self.run_in_host("vserver --silent %s delete"%self.vservername)
275 # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
276 # it sounds safer to have the former uninstalled too
277 # now the vserver method cannot be invoked for chroot instances as vservername is required
279 self.uninstall_vserver()
280 self.uninstall_chroot()
282 self.uninstall_chroot()
286 def install_chroot(self):
290 def install_vserver(self):
291 # we need build dir for vtest-init-vserver
293 # a full path for the local calls
294 build_dir=os.path(sys.argv[0])+"/build"
296 # use a standard name - will be relative to HOME
297 build_dir="options.buildname"
298 # run checkout in any case - would do an update if already exists
299 build_checkout = "svn checkout %s %s"%(self.options.build_url,build_dir)
300 if self.run_in_host(build_checkout) != 0:
302 # the repo url is taken from myplc-url
303 # with the last two steps (i386/myplc...) removed
304 repo_url = self.options.myplc_url
305 for level in [ 'rpmname','arch' ]:
306 repo_url = os.path.dirname(repo_url)
307 create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
308 (build_dir,self.vservername,repo_url,self.vserverip)
309 return self.run_in_host(create_vserver) == 0
313 return self.install_vserver()
315 return self.install_chroot()
317 ### install_rpm - make this an optional step
319 url = self.options.myplc_url
320 rpm = os.path.basename(url)
321 cache_fetch="pwd;if [ -f %(rpm)s ] ; then echo Using cached rpm %(rpm)s ; else echo Fetching %(url)s ; curl -O %(url)s; fi"%locals()
322 return self.run_in_host(cache_fetch)==0
324 def install_rpm_chroot(self):
325 url = self.options.myplc_url
326 rpm = os.path.basename(url)
327 if not self.cache_rpm():
329 utils.header('Installing the : %s'%rpm)
330 return self.run_in_host('rpm -Uvh '+rpm)==0 and self.run_in_host('service plc mount')==0
332 def install_rpm_vserver(self):
333 return self.run_in_guest("yum -y install myplc-native")==0
335 def install_rpm(self):
337 return self.install_rpm_vserver()
339 return self.install_rpm_chroot()
343 tmpname='%s.plc-config-tty'%(self.name())
344 fileconf=open(tmpname,'w')
345 for var in [ 'PLC_NAME',
349 'PLC_MAIL_SUPPORT_ADDRESS',
356 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
357 fileconf.write('w\n')
358 fileconf.write('q\n')
360 utils.system('cat %s'%tmpname)
361 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
362 utils.system('rm %s'%tmpname)
365 # the chroot install is slightly different to this respect
368 self.run_in_guest('service plc start')
370 self.run_in_host('service plc start')
375 self.run_in_guest('service plc stop')
377 self.run_in_host('service plc stop')
380 # could use a TestKey class
381 def store_keys(self):
382 for key_spec in self.plc_spec['keys']:
383 TestKey(self,key_spec).store_key()
386 def clean_keys(self):
387 utils.system("rm -rf %s/keys/"%os.path(sys.argv[0]))
390 return self.do_sites()
392 def clean_sites (self):
393 return self.do_sites(action="delete")
395 def do_sites (self,action="add"):
396 for site_spec in self.plc_spec['sites']:
397 test_site = TestSite (self,site_spec)
398 if (action != "add"):
399 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
400 test_site.delete_site()
401 # deleted with the site
402 #test_site.delete_users()
405 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
406 test_site.create_site()
407 test_site.create_users()
411 return self.do_nodes()
412 def clean_nodes (self):
413 return self.do_nodes(action="delete")
415 def do_nodes (self,action="add"):
416 for site_spec in self.plc_spec['sites']:
417 test_site = TestSite (self,site_spec)
419 utils.header("Deleting nodes in site %s"%test_site.name())
420 for node_spec in site_spec['nodes']:
421 test_node=TestNode(self,test_site,node_spec)
422 utils.header("Deleting %s"%test_node.name())
423 test_node.delete_node()
425 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
426 for node_spec in site_spec['nodes']:
427 utils.pprint('Creating node %s'%node_spec,node_spec)
428 test_node = TestNode (self,test_site,node_spec)
429 test_node.create_node ()
432 # create nodegroups if needed, and populate
433 # no need for a clean_nodegroups if we are careful enough
434 def nodegroups (self):
435 # 1st pass to scan contents
437 for site_spec in self.plc_spec['sites']:
438 test_site = TestSite (self,site_spec)
439 for node_spec in site_spec['nodes']:
440 test_node=TestNode (self,test_site,node_spec)
441 if node_spec.has_key('nodegroups'):
442 nodegroupnames=node_spec['nodegroups']
443 if isinstance(nodegroupnames,StringTypes):
444 nodegroupnames = [ nodegroupnames ]
445 for nodegroupname in nodegroupnames:
446 if not groups_dict.has_key(nodegroupname):
447 groups_dict[nodegroupname]=[]
448 groups_dict[nodegroupname].append(test_node.name())
449 auth=self.auth_root()
450 for (nodegroupname,group_nodes) in groups_dict.iteritems():
452 self.apiserver.GetNodeGroups(auth,{'name':nodegroupname})[0]
454 self.apiserver.AddNodeGroup(auth,{'name':nodegroupname})
455 for node in group_nodes:
456 self.apiserver.AddNodeToNodeGroup(auth,node,nodegroupname)
459 def all_hostnames (self) :
461 for site_spec in self.plc_spec['sites']:
462 hostnames += [ node_spec['node_fields']['hostname'] \
463 for node_spec in site_spec['nodes'] ]
466 # gracetime : during the first <gracetime> minutes nothing gets printed
467 def do_nodes_booted (self, minutes, gracetime=2):
468 if self.options.dry_run:
472 timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
473 graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
474 # the nodes that haven't checked yet - start with a full list and shrink over time
475 tocheck = self.all_hostnames()
476 utils.header("checking nodes %r"%tocheck)
477 # create a dict hostname -> status
478 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
481 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
483 for array in tocheck_status:
484 hostname=array['hostname']
485 boot_state=array['boot_state']
486 if boot_state == 'boot':
487 utils.header ("%s has reached the 'boot' state"%hostname)
489 # if it's a real node, never mind
490 (site_spec,node_spec)=self.locate_hostname(hostname)
491 if TestNode.is_real_model(node_spec['node_fields']['model']):
492 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
495 if datetime.datetime.now() > graceout:
496 utils.header ("%s still in '%s' state"%(hostname,boot_state))
497 graceout=datetime.datetime.now()+datetime.timedelta(1)
498 status[hostname] = boot_state
500 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
503 if datetime.datetime.now() > timeout:
504 for hostname in tocheck:
505 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
507 # otherwise, sleep for a while
509 # only useful in empty plcs
512 def nodes_booted(self):
513 return self.do_nodes_booted(minutes=0)
516 def do_nodes_ssh(self,minutes):
518 timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
519 tocheck = self.all_hostnames()
520 # self.scan_publicKeys(tocheck)
521 utils.header("checking Connectivity on nodes %r"%tocheck)
523 for hostname in tocheck:
524 # try to ssh in nodes
525 node_test_ssh = TestSsh (hostname,key="/etc/planetlab/root_ssh_key.rsa")
526 access=self.run_in_guest(node_test_ssh.actual_command("date"))
528 utils.header('The node %s is sshable -->'%hostname)
530 tocheck.remove(hostname)
532 # we will have tried real nodes once, in case they're up - but if not, just skip
533 (site_spec,node_spec)=self.locate_hostname(hostname)
534 if TestNode.is_real_model(node_spec['node_fields']['model']):
535 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
536 tocheck.remove(hostname)
539 if datetime.datetime.now() > timeout:
540 for hostname in tocheck:
541 utils.header("FAILURE to ssh into %s"%hostname)
543 # otherwise, sleep for a while
545 # only useful in empty plcs
549 return self.do_nodes_ssh(minutes=2)
552 def init_node (self): pass
554 def bootcd (self): pass
556 def configure_qemu (self): pass
558 def reinstall_node (self): pass
560 def export_qemu (self): pass
562 def do_check_initscripts(self):
564 for slice_spec in self.plc_spec['slices']:
565 if not slice_spec.has_key('initscriptname'):
567 initscript=slice_spec['initscriptname']
568 for nodename in slice_spec['nodenames']:
569 (site,node) = self.locate_node (nodename)
570 # xxx - passing the wrong site - probably harmless
571 test_site = TestSite (self,site)
572 test_slice = TestSlice (self,test_site,slice_spec)
573 test_node = TestNode (self,test_site,node)
574 test_sliver = TestSliver (self, test_node, test_slice)
575 if not test_sliver.check_initscript(initscript):
579 def check_initscripts(self):
580 return self.do_check_initscripts()
582 def initscripts (self):
583 for initscript in self.plc_spec['initscripts']:
584 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
585 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
589 return self.do_slices()
591 def clean_slices (self):
592 return self.do_slices("delete")
594 def do_slices (self, action="add"):
595 for slice in self.plc_spec['slices']:
596 site_spec = self.locate_site (slice['sitename'])
597 test_site = TestSite(self,site_spec)
598 test_slice=TestSlice(self,test_site,slice)
600 utils.header("Deleting slices in site %s"%test_site.name())
601 test_slice.delete_slice()
603 utils.pprint("Creating slice",slice)
604 test_slice.create_slice()
605 utils.header('Created Slice %s'%slice['slice_fields']['name'])
608 @slice_mapper_options
609 def check_slice(self): pass
612 def clear_known_hosts (self): pass
615 def start_node (self) : pass
617 def locate_first_sliver (self):
618 slice_spec = self.plc_spec['slices'][0]
619 slicename = slice_spec['slice_fields']['name']
620 nodename = slice_spec['nodenames'][0]
621 return self.locate_sliver_obj(nodename,slicename)
623 def locate_sliver_obj (self,nodename,slicename):
624 (site,node) = self.locate_node(nodename)
625 slice = self.locate_slice (slicename)
627 test_site = TestSite (self, site)
628 test_node = TestNode (self, test_site,node)
629 # xxx the slice site is assumed to be the node site - mhh - probably harmless
630 test_slice = TestSlice (self, test_site, slice)
631 return TestSliver (self, test_node, test_slice)
633 def check_tcp (self):
634 specs = self.plc_spec['tcp_test']
639 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
640 if not s_test_sliver.run_tcp_server(port,timeout=10):
644 # idem for the client side
645 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
646 if not c_test_sliver.run_tcp_client(s_test_sliver.test_node.name(),port):
651 def gather_logs (self):
652 # (1) get the plc's /var/log and store it locally in logs/<plcname>-var-log/*
653 # (2) get all the nodes qemu log and store it as logs/<node>-qemu.log
654 # (3) get the nodes /var/log and store is as logs/<node>-var-log/*
655 # (4) as far as possible get the slice's /var/log as logs/<slice>-<node>-var-log/*
657 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
658 self.gather_var_logs ()
660 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
661 for site_spec in self.plc_spec['sites']:
662 test_site = TestSite (self,site_spec)
663 for node_spec in site_spec['nodes']:
664 test_node=TestNode(self,test_site,node_spec)
665 test_node.gather_qemu_logs()
667 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
668 self.gather_nodes_var_logs()
670 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
671 self.gather_first_sliver_logs()
674 def gather_first_sliver_logs(self):
676 test_sliver = self.locate_first_sliver()
677 remote = test_sliver.tar_var_logs()
678 utils.system("mkdir -p logs/%s-var-log"%test_sliver.name())
679 command = remote + " | tar -C logs/%s-var-log -xf -"%test_sliver.name()
680 utils.system(command)
682 print 'Cannot locate first sliver - giving up',e
685 def gather_var_logs (self):
686 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
687 command = to_plc + "| tar -C logs/%s-var-log -xf -"%self.name()
688 utils.system("mkdir -p logs/%s-var-log"%self.name())
689 utils.system(command)
691 def gather_nodes_var_logs (self):
692 for site_spec in self.plc_spec['sites']:
693 test_site = TestSite (self,site_spec)
694 for node_spec in site_spec['nodes']:
695 test_node=TestNode(self,test_site,node_spec)
696 test_ssh = TestSsh (test_node.name(),key="/etc/planetlab/root_ssh_key.rsa")
697 to_plc = self.actual_command_in_guest ( test_ssh.actual_command("tar -C /var/log -cf - ."))
698 command = to_plc + "| tar -C logs/%s-var-log -xf -"%test_node.name()
699 utils.system("mkdir -p logs/%s-var-log"%test_node.name())
700 utils.system(command)
703 # returns the filename to use for sql dump/restore, using options.dbname if set
704 def dbfile (self, database):
705 # uses options.dbname if it is found
707 name=self.options.dbname
708 if not isinstance(name,StringTypes):
711 t=datetime.datetime.now()
714 return "/root/%s-%s.sql"%(database,name)
717 dump=self.dbfile("planetab4")
718 self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
719 utils.header('Dumped planetlab4 database in %s'%dump)
722 def db_restore(self):
723 dump=self.dbfile("planetab4")
725 self.run_in_guest('service httpd stop')
726 # xxx - need another wrapper
727 self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
728 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
729 self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
730 ##starting httpd service
731 self.run_in_guest('service httpd start')
733 utils.header('Database restored from ' + dump)
736 def standby_1(): pass
738 def standby_2(): pass
740 def standby_3(): pass
742 def standby_4(): pass
744 def standby_5(): pass
746 def standby_6(): pass
748 def standby_7(): pass
750 def standby_8(): pass
752 def standby_9(): pass
754 def standby_10(): pass
756 def standby_11(): pass
758 def standby_12(): pass
760 def standby_13(): pass
762 def standby_14(): pass
764 def standby_15(): pass
766 def standby_16(): pass
768 def standby_17(): pass
770 def standby_18(): pass
772 def standby_19(): pass
774 def standby_20(): pass