8 from types import StringTypes
11 from TestSite import TestSite
12 from TestNode import TestNode
13 from TestUser import TestUser
14 from TestKey import TestKey
15 from TestSlice import TestSlice
16 from TestSliver import TestSliver
17 from TestBox import TestBox
18 from TestSsh import TestSsh
19 from TestApiserver import TestApiserver
21 # step methods must take (self) and return a boolean (options is a member of the class)
23 def standby(minutes,dry_run):
24 utils.header('Entering StandBy for %d mn'%minutes)
28 time.sleep(60*minutes)
31 def standby_generic (func):
33 minutes=int(func.__name__.split("_")[1])
34 return standby(minutes,self.options.dry_run)
37 def node_mapper (method):
40 node_method = TestNode.__dict__[method.__name__]
41 for site_spec in self.plc_spec['sites']:
42 test_site = TestSite (self,site_spec)
43 for node_spec in site_spec['nodes']:
44 test_node = TestNode (self,test_site,node_spec)
45 if not node_method(test_node): overall=False
49 def slice_mapper_options (method):
52 slice_method = TestSlice.__dict__[method.__name__]
53 for slice_spec in self.plc_spec['slices']:
54 site_spec = self.locate_site (slice_spec['sitename'])
55 test_site = TestSite(self,site_spec)
56 test_slice=TestSlice(self,test_site,slice_spec)
57 if not slice_method(test_slice,self.options): overall=False
63 default_steps = ['uninstall','install','install_rpm',
64 'configure', 'start', SEP,
65 'store_keys', 'clear_known_hosts', 'initscripts', SEP,
66 'sites', 'nodes', 'slices', 'nodegroups', SEP,
67 'init_node','bootcd', 'configure_qemu',
68 'kill_all_qemus', 'reinstall_node','start_node', SEP,
70 'nodes_booted', 'nodes_ssh', 'check_slice',
71 'check_initscripts', 'check_tcp',SEP,
72 'force_gather_logs', 'force_kill_qemus', ]
73 other_steps = [ 'stop_all_vservers','fresh_install', 'cache_rpm', 'stop', SEP,
74 'clean_sites', 'clean_nodes', 'clean_slices', 'clean_keys', SEP,
75 'show_boxes', 'list_all_qemus', 'list_qemus', SEP,
76 'db_dump' , 'db_restore',
77 'standby_1 through 20'
79 def __init__ (self,plc_spec,options):
80 self.plc_spec=plc_spec
82 self.test_ssh=TestSsh(self.plc_spec['hostname'],self.options.buildname)
84 self.vserverip=plc_spec['vserverip']
85 self.vservername=plc_spec['vservername']
86 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
90 self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
91 # utils.header('Using API url %s'%self.url)
92 self.apiserver=TestApiserver(self.url,options.dry_run)
95 name=self.plc_spec['name']
97 return name+".vserver.%s"%self.vservername
102 return self.plc_spec['hostname']
105 return self.test_ssh.is_local()
107 # define the API methods on this object through xmlrpc
108 # would help, but not strictly necessary
112 def actual_command_in_guest (self,command):
113 return self.test_ssh.actual_command(self.host_to_guest(command))
115 def run_in_guest (self,command):
116 return utils.system(self.actual_command_in_guest(command))
118 def run_in_host (self,command):
119 return self.test_ssh.run_in_buildname(command)
121 #command gets run in the chroot/vserver
122 def host_to_guest(self,command):
124 return "vserver %s exec %s"%(self.vservername,command)
126 return "chroot /plc/root %s"%TestSsh.backslash_shell_specials(command)
128 # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
129 def copy_in_guest (self, localfile, remotefile, in_data=False):
131 chroot_dest="/plc/data"
133 chroot_dest="/plc/root"
136 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
138 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
141 utils.system("scp %s %s:%s/%s"%(localfile,self.hostname(),chroot_dest,remotefile))
143 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.hostname(),self.vservername,remotefile))
147 def run_in_guest_piped (self,local,remote):
148 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote)))
150 def auth_root (self):
151 return {'Username':self.plc_spec['PLC_ROOT_USER'],
152 'AuthMethod':'password',
153 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
154 'Role' : self.plc_spec['role']
156 def locate_site (self,sitename):
157 for site in self.plc_spec['sites']:
158 if site['site_fields']['name'] == sitename:
160 if site['site_fields']['login_base'] == sitename:
162 raise Exception,"Cannot locate site %s"%sitename
164 def locate_node (self,nodename):
165 for site in self.plc_spec['sites']:
166 for node in site['nodes']:
167 if node['name'] == nodename:
169 raise Exception,"Cannot locate node %s"%nodename
171 def locate_hostname (self,hostname):
172 for site in self.plc_spec['sites']:
173 for node in site['nodes']:
174 if node['node_fields']['hostname'] == hostname:
176 raise Exception,"Cannot locate hostname %s"%hostname
178 def locate_key (self,keyname):
179 for key in self.plc_spec['keys']:
180 if key['name'] == keyname:
182 raise Exception,"Cannot locate key %s"%keyname
184 def locate_slice (self, slicename):
185 for slice in self.plc_spec['slices']:
186 if slice['slice_fields']['name'] == slicename:
188 raise Exception,"Cannot locate slice %s"%slicename
190 # all different hostboxes used in this plc
191 def gather_hostBoxes(self):
192 # maps on sites and nodes, return [ (host_box,test_node) ]
194 for site_spec in self.plc_spec['sites']:
195 test_site = TestSite (self,site_spec)
196 for node_spec in site_spec['nodes']:
197 test_node = TestNode (self, test_site, node_spec)
198 if not test_node.is_real():
199 tuples.append( (test_node.host_box(),test_node) )
200 # transform into a dict { 'host_box' -> [ test_node .. ] }
202 for (box,node) in tuples:
203 if not result.has_key(box):
206 result[box].append(node)
209 # a step for checking this stuff
210 def show_boxes (self):
211 for (box,nodes) in self.gather_hostBoxes().iteritems():
212 print box,":"," + ".join( [ node.name() for node in nodes ] )
215 # make this a valid step
216 def kill_all_qemus(self):
217 # this is the brute force version, kill all qemus on that host box
218 for (box,nodes) in self.gather_hostBoxes().iteritems():
219 # pass the first nodename, as we don't push template-qemu on testboxes
220 nodename=nodes[0].name()
221 TestBox(box,self.options.buildname).kill_all_qemus(nodename)
224 # make this a valid step
225 def list_all_qemus(self):
226 for (box,nodes) in self.gather_hostBoxes().iteritems():
227 # this is the brute force version, kill all qemus on that host box
228 TestBox(box,self.options.buildname).list_all_qemus()
231 # kill only the right qemus
232 def list_qemus(self):
233 for (box,nodes) in self.gather_hostBoxes().iteritems():
234 # the fine-grain version
239 # kill only the right qemus
240 def kill_qemus(self):
241 for (box,nodes) in self.gather_hostBoxes().iteritems():
242 # the fine-grain version
247 #################### step methods
250 def uninstall_chroot(self):
251 self.run_in_host('service plc safestop')
252 #####detecting the last myplc version installed and remove it
253 self.run_in_host('rpm -e myplc')
254 ##### Clean up the /plc directory
255 self.run_in_host('rm -rf /plc/data')
256 ##### stop any running vservers
257 self.run_in_host('for vserver in $(ls -d /vservers/* | sed -e s,/vservers/,,) ; do case $vserver in vtest*) echo Shutting down vserver $vserver ; vserver $vserver stop ;; esac ; done')
260 def uninstall_vserver(self):
261 self.run_in_host("vserver --silent %s delete"%self.vservername)
265 # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
266 # it sounds safer to have the former uninstalled too
267 # now the vserver method cannot be invoked for chroot instances as vservername is required
269 self.uninstall_vserver()
270 self.uninstall_chroot()
272 self.uninstall_chroot()
276 def install_chroot(self):
280 def install_vserver(self):
281 # we need build dir for vtest-init-vserver
283 # a full path for the local calls
284 build_dir=os.path(sys.argv[0])+"/build"
286 # use a standard name - will be relative to HOME
287 build_dir="options.buildname"
288 # run checkout in any case - would do an update if already exists
289 build_checkout = "svn checkout %s %s"%(self.options.build_url,build_dir)
290 if self.run_in_host(build_checkout) != 0:
292 # the repo url is taken from myplc-url
293 # with the last two steps (i386/myplc...) removed
294 repo_url = self.options.myplc_url
295 for level in [ 'rpmname','arch' ]:
296 repo_url = os.path.dirname(repo_url)
297 create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
298 (build_dir,self.vservername,repo_url,self.vserverip)
299 return self.run_in_host(create_vserver) == 0
303 return self.install_vserver()
305 return self.install_chroot()
307 ### install_rpm - make this an optional step
309 url = self.options.myplc_url
310 rpm = os.path.basename(url)
311 cache_fetch="pwd;if [ -f %(rpm)s ] ; then echo Using cached rpm %(rpm)s ; else echo Fetching %(url)s ; curl -O %(url)s; fi"%locals()
312 return self.run_in_host(cache_fetch)==0
314 def install_rpm_chroot(self):
315 url = self.options.myplc_url
316 rpm = os.path.basename(url)
317 if not self.cache_rpm():
319 utils.header('Installing the : %s'%rpm)
320 return self.run_in_host('rpm -Uvh '+rpm)==0 and self.run_in_host('service plc mount')==0
322 def install_rpm_vserver(self):
323 return self.run_in_guest("yum -y install myplc-native")==0
325 def install_rpm(self):
327 return self.install_rpm_vserver()
329 return self.install_rpm_chroot()
333 tmpname='%s.plc-config-tty'%(self.name())
334 fileconf=open(tmpname,'w')
335 for var in [ 'PLC_NAME',
339 'PLC_MAIL_SUPPORT_ADDRESS',
346 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
347 fileconf.write('w\n')
348 fileconf.write('q\n')
350 utils.system('cat %s'%tmpname)
351 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
352 utils.system('rm %s'%tmpname)
355 # the chroot install is slightly different to this respect
358 self.run_in_guest('service plc start')
360 self.run_in_host('service plc start')
365 self.run_in_guest('service plc stop')
367 self.run_in_host('service plc stop')
370 # could use a TestKey class
371 def store_keys(self):
372 for key_spec in self.plc_spec['keys']:
373 TestKey(self,key_spec).store_key()
376 def clean_keys(self):
377 utils.system("rm -rf %s/keys/"%os.path(sys.argv[0]))
380 return self.do_sites()
382 def clean_sites (self):
383 return self.do_sites(action="delete")
385 def do_sites (self,action="add"):
386 for site_spec in self.plc_spec['sites']:
387 test_site = TestSite (self,site_spec)
388 if (action != "add"):
389 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
390 test_site.delete_site()
391 # deleted with the site
392 #test_site.delete_users()
395 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
396 test_site.create_site()
397 test_site.create_users()
401 return self.do_nodes()
402 def clean_nodes (self):
403 return self.do_nodes(action="delete")
405 def do_nodes (self,action="add"):
406 for site_spec in self.plc_spec['sites']:
407 test_site = TestSite (self,site_spec)
409 utils.header("Deleting nodes in site %s"%test_site.name())
410 for node_spec in site_spec['nodes']:
411 test_node=TestNode(self,test_site,node_spec)
412 utils.header("Deleting %s"%test_node.name())
413 test_node.delete_node()
415 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
416 for node_spec in site_spec['nodes']:
417 utils.pprint('Creating node %s'%node_spec,node_spec)
418 test_node = TestNode (self,test_site,node_spec)
419 test_node.create_node ()
422 # create nodegroups if needed, and populate
423 # no need for a clean_nodegroups if we are careful enough
424 def nodegroups (self):
425 # 1st pass to scan contents
427 for site_spec in self.plc_spec['sites']:
428 test_site = TestSite (self,site_spec)
429 for node_spec in site_spec['nodes']:
430 test_node=TestNode (self,test_site,node_spec)
431 if node_spec.has_key('nodegroups'):
432 nodegroupnames=node_spec['nodegroups']
433 if isinstance(nodegroupnames,StringTypes):
434 nodegroupnames = [ nodegroupnames ]
435 for nodegroupname in nodegroupnames:
436 if not groups_dict.has_key(nodegroupname):
437 groups_dict[nodegroupname]=[]
438 groups_dict[nodegroupname].append(test_node.name())
439 auth=self.auth_root()
440 for (nodegroupname,group_nodes) in groups_dict.iteritems():
442 self.apiserver.GetNodeGroups(auth,{'name':nodegroupname})[0]
444 self.apiserver.AddNodeGroup(auth,{'name':nodegroupname})
445 for node in group_nodes:
446 self.apiserver.AddNodeToNodeGroup(auth,node,nodegroupname)
449 def all_hostnames (self) :
451 for site_spec in self.plc_spec['sites']:
452 hostnames += [ node_spec['node_fields']['hostname'] \
453 for node_spec in site_spec['nodes'] ]
456 # gracetime : during the first <gracetime> minutes nothing gets printed
457 def do_nodes_booted (self, minutes, gracetime=2):
458 if self.options.dry_run:
462 timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
463 graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
464 # the nodes that haven't checked yet - start with a full list and shrink over time
465 tocheck = self.all_hostnames()
466 utils.header("checking nodes %r"%tocheck)
467 # create a dict hostname -> status
468 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
471 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
473 for array in tocheck_status:
474 hostname=array['hostname']
475 boot_state=array['boot_state']
476 if boot_state == 'boot':
477 utils.header ("%s has reached the 'boot' state"%hostname)
479 # if it's a real node, never mind
480 (site_spec,node_spec)=self.locate_hostname(hostname)
481 if TestNode.is_real_model(node_spec['node_fields']['model']):
482 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
485 if datetime.datetime.now() > graceout:
486 utils.header ("%s still in '%s' state"%(hostname,boot_state))
487 graceout=datetime.datetime.now()+datetime.timedelta(1)
488 status[hostname] = boot_state
490 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
493 if datetime.datetime.now() > timeout:
494 for hostname in tocheck:
495 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
497 # otherwise, sleep for a while
499 # only useful in empty plcs
502 def nodes_booted(self):
503 return self.do_nodes_booted(minutes=0)
506 def do_nodes_ssh(self,minutes):
508 timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
509 tocheck = self.all_hostnames()
510 # self.scan_publicKeys(tocheck)
511 utils.header("checking Connectivity on nodes %r"%tocheck)
513 for hostname in tocheck:
514 # try to ssh in nodes
515 node_test_ssh = TestSsh (hostname,key="/etc/planetlab/root_ssh_key.rsa")
516 access=self.run_in_guest(node_test_ssh.actual_command("date"))
518 utils.header('The node %s is sshable -->'%hostname)
520 tocheck.remove(hostname)
522 # we will have tried real nodes once, in case they're up - but if not, just skip
523 (site_spec,node_spec)=self.locate_hostname(hostname)
524 if TestNode.is_real_model(node_spec['node_fields']['model']):
525 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
526 tocheck.remove(hostname)
529 if datetime.datetime.now() > timeout:
530 for hostname in tocheck:
531 utils.header("FAILURE to ssh into %s"%hostname)
533 # otherwise, sleep for a while
535 # only useful in empty plcs
539 return self.do_nodes_ssh(minutes=2)
542 def init_node (self): pass
544 def bootcd (self): pass
546 def configure_qemu (self): pass
548 def reinstall_node (self): pass
550 def do_check_initscripts(self):
552 for slice_spec in self.plc_spec['slices']:
553 if not slice_spec.has_key('initscriptname'):
555 initscript=slice_spec['initscriptname']
556 for nodename in slice_spec['nodenames']:
557 (site,node) = self.locate_node (nodename)
558 # xxx - passing the wrong site - probably harmless
559 test_site = TestSite (self,site)
560 test_slice = TestSlice (self,test_site,slice_spec)
561 test_node = TestNode (self,test_site,node)
562 test_sliver = TestSliver (self, test_node, test_slice)
563 if not test_sliver.check_initscript(initscript):
567 def check_initscripts(self):
568 return self.do_check_initscripts()
570 def initscripts (self):
571 for initscript in self.plc_spec['initscripts']:
572 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
573 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
577 return self.do_slices()
579 def clean_slices (self):
580 return self.do_slices("delete")
582 def do_slices (self, action="add"):
583 for slice in self.plc_spec['slices']:
584 site_spec = self.locate_site (slice['sitename'])
585 test_site = TestSite(self,site_spec)
586 test_slice=TestSlice(self,test_site,slice)
588 utils.header("Deleting slices in site %s"%test_site.name())
589 test_slice.delete_slice()
591 utils.pprint("Creating slice",slice)
592 test_slice.create_slice()
593 utils.header('Created Slice %s'%slice['slice_fields']['name'])
596 @slice_mapper_options
597 def check_slice(self): pass
600 def clear_known_hosts (self): pass
603 def start_node (self) : pass
605 def locate_first_sliver (self):
606 slice_spec = self.plc_spec['slices'][0]
607 slicename = slice_spec['slice_fields']['name']
608 nodename = slice_spec['nodenames'][0]
609 return self.locate_sliver_obj(nodename,slicename)
611 def locate_sliver_obj (self,nodename,slicename):
612 (site,node) = self.locate_node(nodename)
613 slice = self.locate_slice (slicename)
615 test_site = TestSite (self, site)
616 test_node = TestNode (self, test_site,node)
617 # xxx the slice site is assumed to be the node site - mhh - probably harmless
618 test_slice = TestSlice (self, test_site, slice)
619 return TestSliver (self, test_node, test_slice)
621 def check_tcp (self):
622 specs = self.plc_spec['tcp_test']
627 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
628 if not s_test_sliver.run_tcp_server(port,timeout=10):
632 # idem for the client side
633 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
634 if not c_test_sliver.run_tcp_client(s_test_sliver.test_node.name(),port):
639 def gather_logs (self):
640 # (1) get the plc's /var/log and store it locally in logs/<plcname>-var-log/*
641 # (2) get all the nodes qemu log and store it as logs/<node>-qemu.log
642 # (3) get the nodes /var/log and store is as logs/<node>-var-log/*
643 # (4) as far as possible get the slice's /var/log as logs/<slice>-<node>-var-log/*
645 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
646 self.gather_var_logs ()
648 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
649 for site_spec in self.plc_spec['sites']:
650 test_site = TestSite (self,site_spec)
651 for node_spec in site_spec['nodes']:
652 test_node=TestNode(self,test_site,node_spec)
653 test_node.gather_qemu_logs()
655 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
656 self.gather_nodes_var_logs()
658 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
659 self.gather_first_sliver_logs()
662 def gather_first_sliver_logs(self):
664 test_sliver = self.locate_first_sliver()
665 remote = test_sliver.tar_var_logs()
666 utils.system("mkdir -p logs/%s-var-log"%test_sliver.name())
667 command = remote + " | tar -C logs/%s-var-log -xf -"%test_sliver.name()
668 utils.system(command)
670 print 'Cannot locate first sliver - giving up',e
673 def gather_var_logs (self):
674 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
675 command = to_plc + "| tar -C logs/%s-var-log -xf -"%self.name()
676 utils.system("mkdir -p logs/%s-var-log"%self.name())
677 utils.system(command)
679 def gather_nodes_var_logs (self):
680 for site_spec in self.plc_spec['sites']:
681 test_site = TestSite (self,site_spec)
682 for node_spec in site_spec['nodes']:
683 test_node=TestNode(self,test_site,node_spec)
684 test_ssh = TestSsh (test_node.name(),key="/etc/planetlab/root_ssh_key.rsa")
685 to_plc = self.actual_command_in_guest ( test_ssh.actual_command("tar -C /var/log -cf - ."))
686 command = to_plc + "| tar -C logs/%s-var-log -xf -"%test_node.name()
687 utils.system("mkdir -p logs/%s-var-log"%test_node.name())
688 utils.system(command)
691 # returns the filename to use for sql dump/restore, using options.dbname if set
692 def dbfile (self, database):
693 # uses options.dbname if it is found
695 name=self.options.dbname
696 if not isinstance(name,StringTypes):
699 t=datetime.datetime.now()
702 return "/root/%s-%s.sql"%(database,name)
705 dump=self.dbfile("planetab4")
706 self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
707 utils.header('Dumped planetlab4 database in %s'%dump)
710 def db_restore(self):
711 dump=self.dbfile("planetab4")
713 self.run_in_guest('service httpd stop')
714 # xxx - need another wrapper
715 self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
716 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
717 self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
718 ##starting httpd service
719 self.run_in_guest('service httpd start')
721 utils.header('Database restored from ' + dump)
724 def standby_1(): pass
726 def standby_2(): pass
728 def standby_3(): pass
730 def standby_4(): pass
732 def standby_5(): pass
734 def standby_6(): pass
736 def standby_7(): pass
738 def standby_8(): pass
740 def standby_9(): pass
742 def standby_10(): pass
744 def standby_11(): pass
746 def standby_12(): pass
748 def standby_13(): pass
750 def standby_14(): pass
752 def standby_15(): pass
754 def standby_16(): pass
756 def standby_17(): pass
758 def standby_18(): pass
760 def standby_19(): pass
762 def standby_20(): pass