7 from types import StringTypes
10 from TestSite import TestSite
11 from TestNode import TestNode
12 from TestUser import TestUser
13 from TestKey import TestKey
14 from TestSlice import TestSlice
15 from TestSliver import TestSliver
16 from TestBox import TestBox
17 from TestSsh import TestSsh
18 from TestApiserver import TestApiserver
20 # step methods must take (self) and return a boolean (options is a member of the class)
22 def standby(minutes,dry_run):
23 utils.header('Entering StandBy for %d mn'%minutes)
27 time.sleep(60*minutes)
30 def standby_generic (func):
32 minutes=int(func.__name__.split("_")[1])
33 return standby(minutes,self.options.dry_run)
36 def node_mapper (method):
39 node_method = TestNode.__dict__[method.__name__]
40 for site_spec in self.plc_spec['sites']:
41 test_site = TestSite (self,site_spec)
42 for node_spec in site_spec['nodes']:
43 test_node = TestNode (self,test_site,node_spec)
44 if not node_method(test_node): overall=False
48 def slice_mapper_options (method):
51 slice_method = TestSlice.__dict__[method.__name__]
52 for slice_spec in self.plc_spec['slices']:
53 site_spec = self.locate_site (slice_spec['sitename'])
54 test_site = TestSite(self,site_spec)
55 test_slice=TestSlice(self,test_site,slice_spec)
56 if not slice_method(test_slice,self.options): overall=False
64 default_steps = ['uninstall','install','install_rpm',
65 'configure', 'start', SEP,
66 'store_keys', 'clear_known_hosts', 'initscripts', SEP,
67 'sites', 'nodes', 'slices', 'nodegroups', SEP,
68 'init_node','bootcd', 'configure_qemu', 'export_qemu',
69 'kill_all_qemus', 'reinstall_node','start_node', SEP,
70 'nodes_booted', 'nodes_ssh', 'check_slice',
71 'check_initscripts', 'check_tcp',SEP,
72 'force_gather_logs', 'force_kill_qemus', ]
73 other_steps = [ 'stop_all_vservers','fresh_install', 'cache_rpm', 'stop', SEP,
74 'clean_sites', 'clean_nodes', 'clean_slices', 'clean_keys', SEP,
75 'show_boxes', 'list_all_qemus', 'list_qemus', SEP,
76 'db_dump' , 'db_restore',
77 'standby_1 through 20'
81 def printable_steps (list):
82 return " ".join(list).replace(" "+SEP+" ","\n")
84 def valid_step (step):
87 def __init__ (self,plc_spec,options):
88 self.plc_spec=plc_spec
90 self.test_ssh=TestSsh(self.plc_spec['hostname'],self.options.buildname)
92 self.vserverip=plc_spec['vserverip']
93 self.vservername=plc_spec['vservername']
94 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
98 self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
99 # utils.header('Using API url %s'%self.url)
100 self.apiserver=TestApiserver(self.url,options.dry_run)
103 name=self.plc_spec['name']
105 return name+".vserver.%s"%self.vservername
107 return name+".chroot"
110 return self.plc_spec['hostname']
113 return self.test_ssh.is_local()
115 # define the API methods on this object through xmlrpc
116 # would help, but not strictly necessary
120 def actual_command_in_guest (self,command):
121 return self.test_ssh.actual_command(self.host_to_guest(command))
123 def run_in_guest (self,command):
124 return utils.system(self.actual_command_in_guest(command))
126 def run_in_host (self,command):
127 return self.test_ssh.run_in_buildname(command)
129 #command gets run in the chroot/vserver
130 def host_to_guest(self,command):
132 return "vserver %s exec %s"%(self.vservername,command)
134 return "chroot /plc/root %s"%TestSsh.backslash_shell_specials(command)
136 # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
137 def copy_in_guest (self, localfile, remotefile, in_data=False):
139 chroot_dest="/plc/data"
141 chroot_dest="/plc/root"
144 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
146 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
149 utils.system("scp %s %s:%s/%s"%(localfile,self.hostname(),chroot_dest,remotefile))
151 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.hostname(),self.vservername,remotefile))
155 def run_in_guest_piped (self,local,remote):
156 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),keep_stdin=True))
158 def auth_root (self):
159 return {'Username':self.plc_spec['PLC_ROOT_USER'],
160 'AuthMethod':'password',
161 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
162 'Role' : self.plc_spec['role']
164 def locate_site (self,sitename):
165 for site in self.plc_spec['sites']:
166 if site['site_fields']['name'] == sitename:
168 if site['site_fields']['login_base'] == sitename:
170 raise Exception,"Cannot locate site %s"%sitename
172 def locate_node (self,nodename):
173 for site in self.plc_spec['sites']:
174 for node in site['nodes']:
175 if node['name'] == nodename:
177 raise Exception,"Cannot locate node %s"%nodename
179 def locate_hostname (self,hostname):
180 for site in self.plc_spec['sites']:
181 for node in site['nodes']:
182 if node['node_fields']['hostname'] == hostname:
184 raise Exception,"Cannot locate hostname %s"%hostname
186 def locate_key (self,keyname):
187 for key in self.plc_spec['keys']:
188 if key['name'] == keyname:
190 raise Exception,"Cannot locate key %s"%keyname
192 def locate_slice (self, slicename):
193 for slice in self.plc_spec['slices']:
194 if slice['slice_fields']['name'] == slicename:
196 raise Exception,"Cannot locate slice %s"%slicename
198 # all different hostboxes used in this plc
199 def gather_hostBoxes(self):
200 # maps on sites and nodes, return [ (host_box,test_node) ]
202 for site_spec in self.plc_spec['sites']:
203 test_site = TestSite (self,site_spec)
204 for node_spec in site_spec['nodes']:
205 test_node = TestNode (self, test_site, node_spec)
206 if not test_node.is_real():
207 tuples.append( (test_node.host_box(),test_node) )
208 # transform into a dict { 'host_box' -> [ test_node .. ] }
210 for (box,node) in tuples:
211 if not result.has_key(box):
214 result[box].append(node)
217 # a step for checking this stuff
218 def show_boxes (self):
219 for (box,nodes) in self.gather_hostBoxes().iteritems():
220 print box,":"," + ".join( [ node.name() for node in nodes ] )
223 # make this a valid step
224 def kill_all_qemus(self):
225 # this is the brute force version, kill all qemus on that host box
226 for (box,nodes) in self.gather_hostBoxes().iteritems():
227 # pass the first nodename, as we don't push template-qemu on testboxes
228 nodedir=nodes[0].nodedir()
229 TestBox(box,self.options.buildname).kill_all_qemus(nodedir)
232 # make this a valid step
233 def list_all_qemus(self):
234 for (box,nodes) in self.gather_hostBoxes().iteritems():
235 # this is the brute force version, kill all qemus on that host box
236 TestBox(box,self.options.buildname).list_all_qemus()
239 # kill only the right qemus
240 def list_qemus(self):
241 for (box,nodes) in self.gather_hostBoxes().iteritems():
242 # the fine-grain version
247 # kill only the right qemus
248 def kill_qemus(self):
249 for (box,nodes) in self.gather_hostBoxes().iteritems():
250 # the fine-grain version
255 #################### step methods
258 def uninstall_chroot(self):
259 self.run_in_host('service plc safestop')
260 #####detecting the last myplc version installed and remove it
261 self.run_in_host('rpm -e myplc')
262 ##### Clean up the /plc directory
263 self.run_in_host('rm -rf /plc/data')
264 ##### stop any running vservers
265 self.run_in_host('for vserver in $(ls -d /vservers/* | sed -e s,/vservers/,,) ; do case $vserver in vtest*) echo Shutting down vserver $vserver ; vserver $vserver stop ;; esac ; done')
268 def uninstall_vserver(self):
269 self.run_in_host("vserver --silent %s delete"%self.vservername)
273 # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
274 # it sounds safer to have the former uninstalled too
275 # now the vserver method cannot be invoked for chroot instances as vservername is required
277 self.uninstall_vserver()
278 self.uninstall_chroot()
280 self.uninstall_chroot()
284 def install_chroot(self):
288 def install_vserver(self):
289 # we need build dir for vtest-init-vserver
291 # a full path for the local calls
292 build_dir=os.path.dirname(sys.argv[0])+"/build"
294 # use a standard name - will be relative to HOME
295 build_dir="options.buildname"
296 # run checkout in any case - would do an update if already exists
297 build_checkout = "svn checkout %s %s"%(self.options.build_url,build_dir)
298 if self.run_in_host(build_checkout) != 0:
300 # the repo url is taken from myplc-url
301 # with the last two steps (i386/myplc...) removed
302 repo_url = self.options.myplc_url
303 for level in [ 'rpmname','arch' ]:
304 repo_url = os.path.dirname(repo_url)
305 create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
306 (build_dir,self.vservername,repo_url,self.vserverip)
307 return self.run_in_host(create_vserver) == 0
311 return self.install_vserver()
313 return self.install_chroot()
315 ### install_rpm - make this an optional step
317 url = self.options.myplc_url
318 rpm = os.path.basename(url)
319 cache_fetch="pwd;if [ -f %(rpm)s ] ; then echo Using cached rpm %(rpm)s ; else echo Fetching %(url)s ; curl -O %(url)s; fi"%locals()
320 return self.run_in_host(cache_fetch)==0
322 def install_rpm_chroot(self):
323 url = self.options.myplc_url
324 rpm = os.path.basename(url)
325 if not self.cache_rpm():
327 utils.header('Installing the : %s'%rpm)
328 return self.run_in_host('rpm -Uvh '+rpm)==0 and self.run_in_host('service plc mount')==0
330 def install_rpm_vserver(self):
331 return self.run_in_guest("yum -y install myplc-native")==0
333 def install_rpm(self):
335 return self.install_rpm_vserver()
337 return self.install_rpm_chroot()
341 tmpname='%s.plc-config-tty'%(self.name())
342 fileconf=open(tmpname,'w')
343 for var in [ 'PLC_NAME',
347 'PLC_MAIL_SUPPORT_ADDRESS',
354 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
355 fileconf.write('w\n')
356 fileconf.write('q\n')
358 utils.system('cat %s'%tmpname)
359 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
360 utils.system('rm %s'%tmpname)
363 # the chroot install is slightly different to this respect
366 self.run_in_guest('service plc start')
368 self.run_in_host('service plc start')
373 self.run_in_guest('service plc stop')
375 self.run_in_host('service plc stop')
378 # could use a TestKey class
379 def store_keys(self):
380 for key_spec in self.plc_spec['keys']:
381 TestKey(self,key_spec).store_key()
384 def clean_keys(self):
385 utils.system("rm -rf %s/keys/"%os.path(sys.argv[0]))
388 return self.do_sites()
390 def clean_sites (self):
391 return self.do_sites(action="delete")
393 def do_sites (self,action="add"):
394 for site_spec in self.plc_spec['sites']:
395 test_site = TestSite (self,site_spec)
396 if (action != "add"):
397 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
398 test_site.delete_site()
399 # deleted with the site
400 #test_site.delete_users()
403 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
404 test_site.create_site()
405 test_site.create_users()
409 return self.do_nodes()
410 def clean_nodes (self):
411 return self.do_nodes(action="delete")
413 def do_nodes (self,action="add"):
414 for site_spec in self.plc_spec['sites']:
415 test_site = TestSite (self,site_spec)
417 utils.header("Deleting nodes in site %s"%test_site.name())
418 for node_spec in site_spec['nodes']:
419 test_node=TestNode(self,test_site,node_spec)
420 utils.header("Deleting %s"%test_node.name())
421 test_node.delete_node()
423 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
424 for node_spec in site_spec['nodes']:
425 utils.pprint('Creating node %s'%node_spec,node_spec)
426 test_node = TestNode (self,test_site,node_spec)
427 test_node.create_node ()
430 # create nodegroups if needed, and populate
431 # no need for a clean_nodegroups if we are careful enough
432 def nodegroups (self):
433 # 1st pass to scan contents
435 for site_spec in self.plc_spec['sites']:
436 test_site = TestSite (self,site_spec)
437 for node_spec in site_spec['nodes']:
438 test_node=TestNode (self,test_site,node_spec)
439 if node_spec.has_key('nodegroups'):
440 nodegroupnames=node_spec['nodegroups']
441 if isinstance(nodegroupnames,StringTypes):
442 nodegroupnames = [ nodegroupnames ]
443 for nodegroupname in nodegroupnames:
444 if not groups_dict.has_key(nodegroupname):
445 groups_dict[nodegroupname]=[]
446 groups_dict[nodegroupname].append(test_node.name())
447 auth=self.auth_root()
448 for (nodegroupname,group_nodes) in groups_dict.iteritems():
450 self.apiserver.GetNodeGroups(auth,{'name':nodegroupname})[0]
452 self.apiserver.AddNodeGroup(auth,{'name':nodegroupname})
453 for node in group_nodes:
454 self.apiserver.AddNodeToNodeGroup(auth,node,nodegroupname)
457 def all_hostnames (self) :
459 for site_spec in self.plc_spec['sites']:
460 hostnames += [ node_spec['node_fields']['hostname'] \
461 for node_spec in site_spec['nodes'] ]
464 # gracetime : during the first <gracetime> minutes nothing gets printed
465 def do_nodes_booted (self, minutes, gracetime,period=30):
466 if self.options.dry_run:
470 timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
471 graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
472 # the nodes that haven't checked yet - start with a full list and shrink over time
473 tocheck = self.all_hostnames()
474 utils.header("checking nodes %r"%tocheck)
475 # create a dict hostname -> status
476 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
479 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
481 for array in tocheck_status:
482 hostname=array['hostname']
483 boot_state=array['boot_state']
484 if boot_state == 'boot':
485 utils.header ("%s has reached the 'boot' state"%hostname)
487 # if it's a real node, never mind
488 (site_spec,node_spec)=self.locate_hostname(hostname)
489 if TestNode.is_real_model(node_spec['node_fields']['model']):
490 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
493 elif datetime.datetime.now() > graceout:
494 utils.header ("%s still in '%s' state"%(hostname,boot_state))
495 graceout=datetime.datetime.now()+datetime.timedelta(1)
496 status[hostname] = boot_state
498 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
501 if datetime.datetime.now() > timeout:
502 for hostname in tocheck:
503 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
505 # otherwise, sleep for a while
507 # only useful in empty plcs
510 def nodes_booted(self):
511 return self.do_nodes_booted(minutes=20,gracetime=15)
513 def do_nodes_ssh(self,minutes,gracetime,period=30):
515 timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
516 graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
517 tocheck = self.all_hostnames()
518 # self.scan_publicKeys(tocheck)
519 utils.header("checking Connectivity on nodes %r"%tocheck)
521 for hostname in tocheck:
522 # try to ssh in nodes
523 node_test_ssh = TestSsh (hostname,key="/etc/planetlab/root_ssh_key.rsa")
524 success=self.run_in_guest(node_test_ssh.actual_command("hostname"))==0
526 utils.header('The node %s is sshable -->'%hostname)
528 tocheck.remove(hostname)
530 # we will have tried real nodes once, in case they're up - but if not, just skip
531 (site_spec,node_spec)=self.locate_hostname(hostname)
532 if TestNode.is_real_model(node_spec['node_fields']['model']):
533 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
534 tocheck.remove(hostname)
535 elif datetime.datetime.now() > graceout:
536 utils.header("Could not ssh-enter root context on %s"%hostname)
539 if datetime.datetime.now() > timeout:
540 for hostname in tocheck:
541 utils.header("FAILURE to ssh into %s"%hostname)
543 # otherwise, sleep for a while
545 # only useful in empty plcs
549 return self.do_nodes_ssh(minutes=6,gracetime=4)
552 def init_node (self): pass
554 def bootcd (self): pass
556 def configure_qemu (self): pass
558 def reinstall_node (self): pass
560 def export_qemu (self): pass
562 def do_check_initscripts(self):
564 for slice_spec in self.plc_spec['slices']:
565 if not slice_spec.has_key('initscriptname'):
567 initscript=slice_spec['initscriptname']
568 for nodename in slice_spec['nodenames']:
569 (site,node) = self.locate_node (nodename)
570 # xxx - passing the wrong site - probably harmless
571 test_site = TestSite (self,site)
572 test_slice = TestSlice (self,test_site,slice_spec)
573 test_node = TestNode (self,test_site,node)
574 test_sliver = TestSliver (self, test_node, test_slice)
575 if not test_sliver.check_initscript(initscript):
579 def check_initscripts(self):
580 return self.do_check_initscripts()
582 def initscripts (self):
583 for initscript in self.plc_spec['initscripts']:
584 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
585 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
589 return self.do_slices()
591 def clean_slices (self):
592 return self.do_slices("delete")
594 def do_slices (self, action="add"):
595 for slice in self.plc_spec['slices']:
596 site_spec = self.locate_site (slice['sitename'])
597 test_site = TestSite(self,site_spec)
598 test_slice=TestSlice(self,test_site,slice)
600 utils.header("Deleting slices in site %s"%test_site.name())
601 test_slice.delete_slice()
603 utils.pprint("Creating slice",slice)
604 test_slice.create_slice()
605 utils.header('Created Slice %s'%slice['slice_fields']['name'])
608 @slice_mapper_options
609 def check_slice(self): pass
612 def clear_known_hosts (self): pass
615 def start_node (self) : pass
617 def all_sliver_objs (self):
619 for slice_spec in self.plc_spec['slices']:
620 slicename = slice_spec['slice_fields']['name']
621 for nodename in slice_spec['nodenames']:
622 result.append(self.locate_sliver_obj (nodename,slicename))
625 def locate_sliver_obj (self,nodename,slicename):
626 (site,node) = self.locate_node(nodename)
627 slice = self.locate_slice (slicename)
629 test_site = TestSite (self, site)
630 test_node = TestNode (self, test_site,node)
631 # xxx the slice site is assumed to be the node site - mhh - probably harmless
632 test_slice = TestSlice (self, test_site, slice)
633 return TestSliver (self, test_node, test_slice)
635 def check_tcp (self):
636 specs = self.plc_spec['tcp_test']
641 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
642 if not s_test_sliver.run_tcp_server(port,timeout=10):
646 # idem for the client side
647 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
648 if not c_test_sliver.run_tcp_client(s_test_sliver.test_node.name(),port):
653 def gather_logs (self):
654 # (1) get the plc's /var/log and store it locally in logs/myplc.var-log.<plcname>/*
655 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
656 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
657 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
659 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
660 self.gather_var_logs ()
662 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
663 for site_spec in self.plc_spec['sites']:
664 test_site = TestSite (self,site_spec)
665 for node_spec in site_spec['nodes']:
666 test_node=TestNode(self,test_site,node_spec)
667 test_node.gather_qemu_logs()
669 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
670 self.gather_nodes_var_logs()
672 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
673 self.gather_slivers_var_logs()
676 def gather_slivers_var_logs(self):
677 for test_sliver in self.all_sliver_objs():
678 remote = test_sliver.tar_var_logs()
679 utils.system("mkdir -p logs/sliver.var-log.%s"%test_sliver.name())
680 command = remote + " | tar -C logs/sliver.var-log.%s -xf -"%test_sliver.name()
681 utils.system(command)
684 def gather_var_logs (self):
685 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
686 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -"%self.name()
687 utils.system("mkdir -p logs/myplc.var-log.%s"%self.name())
688 utils.system(command)
690 def gather_nodes_var_logs (self):
691 for site_spec in self.plc_spec['sites']:
692 test_site = TestSite (self,site_spec)
693 for node_spec in site_spec['nodes']:
694 test_node=TestNode(self,test_site,node_spec)
695 test_ssh = TestSsh (test_node.name(),key="/etc/planetlab/root_ssh_key.rsa")
696 to_plc = self.actual_command_in_guest ( test_ssh.actual_command("tar -C /var/log -cf - ."))
697 command = to_plc + "| tar -C logs/node.var-log.%s -xf -"%test_node.name()
698 utils.system("mkdir -p logs/node.var-log.%s"%test_node.name())
699 utils.system(command)
702 # returns the filename to use for sql dump/restore, using options.dbname if set
703 def dbfile (self, database):
704 # uses options.dbname if it is found
706 name=self.options.dbname
707 if not isinstance(name,StringTypes):
710 t=datetime.datetime.now()
713 return "/root/%s-%s.sql"%(database,name)
716 dump=self.dbfile("planetab4")
717 self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
718 utils.header('Dumped planetlab4 database in %s'%dump)
721 def db_restore(self):
722 dump=self.dbfile("planetab4")
724 self.run_in_guest('service httpd stop')
725 # xxx - need another wrapper
726 self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
727 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
728 self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
729 ##starting httpd service
730 self.run_in_guest('service httpd start')
732 utils.header('Database restored from ' + dump)
735 def standby_1(): pass
737 def standby_2(): pass
739 def standby_3(): pass
741 def standby_4(): pass
743 def standby_5(): pass
745 def standby_6(): pass
747 def standby_7(): pass
749 def standby_8(): pass
751 def standby_9(): pass
753 def standby_10(): pass
755 def standby_11(): pass
757 def standby_12(): pass
759 def standby_13(): pass
761 def standby_14(): pass
763 def standby_15(): pass
765 def standby_16(): pass
767 def standby_17(): pass
769 def standby_18(): pass
771 def standby_19(): pass
773 def standby_20(): pass