8 from types import StringTypes
11 from TestSite import TestSite
12 from TestNode import TestNode
13 from TestUser import TestUser
14 from TestKey import TestKey
15 from TestSlice import TestSlice
16 from TestSliver import TestSliver
17 from TestBox import TestBox
18 from TestSsh import TestSsh
19 from TestApiserver import TestApiserver
21 # step methods must take (self) and return a boolean (options is a member of the class)
23 def standby(minutes,dry_run):
24 utils.header('Entering StandBy for %d mn'%minutes)
28 time.sleep(60*minutes)
31 def standby_generic (func):
33 minutes=int(func.__name__.split("_")[1])
34 return standby(minutes,self.options.dry_run)
37 def node_mapper (method):
40 node_method = TestNode.__dict__[method.__name__]
41 for site_spec in self.plc_spec['sites']:
42 test_site = TestSite (self,site_spec)
43 for node_spec in site_spec['nodes']:
44 test_node = TestNode (self,test_site,node_spec)
45 if not node_method(test_node): overall=False
49 def slice_mapper_options (method):
52 slice_method = TestSlice.__dict__[method.__name__]
53 for slice_spec in self.plc_spec['slices']:
54 site_spec = self.locate_site (slice_spec['sitename'])
55 test_site = TestSite(self,site_spec)
56 test_slice=TestSlice(self,test_site,slice_spec)
57 if not slice_method(test_slice,self.options): overall=False
63 def __init__ (self,plc_spec,options):
64 self.plc_spec=plc_spec
66 self.test_ssh=TestSsh(self.plc_spec['hostname'],self.options.buildname)
68 self.vserverip=plc_spec['vserverip']
69 self.vservername=plc_spec['vservername']
70 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
74 self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
75 # utils.header('Using API url %s'%self.url)
76 self.apiserver=TestApiserver(self.url,options.dry_run)
79 name=self.plc_spec['name']
81 return name+".vserver.%s"%self.vservername
86 return self.plc_spec['hostname']
89 return self.test_ssh.is_local()
91 # define the API methods on this object through xmlrpc
92 # would help, but not strictly necessary
96 def actual_command_in_guest (self,command):
97 return self.test_ssh.actual_command(self.host_to_guest(command))
99 def run_in_guest (self,command):
100 return utils.system(self.actual_command_in_guest(command))
102 def run_in_host (self,command):
103 return self.test_ssh.run_in_buildname(command)
105 #command gets run in the chroot/vserver
106 def host_to_guest(self,command):
108 return "vserver %s exec %s"%(self.vservername,command)
110 return "chroot /plc/root %s"%TestSsh.backslash_shell_specials(command)
112 # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
113 def copy_in_guest (self, localfile, remotefile, in_data=False):
115 chroot_dest="/plc/data"
117 chroot_dest="/plc/root"
120 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
122 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
125 utils.system("scp %s %s:%s/%s"%(localfile,self.hostname(),chroot_dest,remotefile))
127 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.hostname(),self.vservername,remotefile))
131 def run_in_guest_piped (self,local,remote):
132 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote)))
134 def auth_root (self):
135 return {'Username':self.plc_spec['PLC_ROOT_USER'],
136 'AuthMethod':'password',
137 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
138 'Role' : self.plc_spec['role']
140 def locate_site (self,sitename):
141 for site in self.plc_spec['sites']:
142 if site['site_fields']['name'] == sitename:
144 if site['site_fields']['login_base'] == sitename:
146 raise Exception,"Cannot locate site %s"%sitename
148 def locate_node (self,nodename):
149 for site in self.plc_spec['sites']:
150 for node in site['nodes']:
151 if node['name'] == nodename:
153 raise Exception,"Cannot locate node %s"%nodename
155 def locate_hostname (self,hostname):
156 for site in self.plc_spec['sites']:
157 for node in site['nodes']:
158 if node['node_fields']['hostname'] == hostname:
160 raise Exception,"Cannot locate hostname %s"%hostname
162 def locate_key (self,keyname):
163 for key in self.plc_spec['keys']:
164 if key['name'] == keyname:
166 raise Exception,"Cannot locate key %s"%keyname
168 def locate_slice (self, slicename):
169 for slice in self.plc_spec['slices']:
170 if slice['slice_fields']['name'] == slicename:
172 raise Exception,"Cannot locate slice %s"%slicename
174 # all different hostboxes used in this plc
175 def gather_hostBoxes(self):
176 # maps on sites and nodes, return [ (host_box,test_node) ]
178 for site_spec in self.plc_spec['sites']:
179 test_site = TestSite (self,site_spec)
180 for node_spec in site_spec['nodes']:
181 test_node = TestNode (self, test_site, node_spec)
182 if not test_node.is_real():
183 tuples.append( (test_node.host_box(),test_node) )
184 # transform into a dict { 'host_box' -> [ test_node .. ] }
186 for (box,node) in tuples:
187 if not result.has_key(box):
190 result[box].append(node)
193 # a step for checking this stuff
194 def show_boxes (self):
195 for (box,nodes) in self.gather_hostBoxes().iteritems():
196 print box,":"," + ".join( [ node.name() for node in nodes ] )
199 # make this a valid step
200 def kill_all_qemus(self):
201 # this is the brute force version, kill all qemus on that host box
202 for (box,nodes) in self.gather_hostBoxes().iteritems():
203 # pass the first nodename, as we don't push template-qemu on testboxes
204 nodename=nodes[0].name()
205 TestBox(box,self.options.buildname).kill_all_qemus(nodename)
208 # make this a valid step
209 def list_all_qemus(self):
210 for (box,nodes) in self.gather_hostBoxes().iteritems():
211 # this is the brute force version, kill all qemus on that host box
212 TestBox(box,self.options.buildname).list_all_qemus()
215 # kill only the right qemus
216 def list_qemus(self):
217 for (box,nodes) in self.gather_hostBoxes().iteritems():
218 # the fine-grain version
223 # kill only the right qemus
224 def kill_qemus(self):
225 for (box,nodes) in self.gather_hostBoxes().iteritems():
226 # the fine-grain version
231 #################### step methods
234 def uninstall_chroot(self):
235 self.run_in_host('service plc safestop')
236 #####detecting the last myplc version installed and remove it
237 self.run_in_host('rpm -e myplc')
238 ##### Clean up the /plc directory
239 self.run_in_host('rm -rf /plc/data')
240 ##### stop any running vservers
241 self.run_in_host('for vserver in $(ls -d /vservers/* | sed -e s,/vservers/,,) ; do case $vserver in vtest*) echo Shutting down vserver $vserver ; vserver $vserver stop ;; esac ; done')
244 def uninstall_vserver(self):
245 self.run_in_host("vserver --silent %s delete"%self.vservername)
249 # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
250 # it sounds safer to have the former uninstalled too
251 # now the vserver method cannot be invoked for chroot instances as vservername is required
253 self.uninstall_vserver()
254 self.uninstall_chroot()
256 self.uninstall_chroot()
260 def install_chroot(self):
264 def install_vserver(self):
265 # we need build dir for vtest-init-vserver
267 # a full path for the local calls
268 build_dir=os.path(sys.argv[0])+"/build"
270 # use a standard name - will be relative to HOME
271 build_dir="options.buildname"
272 # run checkout in any case - would do an update if already exists
273 build_checkout = "svn checkout %s %s"%(self.options.build_url,build_dir)
274 if self.run_in_host(build_checkout) != 0:
276 # the repo url is taken from myplc-url
277 # with the last two steps (i386/myplc...) removed
278 repo_url = self.options.myplc_url
279 for level in [ 'rpmname','arch' ]:
280 repo_url = os.path.dirname(repo_url)
281 create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
282 (build_dir,self.vservername,repo_url,self.vserverip)
283 return self.run_in_host(create_vserver) == 0
287 return self.install_vserver()
289 return self.install_chroot()
291 ### install_rpm - make this an optional step
293 url = self.options.myplc_url
294 rpm = os.path.basename(url)
295 cache_fetch="pwd;if [ -f %(rpm)s ] ; then echo Using cached rpm %(rpm)s ; else echo Fetching %(url)s ; curl -O %(url)s; fi"%locals()
296 return self.run_in_host(cache_fetch)==0
298 def install_rpm_chroot(self):
299 url = self.options.myplc_url
300 rpm = os.path.basename(url)
301 if not self.cache_rpm():
303 utils.header('Installing the : %s'%rpm)
304 return self.run_in_host('rpm -Uvh '+rpm)==0 and self.run_in_host('service plc mount')==0
306 def install_rpm_vserver(self):
307 return self.run_in_guest("yum -y install myplc-native")==0
309 def install_rpm(self):
311 return self.install_rpm_vserver()
313 return self.install_rpm_chroot()
317 tmpname='%s.plc-config-tty'%(self.name())
318 fileconf=open(tmpname,'w')
319 for var in [ 'PLC_NAME',
323 'PLC_MAIL_SUPPORT_ADDRESS',
330 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
331 fileconf.write('w\n')
332 fileconf.write('q\n')
334 utils.system('cat %s'%tmpname)
335 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
336 utils.system('rm %s'%tmpname)
339 # the chroot install is slightly different to this respect
342 self.run_in_guest('service plc start')
344 self.run_in_host('service plc start')
349 self.run_in_guest('service plc stop')
351 self.run_in_host('service plc stop')
354 # could use a TestKey class
355 def store_keys(self):
356 for key_spec in self.plc_spec['keys']:
357 TestKey(self,key_spec).store_key()
360 def clean_keys(self):
361 utils.system("rm -rf %s/keys/"%os.path(sys.argv[0]))
364 return self.do_sites()
366 def clean_sites (self):
367 return self.do_sites(action="delete")
369 def do_sites (self,action="add"):
370 for site_spec in self.plc_spec['sites']:
371 test_site = TestSite (self,site_spec)
372 if (action != "add"):
373 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
374 test_site.delete_site()
375 # deleted with the site
376 #test_site.delete_users()
379 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
380 test_site.create_site()
381 test_site.create_users()
385 return self.do_nodes()
386 def clean_nodes (self):
387 return self.do_nodes(action="delete")
389 def do_nodes (self,action="add"):
390 for site_spec in self.plc_spec['sites']:
391 test_site = TestSite (self,site_spec)
393 utils.header("Deleting nodes in site %s"%test_site.name())
394 for node_spec in site_spec['nodes']:
395 test_node=TestNode(self,test_site,node_spec)
396 utils.header("Deleting %s"%test_node.name())
397 test_node.delete_node()
399 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
400 for node_spec in site_spec['nodes']:
401 utils.pprint('Creating node %s'%node_spec,node_spec)
402 test_node = TestNode (self,test_site,node_spec)
403 test_node.create_node ()
406 # create nodegroups if needed, and populate
407 # no need for a clean_nodegroups if we are careful enough
408 def nodegroups (self):
409 # 1st pass to scan contents
411 for site_spec in self.plc_spec['sites']:
412 test_site = TestSite (self,site_spec)
413 for node_spec in site_spec['nodes']:
414 test_node=TestNode (self,test_site,node_spec)
415 if node_spec.has_key('nodegroups'):
416 nodegroupnames=node_spec['nodegroups']
417 if isinstance(nodegroupnames,StringTypes):
418 nodegroupnames = [ nodegroupnames ]
419 for nodegroupname in nodegroupnames:
420 if not groups_dict.has_key(nodegroupname):
421 groups_dict[nodegroupname]=[]
422 groups_dict[nodegroupname].append(test_node.name())
423 auth=self.auth_root()
424 for (nodegroupname,group_nodes) in groups_dict.iteritems():
426 self.apiserver.GetNodeGroups(auth,{'name':nodegroupname})[0]
428 self.apiserver.AddNodeGroup(auth,{'name':nodegroupname})
429 for node in group_nodes:
430 self.apiserver.AddNodeToNodeGroup(auth,node,nodegroupname)
433 def all_hostnames (self) :
435 for site_spec in self.plc_spec['sites']:
436 hostnames += [ node_spec['node_fields']['hostname'] \
437 for node_spec in site_spec['nodes'] ]
440 # gracetime : during the first <gracetime> minutes nothing gets printed
441 def do_nodes_booted (self, minutes, gracetime=2):
442 if self.options.dry_run:
446 timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
447 graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
448 # the nodes that haven't checked yet - start with a full list and shrink over time
449 tocheck = self.all_hostnames()
450 utils.header("checking nodes %r"%tocheck)
451 # create a dict hostname -> status
452 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
455 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
457 for array in tocheck_status:
458 hostname=array['hostname']
459 boot_state=array['boot_state']
460 if boot_state == 'boot':
461 utils.header ("%s has reached the 'boot' state"%hostname)
463 # if it's a real node, never mind
464 (site_spec,node_spec)=self.locate_hostname(hostname)
465 if TestNode.is_real_model(node_spec['node_fields']['model']):
466 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
469 if datetime.datetime.now() > graceout:
470 utils.header ("%s still in '%s' state"%(hostname,boot_state))
471 graceout=datetime.datetime.now()+datetime.timedelta(1)
472 status[hostname] = boot_state
474 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
477 if datetime.datetime.now() > timeout:
478 for hostname in tocheck:
479 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
481 # otherwise, sleep for a while
483 # only useful in empty plcs
486 def nodes_booted(self):
487 return self.do_nodes_booted(minutes=0)
490 def do_nodes_ssh(self,minutes):
492 timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
493 tocheck = self.all_hostnames()
494 # self.scan_publicKeys(tocheck)
495 utils.header("checking Connectivity on nodes %r"%tocheck)
497 for hostname in tocheck:
498 # try to ssh in nodes
499 node_test_ssh = TestSsh (hostname,key="/etc/planetlab/root_ssh_key.rsa")
500 access=self.run_in_guest(node_test_ssh.actual_command("date"))
502 utils.header('The node %s is sshable -->'%hostname)
504 tocheck.remove(hostname)
506 # we will have tried real nodes once, in case they're up - but if not, just skip
507 (site_spec,node_spec)=self.locate_hostname(hostname)
508 if TestNode.is_real_model(node_spec['node_fields']['model']):
509 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
510 tocheck.remove(hostname)
513 if datetime.datetime.now() > timeout:
514 for hostname in tocheck:
515 utils.header("FAILURE to ssh into %s"%hostname)
517 # otherwise, sleep for a while
519 # only useful in empty plcs
523 return self.do_nodes_ssh(minutes=2)
526 def init_node (self): pass
528 def bootcd (self): pass
530 def configure_qemu (self): pass
532 def reinstall_node (self): pass
534 def do_check_initscripts(self):
536 for slice_spec in self.plc_spec['slices']:
537 if not slice_spec.has_key('initscriptname'):
539 initscript=slice_spec['initscriptname']
540 for nodename in slice_spec['nodenames']:
541 (site,node) = self.locate_node (nodename)
542 # xxx - passing the wrong site - probably harmless
543 test_site = TestSite (self,site)
544 test_slice = TestSlice (self,test_site,slice_spec)
545 test_node = TestNode (self,test_site,node)
546 test_sliver = TestSliver (self, test_node, test_slice)
547 if not test_sliver.check_initscript(initscript):
551 def check_initscripts(self):
552 return self.do_check_initscripts()
554 def initscripts (self):
555 for initscript in self.plc_spec['initscripts']:
556 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
557 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
561 return self.do_slices()
563 def clean_slices (self):
564 return self.do_slices("delete")
566 def do_slices (self, action="add"):
567 for slice in self.plc_spec['slices']:
568 site_spec = self.locate_site (slice['sitename'])
569 test_site = TestSite(self,site_spec)
570 test_slice=TestSlice(self,test_site,slice)
572 utils.header("Deleting slices in site %s"%test_site.name())
573 test_slice.delete_slice()
575 utils.pprint("Creating slice",slice)
576 test_slice.create_slice()
577 utils.header('Created Slice %s'%slice['slice_fields']['name'])
580 @slice_mapper_options
581 def check_slice(self): pass
584 def clear_known_hosts (self): pass
587 def start_node (self) : pass
589 def locate_first_sliver (self):
590 slice_spec = self.plc_spec['slices'][0]
591 slicename = slice_spec['slice_fields']['name']
592 nodename = slice_spec['nodenames'][0]
593 return self.locate_sliver_obj(nodename,slicename)
595 def locate_sliver_obj (self,nodename,slicename):
596 (site,node) = self.locate_node(nodename)
597 slice = self.locate_slice (slicename)
599 test_site = TestSite (self, site)
600 test_node = TestNode (self, test_site,node)
601 # xxx the slice site is assumed to be the node site - mhh - probably harmless
602 test_slice = TestSlice (self, test_site, slice)
603 return TestSliver (self, test_node, test_slice)
605 def check_tcp (self):
606 specs = self.plc_spec['tcp_test']
611 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
612 if not s_test_sliver.run_tcp_server(port,timeout=10):
616 # idem for the client side
617 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
618 if not c_test_sliver.run_tcp_client(s_test_sliver.test_node.name(),port):
623 def gather_logs (self):
624 # (1) get the plc's /var/log and store it locally in logs/<plcname>-var-log/*
625 # (2) get all the nodes qemu log and store it as logs/<node>-qemu.log
626 # (3) get the nodes /var/log and store is as logs/<node>-var-log/*
627 # (4) as far as possible get the slice's /var/log as logs/<slice>-<node>-var-log/*
629 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
630 self.gather_var_logs ()
632 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
633 for site_spec in self.plc_spec['sites']:
634 test_site = TestSite (self,site_spec)
635 for node_spec in site_spec['nodes']:
636 test_node=TestNode(self,test_site,node_spec)
637 test_node.gather_qemu_logs()
639 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
640 self.gather_nodes_var_logs()
642 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
643 self.gather_first_sliver_logs()
646 def gather_first_sliver_logs(self):
648 test_sliver = self.locate_first_sliver()
649 remote = test_sliver.tar_var_logs()
650 utils.system("mkdir -p logs/%s-var-log"%test_sliver.name())
651 command = remote + " | tar -C logs/%s-var-log -xf -"%test_sliver.name()
652 utils.system(command)
654 print 'Cannot locate first sliver - giving up',e
657 def gather_var_logs (self):
658 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
659 command = to_plc + "| tar -C logs/%s-var-log -xf -"%self.name()
660 utils.system("mkdir -p logs/%s-var-log"%self.name())
661 utils.system(command)
663 def gather_nodes_var_logs (self):
664 for site_spec in self.plc_spec['sites']:
665 test_site = TestSite (self,site_spec)
666 for node_spec in site_spec['nodes']:
667 test_node=TestNode(self,test_site,node_spec)
668 test_ssh = TestSsh (test_node.name(),key="/etc/planetlab/root_ssh_key.rsa")
669 to_plc = self.actual_command_in_guest ( test_ssh.actual_command("tar -C /var/log -cf - ."))
670 command = to_plc + "| tar -C logs/%s-var-log -xf -"%test_node.name()
671 utils.system("mkdir -p logs/%s-var-log"%test_node.name())
672 utils.system(command)
675 # returns the filename to use for sql dump/restore, using options.dbname if set
676 def dbfile (self, database):
677 # uses options.dbname if it is found
679 name=self.options.dbname
680 if not isinstance(name,StringTypes):
683 t=datetime.datetime.now()
686 return "/root/%s-%s.sql"%(database,name)
689 dump=self.dbfile("planetab4")
690 self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
691 utils.header('Dumped planetlab4 database in %s'%dump)
694 def db_restore(self):
695 dump=self.dbfile("planetab4")
697 self.run_in_guest('service httpd stop')
698 # xxx - need another wrapper
699 self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
700 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
701 self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
702 ##starting httpd service
703 self.run_in_guest('service httpd start')
705 utils.header('Database restored from ' + dump)
708 def standby_1(): pass
710 def standby_2(): pass
712 def standby_3(): pass
714 def standby_4(): pass
716 def standby_5(): pass
718 def standby_6(): pass
720 def standby_7(): pass
722 def standby_8(): pass
724 def standby_9(): pass
726 def standby_10(): pass
728 def standby_11(): pass
730 def standby_12(): pass
732 def standby_13(): pass
734 def standby_14(): pass
736 def standby_15(): pass
738 def standby_16(): pass
740 def standby_17(): pass
742 def standby_18(): pass
744 def standby_19(): pass
746 def standby_20(): pass