8 from types import StringTypes
11 from TestSite import TestSite
12 from TestNode import TestNode
13 from TestUser import TestUser
14 from TestKey import TestKey
15 from TestSlice import TestSlice
16 from TestSliver import TestSliver
17 from TestBox import TestBox
18 from TestSsh import TestSsh
19 from TestApiserver import TestApiserver
21 # step methods must take (self) and return a boolean (options is a member of the class)
23 def standby(minutes,dry_run):
24 utils.header('Entering StandBy for %d mn'%minutes)
28 time.sleep(60*minutes)
31 def standby_generic (func):
33 minutes=int(func.__name__.split("_")[1])
34 return standby(minutes,self.options.dry_run)
37 def node_mapper (method):
40 for site_spec in self.plc_spec['sites']:
41 test_site = TestSite (self,site_spec)
42 for node_spec in site_spec['nodes']:
43 test_node = TestNode (self,test_site,node_spec)
44 node_method = TestNode.__dict__[method.__name__]
45 if not node_method(test_node): overall=False
51 def __init__ (self,plc_spec,options):
52 self.plc_spec=plc_spec
54 self.test_ssh=TestSsh(self.plc_spec['hostname'],self.options.buildname)
56 self.vserverip=plc_spec['vserverip']
57 self.vservername=plc_spec['vservername']
58 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
62 self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
63 # utils.header('Using API url %s'%self.url)
64 self.apiserver=TestApiserver(self.url,options.dry_run)
67 name=self.plc_spec['name']
69 return name+".vserver.%s"%self.vservername
74 return self.plc_spec['hostname']
77 return self.test_ssh.is_local()
79 # define the API methods on this object through xmlrpc
80 # would help, but not strictly necessary
84 def actual_command_in_guest (self,command):
85 return self.test_ssh.actual_command(self.host_to_guest(command))
87 def run_in_guest (self,command):
88 return utils.system(self.actual_command_in_guest(command))
90 def run_in_host (self,command):
91 return self.test_ssh.run_in_buildname(command)
93 #command gets run in the chroot/vserver
94 def host_to_guest(self,command):
96 return "vserver %s exec %s"%(self.vservername,command)
98 return "chroot /plc/root %s"%TestSsh.backslash_shell_specials(command)
100 # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
101 def copy_in_guest (self, localfile, remotefile, in_data=False):
103 chroot_dest="/plc/data"
105 chroot_dest="/plc/root"
108 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
110 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
113 utils.system("scp %s %s:%s/%s"%(localfile,self.hostname(),chroot_dest,remotefile))
115 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.hostname(),self.vservername,remotefile))
119 def run_in_guest_piped (self,local,remote):
120 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote)))
122 def auth_root (self):
123 return {'Username':self.plc_spec['PLC_ROOT_USER'],
124 'AuthMethod':'password',
125 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
126 'Role' : self.plc_spec['role']
128 def locate_site (self,sitename):
129 for site in self.plc_spec['sites']:
130 if site['site_fields']['name'] == sitename:
132 if site['site_fields']['login_base'] == sitename:
134 raise Exception,"Cannot locate site %s"%sitename
136 def locate_node (self,nodename):
137 for site in self.plc_spec['sites']:
138 for node in site['nodes']:
139 if node['name'] == nodename:
141 raise Exception,"Cannot locate node %s"%nodename
143 def locate_hostname (self,hostname):
144 for site in self.plc_spec['sites']:
145 for node in site['nodes']:
146 if node['node_fields']['hostname'] == hostname:
148 raise Exception,"Cannot locate hostname %s"%hostname
150 def locate_key (self,keyname):
151 for key in self.plc_spec['keys']:
152 if key['name'] == keyname:
154 raise Exception,"Cannot locate key %s"%keyname
156 def locate_slice (self, slicename):
157 for slice in self.plc_spec['slices']:
158 if slice['slice_fields']['name'] == slicename:
160 raise Exception,"Cannot locate slice %s"%slicename
162 # all different hostboxes used in this plc
163 def gather_hostBoxes(self):
164 # maps on sites and nodes, return [ (host_box,test_node) ]
166 for site_spec in self.plc_spec['sites']:
167 test_site = TestSite (self,site_spec)
168 for node_spec in site_spec['nodes']:
169 test_node = TestNode (self, test_site, node_spec)
170 if not test_node.is_real():
171 tuples.append( (test_node.host_box(),test_node) )
172 # transform into a dict { 'host_box' -> [ hostnames .. ] }
174 for (box,node) in tuples:
175 if not result.has_key(box):
178 result[box].append(node)
181 # a step for checking this stuff
182 def show_boxes (self):
183 for (box,nodes) in self.gather_hostBoxes().iteritems():
184 print box,":"," + ".join( [ node.name() for node in nodes ] )
187 # make this a valid step
188 def kill_all_qemus(self):
189 for (box,nodes) in self.gather_hostBoxes().iteritems():
190 # this is the brute force version, kill all qemus on that host box
191 TestBox(box,self.options.buildname).kill_all_qemus()
194 # make this a valid step
195 def list_all_qemus(self):
196 for (box,nodes) in self.gather_hostBoxes().iteritems():
197 # this is the brute force version, kill all qemus on that host box
198 TestBox(box,self.options.buildname).list_all_qemus()
201 # kill only the right qemus
202 def list_qemus(self):
203 for (box,nodes) in self.gather_hostBoxes().iteritems():
204 # the fine-grain version
209 # kill only the right qemus
210 def kill_qemus(self):
211 for (box,nodes) in self.gather_hostBoxes().iteritems():
212 # the fine-grain version
217 #################### step methods
220 def uninstall_chroot(self):
221 self.run_in_host('service plc safestop')
222 #####detecting the last myplc version installed and remove it
223 self.run_in_host('rpm -e myplc')
224 ##### Clean up the /plc directory
225 self.run_in_host('rm -rf /plc/data')
226 ##### stop any running vservers
227 self.run_in_host('for vserver in $(ls -d /vservers/* | sed -e s,/vservers/,,) ; do case $vserver in vtest*) echo Shutting down vserver $vserver ; vserver $vserver stop ;; esac ; done')
230 def uninstall_vserver(self):
231 self.run_in_host("vserver --silent %s delete"%self.vservername)
235 # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
236 # it sounds safer to have the former uninstalled too
237 # now the vserver method cannot be invoked for chroot instances as vservername is required
239 self.uninstall_vserver()
240 self.uninstall_chroot()
242 self.uninstall_chroot()
246 def install_chroot(self):
250 def install_vserver(self):
251 # we need build dir for vtest-init-vserver
253 # a full path for the local calls
254 build_dir=os.path(sys.argv[0])+"/build"
256 # use a standard name - will be relative to HOME
257 build_dir="options.buildname"
258 # run checkout in any case - would do an update if already exists
259 build_checkout = "svn checkout %s %s"%(self.options.build_url,build_dir)
260 if self.run_in_host(build_checkout) != 0:
262 # the repo url is taken from myplc-url
263 # with the last two steps (i386/myplc...) removed
264 repo_url = self.options.myplc_url
265 for level in [ 'rpmname','arch' ]:
266 repo_url = os.path.dirname(repo_url)
267 create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
268 (build_dir,self.vservername,repo_url,self.vserverip)
269 return self.run_in_host(create_vserver) == 0
273 return self.install_vserver()
275 return self.install_chroot()
277 ### install_rpm - make this an optional step
279 url = self.options.myplc_url
280 rpm = os.path.basename(url)
281 cache_fetch="pwd;if [ -f %(rpm)s ] ; then echo Using cached rpm %(rpm)s ; else echo Fetching %(url)s ; curl -O %(url)s; fi"%locals()
282 return self.run_in_host(cache_fetch)==0
284 def install_rpm_chroot(self):
285 url = self.options.myplc_url
286 rpm = os.path.basename(url)
287 if not self.cache_rpm():
289 utils.header('Installing the : %s'%rpm)
290 return self.run_in_host('rpm -Uvh '+rpm)==0 and self.run_in_host('service plc mount')==0
292 def install_rpm_vserver(self):
293 return self.run_in_guest("yum -y install myplc-native")==0
295 def install_rpm(self):
297 return self.install_rpm_vserver()
299 return self.install_rpm_chroot()
303 tmpname='%s.plc-config-tty'%(self.name())
304 fileconf=open(tmpname,'w')
305 for var in [ 'PLC_NAME',
309 'PLC_MAIL_SUPPORT_ADDRESS',
316 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
317 fileconf.write('w\n')
318 fileconf.write('q\n')
320 utils.system('cat %s'%tmpname)
321 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
322 utils.system('rm %s'%tmpname)
325 # the chroot install is slightly different to this respect
328 self.run_in_guest('service plc start')
330 self.run_in_host('service plc start')
335 self.run_in_guest('service plc stop')
337 self.run_in_host('service plc stop')
340 # could use a TestKey class
341 def store_keys(self):
342 for key_spec in self.plc_spec['keys']:
343 TestKey(self,key_spec).store_key()
346 def clean_keys(self):
347 utils.system("rm -rf %s/keys/"%os.path(sys.argv[0]))
350 return self.do_sites()
352 def clean_sites (self):
353 return self.do_sites(action="delete")
355 def do_sites (self,action="add"):
356 for site_spec in self.plc_spec['sites']:
357 test_site = TestSite (self,site_spec)
358 if (action != "add"):
359 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
360 test_site.delete_site()
361 # deleted with the site
362 #test_site.delete_users()
365 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
366 test_site.create_site()
367 test_site.create_users()
371 return self.do_nodes()
372 def clean_nodes (self):
373 return self.do_nodes(action="delete")
375 def do_nodes (self,action="add"):
376 for site_spec in self.plc_spec['sites']:
377 test_site = TestSite (self,site_spec)
379 utils.header("Deleting nodes in site %s"%test_site.name())
380 for node_spec in site_spec['nodes']:
381 test_node=TestNode(self,test_site,node_spec)
382 utils.header("Deleting %s"%test_node.name())
383 test_node.delete_node()
385 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
386 for node_spec in site_spec['nodes']:
387 utils.pprint('Creating node %s'%node_spec,node_spec)
388 test_node = TestNode (self,test_site,node_spec)
389 test_node.create_node ()
392 # create nodegroups if needed, and populate
393 # no need for a clean_nodegroups if we are careful enough
394 def nodegroups (self):
395 # 1st pass to scan contents
397 for site_spec in self.plc_spec['sites']:
398 test_site = TestSite (self,site_spec)
399 for node_spec in site_spec['nodes']:
400 test_node=TestNode (self,test_site,node_spec)
401 if node_spec.has_key('nodegroups'):
402 nodegroupnames=node_spec['nodegroups']
403 if isinstance(nodegroupnames,StringTypes):
404 nodegroupnames = [ nodegroupnames ]
405 for nodegroupname in nodegroupnames:
406 if not groups_dict.has_key(nodegroupname):
407 groups_dict[nodegroupname]=[]
408 groups_dict[nodegroupname].append(test_node.name())
409 auth=self.auth_root()
410 for (nodegroupname,group_nodes) in groups_dict.iteritems():
412 self.apiserver.GetNodeGroups(auth,{'name':nodegroupname})[0]
414 self.apiserver.AddNodeGroup(auth,{'name':nodegroupname})
415 for node in group_nodes:
416 self.apiserver.AddNodeToNodeGroup(auth,node,nodegroupname)
419 def all_hostnames (self) :
421 for site_spec in self.plc_spec['sites']:
422 hostnames += [ node_spec['node_fields']['hostname'] \
423 for node_spec in site_spec['nodes'] ]
426 # gracetime : during the first <gracetime> minutes nothing gets printed
427 def do_nodes_booted (self, minutes, gracetime=2):
428 if self.options.dry_run:
432 timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
433 graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
434 # the nodes that haven't checked yet - start with a full list and shrink over time
435 tocheck = self.all_hostnames()
436 utils.header("checking nodes %r"%tocheck)
437 # create a dict hostname -> status
438 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
441 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
443 for array in tocheck_status:
444 hostname=array['hostname']
445 boot_state=array['boot_state']
446 if boot_state == 'boot':
447 utils.header ("%s has reached the 'boot' state"%hostname)
449 # if it's a real node, never mind
450 (site_spec,node_spec)=self.locate_hostname(hostname)
451 if TestNode.is_real_model(node_spec['node_fields']['model']):
452 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
455 if datetime.datetime.now() > graceout:
456 utils.header ("%s still in '%s' state"%(hostname,boot_state))
457 graceout=datetime.datetime.now()+datetime.timedelta(1)
458 status[hostname] = boot_state
460 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
463 if datetime.datetime.now() > timeout:
464 for hostname in tocheck:
465 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
467 # otherwise, sleep for a while
469 # only useful in empty plcs
472 def nodes_booted(self):
473 return self.do_nodes_booted(minutes=0)
476 def do_nodes_ssh(self,minutes):
478 timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
479 tocheck = self.all_hostnames()
480 # self.scan_publicKeys(tocheck)
481 utils.header("checking Connectivity on nodes %r"%tocheck)
483 for hostname in tocheck:
484 # try to ssh in nodes
485 node_test_ssh = TestSsh (hostname,key="/etc/planetlab/root_ssh_key.rsa")
486 access=self.run_in_guest(node_test_ssh.actual_command("date"))
488 utils.header('The node %s is sshable -->'%hostname)
490 tocheck.remove(hostname)
492 # we will have tried real nodes once, in case they're up - but if not, just skip
493 (site_spec,node_spec)=self.locate_hostname(hostname)
494 if TestNode.is_real_model(node_spec['node_fields']['model']):
495 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
496 tocheck.remove(hostname)
499 if datetime.datetime.now() > timeout:
500 for hostname in tocheck:
501 utils.header("FAILURE to ssh into %s"%hostname)
503 # otherwise, sleep for a while
505 # only useful in empty plcs
509 return self.do_nodes_ssh(minutes=2)
512 def init_node (self): pass
514 def bootcd (self): pass
516 def configure_qemu (self): pass
518 def do_check_initscripts(self):
520 for slice_spec in self.plc_spec['slices']:
521 if not slice_spec.has_key('initscriptname'):
523 initscript=slice_spec['initscriptname']
524 for nodename in slice_spec['nodenames']:
525 (site,node) = self.locate_node (nodename)
526 # xxx - passing the wrong site - probably harmless
527 test_site = TestSite (self,site)
528 test_slice = TestSlice (self,test_site,slice_spec)
529 test_node = TestNode (self,test_site,node)
530 test_sliver = TestSliver (self, test_node, test_slice)
531 if not test_sliver.check_initscript(initscript):
535 def check_initscripts(self):
536 return self.do_check_initscripts()
538 def initscripts (self):
539 for initscript in self.plc_spec['initscripts']:
540 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
541 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
545 return self.do_slices()
547 def clean_slices (self):
548 return self.do_slices("delete")
550 def do_slices (self, action="add"):
551 for slice in self.plc_spec['slices']:
552 site_spec = self.locate_site (slice['sitename'])
553 test_site = TestSite(self,site_spec)
554 test_slice=TestSlice(self,test_site,slice)
556 utils.header("Deleting slices in site %s"%test_site.name())
557 test_slice.delete_slice()
559 utils.pprint("Creating slice",slice)
560 test_slice.create_slice()
561 utils.header('Created Slice %s'%slice['slice_fields']['name'])
564 def check_slices(self):
565 for slice_spec in self.plc_spec['slices']:
566 site_spec = self.locate_site (slice_spec['sitename'])
567 test_site = TestSite(self,site_spec)
568 test_slice=TestSlice(self,test_site,slice_spec)
569 status=test_slice.do_check_slice(self.options)
574 def start_nodes (self):
575 utils.header("Starting nodes")
576 for site_spec in self.plc_spec['sites']:
577 TestSite(self,site_spec).start_nodes (self.options)
581 def locate_first_sliver (self):
582 slice_spec = self.plc_spec['slices'][0]
583 slicename = slice_spec['slice_fields']['name']
584 nodename = slice_spec['nodenames'][0]
585 return self.locate_sliver_obj(nodename,slicename)
587 def locate_sliver_obj (self,nodename,slicename):
588 (site,node) = self.locate_node(nodename)
589 slice = self.locate_slice (slicename)
591 test_site = TestSite (self, site)
592 test_node = TestNode (self, test_site,node)
593 # xxx the slice site is assumed to be the node site - mhh - probably harmless
594 test_slice = TestSlice (self, test_site, slice)
595 return TestSliver (self, test_node, test_slice)
597 def check_tcp (self):
598 specs = self.plc_spec['tcp_test']
603 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
604 if not s_test_sliver.run_tcp_server(port,timeout=10):
608 # idem for the client side
609 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
610 if not c_test_sliver.run_tcp_client(s_test_node.name(),port):
615 def gather_logs (self):
616 # (1) get the plc's /var/log and store it locally in logs/<plcname>-var-log/*
617 # (2) get all the nodes qemu log and store it as logs/<node>-qemu.log
618 # (3) get the nodes /var/log and store is as logs/<node>-var-log/*
619 # (4) as far as possible get the slice's /var/log as logs/<slice>-<node>-var-log/*
621 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
622 self.gather_var_logs ()
624 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
625 for site_spec in self.plc_spec['sites']:
626 test_site = TestSite (self,site_spec)
627 for node_spec in site_spec['nodes']:
628 test_node=TestNode(self,test_site,node_spec)
629 test_node.gather_qemu_logs()
631 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
632 self.gather_nodes_var_logs()
634 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
635 self.gather_first_sliver_logs()
638 def gather_first_sliver_logs(self):
640 test_sliver = self.locate_first_sliver()
641 remote = test_sliver.tar_var_logs()
642 utils.system("mkdir -p logs/%s-var-log"%test_sliver.name())
643 command = remote + " | tar -C logs/%s-var-log -xf -"%test_sliver.name()
644 utils.system(command)
646 print 'Cannot locate first sliver - giving up',e
649 def gather_var_logs (self):
650 to_plc = self.actual_command_in_guest("tar -cf - /var/log")
651 command = to_plc + "| tar -C logs/%s-var-log -xf -"%self.name()
652 utils.system("mkdir -p logs/%s-var-log"%self.name())
653 utils.system(command)
655 def gather_nodes_var_logs (self):
656 for site_spec in self.plc_spec['sites']:
657 test_site = TestSite (self,site_spec)
658 for node_spec in site_spec['nodes']:
659 test_node=TestNode(self,test_site,node_spec)
660 test_ssh = TestSsh (test_node.name(),key="/etc/planetlab/root_ssh_key.rsa")
661 to_plc = self.actual_command_in_guest ( test_ssh.actual_command("tar -C /var/log -cf - ."))
662 command = to_plc + "| tar -C logs/%s-var-log -xf -"%test_node.name()
663 utils.system("mkdir -p logs/%s-var-log"%test_node.name())
664 utils.system(command)
667 # returns the filename to use for sql dump/restore, using options.dbname if set
668 def dbfile (self, database):
669 # uses options.dbname if it is found
671 name=self.options.dbname
672 if not isinstance(name,StringTypes):
675 t=datetime.datetime.now()
678 return "/root/%s-%s.sql"%(database,name)
681 dump=self.dbfile("planetab4")
682 self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
683 utils.header('Dumped planetlab4 database in %s'%dump)
686 def db_restore(self):
687 dump=self.dbfile("planetab4")
689 self.run_in_guest('service httpd stop')
690 # xxx - need another wrapper
691 self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
692 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
693 self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
694 ##starting httpd service
695 self.run_in_guest('service httpd start')
697 utils.header('Database restored from ' + dump)
700 def standby_1(): pass
702 def standby_2(): pass
704 def standby_3(): pass
706 def standby_4(): pass
708 def standby_5(): pass
710 def standby_6(): pass
712 def standby_7(): pass
714 def standby_8(): pass
716 def standby_9(): pass
718 def standby_10(): pass
720 def standby_11(): pass
722 def standby_12(): pass
724 def standby_13(): pass
726 def standby_14(): pass
728 def standby_15(): pass
730 def standby_16(): pass
732 def standby_17(): pass
734 def standby_18(): pass
736 def standby_19(): pass
738 def standby_20(): pass