8 from types import StringTypes
11 from TestSite import TestSite
12 from TestNode import TestNode
13 from TestUser import TestUser
14 from TestKey import TestKey
15 from TestSlice import TestSlice
16 from TestSliver import TestSliver
17 from TestBox import TestBox
18 from TestSsh import TestSsh
19 from TestApiserver import TestApiserver
21 # step methods must take (self) and return a boolean (options is a member of the class)
23 def standby(minutes,dry_run):
24 utils.header('Entering StandBy for %d mn'%minutes)
28 time.sleep(60*minutes)
31 def standby_generic (func):
33 minutes=int(func.__name__.split("_")[1])
34 return standby(minutes,self.options.dry_run)
37 def node_mapper (method):
40 node_method = TestNode.__dict__[method.__name__]
41 for site_spec in self.plc_spec['sites']:
42 test_site = TestSite (self,site_spec)
43 for node_spec in site_spec['nodes']:
44 test_node = TestNode (self,test_site,node_spec)
45 if not node_method(test_node): overall=False
49 def slice_mapper_options (method):
52 slice_method = TestSlice.__dict__[method.__name__]
53 for slice_spec in self.plc_spec['slices']:
54 site_spec = self.locate_site (slice_spec['sitename'])
55 test_site = TestSite(self,site_spec)
56 test_slice=TestSlice(self,test_site,slice_spec)
57 if not slice_method(test_slice,self.options): overall=False
63 def __init__ (self,plc_spec,options):
64 self.plc_spec=plc_spec
66 self.test_ssh=TestSsh(self.plc_spec['hostname'],self.options.buildname)
68 self.vserverip=plc_spec['vserverip']
69 self.vservername=plc_spec['vservername']
70 self.url="https://%s:443/PLCAPI/"%plc_spec['vserverip']
74 self.url="https://%s:443/PLCAPI/"%plc_spec['hostname']
75 # utils.header('Using API url %s'%self.url)
76 self.apiserver=TestApiserver(self.url,options.dry_run)
79 name=self.plc_spec['name']
81 return name+".vserver.%s"%self.vservername
86 return self.plc_spec['hostname']
89 return self.test_ssh.is_local()
91 # define the API methods on this object through xmlrpc
92 # would help, but not strictly necessary
96 def actual_command_in_guest (self,command):
97 return self.test_ssh.actual_command(self.host_to_guest(command))
99 def run_in_guest (self,command):
100 return utils.system(self.actual_command_in_guest(command))
102 def run_in_host (self,command):
103 return self.test_ssh.run_in_buildname(command)
105 #command gets run in the chroot/vserver
106 def host_to_guest(self,command):
108 return "vserver %s exec %s"%(self.vservername,command)
110 return "chroot /plc/root %s"%TestSsh.backslash_shell_specials(command)
112 # copy a file to the myplc root image - pass in_data=True if the file must go in /plc/data
113 def copy_in_guest (self, localfile, remotefile, in_data=False):
115 chroot_dest="/plc/data"
117 chroot_dest="/plc/root"
120 utils.system("cp %s %s/%s"%(localfile,chroot_dest,remotefile))
122 utils.system("cp %s /vservers/%s/%s"%(localfile,self.vservername,remotefile))
125 utils.system("scp %s %s:%s/%s"%(localfile,self.hostname(),chroot_dest,remotefile))
127 utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.hostname(),self.vservername,remotefile))
131 def run_in_guest_piped (self,local,remote):
132 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote)))
134 def auth_root (self):
135 return {'Username':self.plc_spec['PLC_ROOT_USER'],
136 'AuthMethod':'password',
137 'AuthString':self.plc_spec['PLC_ROOT_PASSWORD'],
138 'Role' : self.plc_spec['role']
140 def locate_site (self,sitename):
141 for site in self.plc_spec['sites']:
142 if site['site_fields']['name'] == sitename:
144 if site['site_fields']['login_base'] == sitename:
146 raise Exception,"Cannot locate site %s"%sitename
148 def locate_node (self,nodename):
149 for site in self.plc_spec['sites']:
150 for node in site['nodes']:
151 if node['name'] == nodename:
153 raise Exception,"Cannot locate node %s"%nodename
155 def locate_hostname (self,hostname):
156 for site in self.plc_spec['sites']:
157 for node in site['nodes']:
158 if node['node_fields']['hostname'] == hostname:
160 raise Exception,"Cannot locate hostname %s"%hostname
162 def locate_key (self,keyname):
163 for key in self.plc_spec['keys']:
164 if key['name'] == keyname:
166 raise Exception,"Cannot locate key %s"%keyname
168 def locate_slice (self, slicename):
169 for slice in self.plc_spec['slices']:
170 if slice['slice_fields']['name'] == slicename:
172 raise Exception,"Cannot locate slice %s"%slicename
174 # all different hostboxes used in this plc
175 def gather_hostBoxes(self):
176 # maps on sites and nodes, return [ (host_box,test_node) ]
178 for site_spec in self.plc_spec['sites']:
179 test_site = TestSite (self,site_spec)
180 for node_spec in site_spec['nodes']:
181 test_node = TestNode (self, test_site, node_spec)
182 if not test_node.is_real():
183 tuples.append( (test_node.host_box(),test_node) )
184 # transform into a dict { 'host_box' -> [ hostnames .. ] }
186 for (box,node) in tuples:
187 if not result.has_key(box):
190 result[box].append(node)
193 # a step for checking this stuff
194 def show_boxes (self):
195 for (box,nodes) in self.gather_hostBoxes().iteritems():
196 print box,":"," + ".join( [ node.name() for node in nodes ] )
199 # make this a valid step
200 def kill_all_qemus(self):
201 for (box,nodes) in self.gather_hostBoxes().iteritems():
202 # this is the brute force version, kill all qemus on that host box
203 TestBox(box,self.options.buildname).kill_all_qemus()
206 # make this a valid step
207 def list_all_qemus(self):
208 for (box,nodes) in self.gather_hostBoxes().iteritems():
209 # this is the brute force version, kill all qemus on that host box
210 TestBox(box,self.options.buildname).list_all_qemus()
213 # kill only the right qemus
214 def list_qemus(self):
215 for (box,nodes) in self.gather_hostBoxes().iteritems():
216 # the fine-grain version
221 # kill only the right qemus
222 def kill_qemus(self):
223 for (box,nodes) in self.gather_hostBoxes().iteritems():
224 # the fine-grain version
229 #################### step methods
232 def uninstall_chroot(self):
233 self.run_in_host('service plc safestop')
234 #####detecting the last myplc version installed and remove it
235 self.run_in_host('rpm -e myplc')
236 ##### Clean up the /plc directory
237 self.run_in_host('rm -rf /plc/data')
238 ##### stop any running vservers
239 self.run_in_host('for vserver in $(ls -d /vservers/* | sed -e s,/vservers/,,) ; do case $vserver in vtest*) echo Shutting down vserver $vserver ; vserver $vserver stop ;; esac ; done')
242 def uninstall_vserver(self):
243 self.run_in_host("vserver --silent %s delete"%self.vservername)
247 # if there's a chroot-based myplc running, and then a native-based myplc is being deployed
248 # it sounds safer to have the former uninstalled too
249 # now the vserver method cannot be invoked for chroot instances as vservername is required
251 self.uninstall_vserver()
252 self.uninstall_chroot()
254 self.uninstall_chroot()
258 def install_chroot(self):
262 def install_vserver(self):
263 # we need build dir for vtest-init-vserver
265 # a full path for the local calls
266 build_dir=os.path(sys.argv[0])+"/build"
268 # use a standard name - will be relative to HOME
269 build_dir="options.buildname"
270 # run checkout in any case - would do an update if already exists
271 build_checkout = "svn checkout %s %s"%(self.options.build_url,build_dir)
272 if self.run_in_host(build_checkout) != 0:
274 # the repo url is taken from myplc-url
275 # with the last two steps (i386/myplc...) removed
276 repo_url = self.options.myplc_url
277 for level in [ 'rpmname','arch' ]:
278 repo_url = os.path.dirname(repo_url)
279 create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\
280 (build_dir,self.vservername,repo_url,self.vserverip)
281 return self.run_in_host(create_vserver) == 0
285 return self.install_vserver()
287 return self.install_chroot()
289 ### install_rpm - make this an optional step
291 url = self.options.myplc_url
292 rpm = os.path.basename(url)
293 cache_fetch="pwd;if [ -f %(rpm)s ] ; then echo Using cached rpm %(rpm)s ; else echo Fetching %(url)s ; curl -O %(url)s; fi"%locals()
294 return self.run_in_host(cache_fetch)==0
296 def install_rpm_chroot(self):
297 url = self.options.myplc_url
298 rpm = os.path.basename(url)
299 if not self.cache_rpm():
301 utils.header('Installing the : %s'%rpm)
302 return self.run_in_host('rpm -Uvh '+rpm)==0 and self.run_in_host('service plc mount')==0
304 def install_rpm_vserver(self):
305 return self.run_in_guest("yum -y install myplc-native")==0
307 def install_rpm(self):
309 return self.install_rpm_vserver()
311 return self.install_rpm_chroot()
315 tmpname='%s.plc-config-tty'%(self.name())
316 fileconf=open(tmpname,'w')
317 for var in [ 'PLC_NAME',
321 'PLC_MAIL_SUPPORT_ADDRESS',
328 fileconf.write ('e %s\n%s\n'%(var,self.plc_spec[var]))
329 fileconf.write('w\n')
330 fileconf.write('q\n')
332 utils.system('cat %s'%tmpname)
333 self.run_in_guest_piped('cat %s'%tmpname,'plc-config-tty')
334 utils.system('rm %s'%tmpname)
337 # the chroot install is slightly different to this respect
340 self.run_in_guest('service plc start')
342 self.run_in_host('service plc start')
347 self.run_in_guest('service plc stop')
349 self.run_in_host('service plc stop')
352 # could use a TestKey class
353 def store_keys(self):
354 for key_spec in self.plc_spec['keys']:
355 TestKey(self,key_spec).store_key()
358 def clean_keys(self):
359 utils.system("rm -rf %s/keys/"%os.path(sys.argv[0]))
362 return self.do_sites()
364 def clean_sites (self):
365 return self.do_sites(action="delete")
367 def do_sites (self,action="add"):
368 for site_spec in self.plc_spec['sites']:
369 test_site = TestSite (self,site_spec)
370 if (action != "add"):
371 utils.header("Deleting site %s in %s"%(test_site.name(),self.name()))
372 test_site.delete_site()
373 # deleted with the site
374 #test_site.delete_users()
377 utils.header("Creating site %s & users in %s"%(test_site.name(),self.name()))
378 test_site.create_site()
379 test_site.create_users()
383 return self.do_nodes()
384 def clean_nodes (self):
385 return self.do_nodes(action="delete")
387 def do_nodes (self,action="add"):
388 for site_spec in self.plc_spec['sites']:
389 test_site = TestSite (self,site_spec)
391 utils.header("Deleting nodes in site %s"%test_site.name())
392 for node_spec in site_spec['nodes']:
393 test_node=TestNode(self,test_site,node_spec)
394 utils.header("Deleting %s"%test_node.name())
395 test_node.delete_node()
397 utils.header("Creating nodes for site %s in %s"%(test_site.name(),self.name()))
398 for node_spec in site_spec['nodes']:
399 utils.pprint('Creating node %s'%node_spec,node_spec)
400 test_node = TestNode (self,test_site,node_spec)
401 test_node.create_node ()
404 # create nodegroups if needed, and populate
405 # no need for a clean_nodegroups if we are careful enough
406 def nodegroups (self):
407 # 1st pass to scan contents
409 for site_spec in self.plc_spec['sites']:
410 test_site = TestSite (self,site_spec)
411 for node_spec in site_spec['nodes']:
412 test_node=TestNode (self,test_site,node_spec)
413 if node_spec.has_key('nodegroups'):
414 nodegroupnames=node_spec['nodegroups']
415 if isinstance(nodegroupnames,StringTypes):
416 nodegroupnames = [ nodegroupnames ]
417 for nodegroupname in nodegroupnames:
418 if not groups_dict.has_key(nodegroupname):
419 groups_dict[nodegroupname]=[]
420 groups_dict[nodegroupname].append(test_node.name())
421 auth=self.auth_root()
422 for (nodegroupname,group_nodes) in groups_dict.iteritems():
424 self.apiserver.GetNodeGroups(auth,{'name':nodegroupname})[0]
426 self.apiserver.AddNodeGroup(auth,{'name':nodegroupname})
427 for node in group_nodes:
428 self.apiserver.AddNodeToNodeGroup(auth,node,nodegroupname)
431 def all_hostnames (self) :
433 for site_spec in self.plc_spec['sites']:
434 hostnames += [ node_spec['node_fields']['hostname'] \
435 for node_spec in site_spec['nodes'] ]
438 # gracetime : during the first <gracetime> minutes nothing gets printed
439 def do_nodes_booted (self, minutes, gracetime=2):
440 if self.options.dry_run:
444 timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
445 graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime)
446 # the nodes that haven't checked yet - start with a full list and shrink over time
447 tocheck = self.all_hostnames()
448 utils.header("checking nodes %r"%tocheck)
449 # create a dict hostname -> status
450 status = dict ( [ (hostname,'undef') for hostname in tocheck ] )
453 tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] )
455 for array in tocheck_status:
456 hostname=array['hostname']
457 boot_state=array['boot_state']
458 if boot_state == 'boot':
459 utils.header ("%s has reached the 'boot' state"%hostname)
461 # if it's a real node, never mind
462 (site_spec,node_spec)=self.locate_hostname(hostname)
463 if TestNode.is_real_model(node_spec['node_fields']['model']):
464 utils.header("WARNING - Real node %s in %s - ignored"%(hostname,boot_state))
467 if datetime.datetime.now() > graceout:
468 utils.header ("%s still in '%s' state"%(hostname,boot_state))
469 graceout=datetime.datetime.now()+datetime.timedelta(1)
470 status[hostname] = boot_state
472 tocheck = [ hostname for (hostname,boot_state) in status.iteritems() if boot_state != 'boot' ]
475 if datetime.datetime.now() > timeout:
476 for hostname in tocheck:
477 utils.header("FAILURE due to %s in '%s' state"%(hostname,status[hostname]))
479 # otherwise, sleep for a while
481 # only useful in empty plcs
484 def nodes_booted(self):
485 return self.do_nodes_booted(minutes=0)
488 def do_nodes_ssh(self,minutes):
490 timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes)
491 tocheck = self.all_hostnames()
492 # self.scan_publicKeys(tocheck)
493 utils.header("checking Connectivity on nodes %r"%tocheck)
495 for hostname in tocheck:
496 # try to ssh in nodes
497 node_test_ssh = TestSsh (hostname,key="/etc/planetlab/root_ssh_key.rsa")
498 access=self.run_in_guest(node_test_ssh.actual_command("date"))
500 utils.header('The node %s is sshable -->'%hostname)
502 tocheck.remove(hostname)
504 # we will have tried real nodes once, in case they're up - but if not, just skip
505 (site_spec,node_spec)=self.locate_hostname(hostname)
506 if TestNode.is_real_model(node_spec['node_fields']['model']):
507 utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname)
508 tocheck.remove(hostname)
511 if datetime.datetime.now() > timeout:
512 for hostname in tocheck:
513 utils.header("FAILURE to ssh into %s"%hostname)
515 # otherwise, sleep for a while
517 # only useful in empty plcs
521 return self.do_nodes_ssh(minutes=2)
524 def init_node (self): pass
526 def bootcd (self): pass
528 def configure_qemu (self): pass
530 def do_check_initscripts(self):
532 for slice_spec in self.plc_spec['slices']:
533 if not slice_spec.has_key('initscriptname'):
535 initscript=slice_spec['initscriptname']
536 for nodename in slice_spec['nodenames']:
537 (site,node) = self.locate_node (nodename)
538 # xxx - passing the wrong site - probably harmless
539 test_site = TestSite (self,site)
540 test_slice = TestSlice (self,test_site,slice_spec)
541 test_node = TestNode (self,test_site,node)
542 test_sliver = TestSliver (self, test_node, test_slice)
543 if not test_sliver.check_initscript(initscript):
547 def check_initscripts(self):
548 return self.do_check_initscripts()
550 def initscripts (self):
551 for initscript in self.plc_spec['initscripts']:
552 utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript)
553 self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields'])
557 return self.do_slices()
559 def clean_slices (self):
560 return self.do_slices("delete")
562 def do_slices (self, action="add"):
563 for slice in self.plc_spec['slices']:
564 site_spec = self.locate_site (slice['sitename'])
565 test_site = TestSite(self,site_spec)
566 test_slice=TestSlice(self,test_site,slice)
568 utils.header("Deleting slices in site %s"%test_site.name())
569 test_slice.delete_slice()
571 utils.pprint("Creating slice",slice)
572 test_slice.create_slice()
573 utils.header('Created Slice %s'%slice['slice_fields']['name'])
576 @slice_mapper_options
577 def check_slice(self): pass
580 def clear_known_hosts (self): pass
582 def start_nodes (self):
583 utils.header("Starting nodes")
584 for site_spec in self.plc_spec['sites']:
585 TestSite(self,site_spec).start_nodes (self.options)
588 def locate_first_sliver (self):
589 slice_spec = self.plc_spec['slices'][0]
590 slicename = slice_spec['slice_fields']['name']
591 nodename = slice_spec['nodenames'][0]
592 return self.locate_sliver_obj(nodename,slicename)
594 def locate_sliver_obj (self,nodename,slicename):
595 (site,node) = self.locate_node(nodename)
596 slice = self.locate_slice (slicename)
598 test_site = TestSite (self, site)
599 test_node = TestNode (self, test_site,node)
600 # xxx the slice site is assumed to be the node site - mhh - probably harmless
601 test_slice = TestSlice (self, test_site, slice)
602 return TestSliver (self, test_node, test_slice)
604 def check_tcp (self):
605 specs = self.plc_spec['tcp_test']
610 s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice'])
611 if not s_test_sliver.run_tcp_server(port,timeout=10):
615 # idem for the client side
616 c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice'])
617 if not c_test_sliver.run_tcp_client(s_test_sliver.test_node.name(),port):
622 def gather_logs (self):
623 # (1) get the plc's /var/log and store it locally in logs/<plcname>-var-log/*
624 # (2) get all the nodes qemu log and store it as logs/<node>-qemu.log
625 # (3) get the nodes /var/log and store is as logs/<node>-var-log/*
626 # (4) as far as possible get the slice's /var/log as logs/<slice>-<node>-var-log/*
628 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
629 self.gather_var_logs ()
631 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
632 for site_spec in self.plc_spec['sites']:
633 test_site = TestSite (self,site_spec)
634 for node_spec in site_spec['nodes']:
635 test_node=TestNode(self,test_site,node_spec)
636 test_node.gather_qemu_logs()
638 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
639 self.gather_nodes_var_logs()
641 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
642 self.gather_first_sliver_logs()
645 def gather_first_sliver_logs(self):
647 test_sliver = self.locate_first_sliver()
648 remote = test_sliver.tar_var_logs()
649 utils.system("mkdir -p logs/%s-var-log"%test_sliver.name())
650 command = remote + " | tar -C logs/%s-var-log -xf -"%test_sliver.name()
651 utils.system(command)
653 print 'Cannot locate first sliver - giving up',e
656 def gather_var_logs (self):
657 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
658 command = to_plc + "| tar -C logs/%s-var-log -xf -"%self.name()
659 utils.system("mkdir -p logs/%s-var-log"%self.name())
660 utils.system(command)
662 def gather_nodes_var_logs (self):
663 for site_spec in self.plc_spec['sites']:
664 test_site = TestSite (self,site_spec)
665 for node_spec in site_spec['nodes']:
666 test_node=TestNode(self,test_site,node_spec)
667 test_ssh = TestSsh (test_node.name(),key="/etc/planetlab/root_ssh_key.rsa")
668 to_plc = self.actual_command_in_guest ( test_ssh.actual_command("tar -C /var/log -cf - ."))
669 command = to_plc + "| tar -C logs/%s-var-log -xf -"%test_node.name()
670 utils.system("mkdir -p logs/%s-var-log"%test_node.name())
671 utils.system(command)
674 # returns the filename to use for sql dump/restore, using options.dbname if set
675 def dbfile (self, database):
676 # uses options.dbname if it is found
678 name=self.options.dbname
679 if not isinstance(name,StringTypes):
682 t=datetime.datetime.now()
685 return "/root/%s-%s.sql"%(database,name)
688 dump=self.dbfile("planetab4")
689 self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump)
690 utils.header('Dumped planetlab4 database in %s'%dump)
693 def db_restore(self):
694 dump=self.dbfile("planetab4")
696 self.run_in_guest('service httpd stop')
697 # xxx - need another wrapper
698 self.run_in_guest_piped('echo drop database planetlab4','psql --user=pgsqluser template1')
699 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab4')
700 self.run_in_guest('psql -U pgsqluser planetlab4 -f '+dump)
701 ##starting httpd service
702 self.run_in_guest('service httpd start')
704 utils.header('Database restored from ' + dump)
707 def standby_1(): pass
709 def standby_2(): pass
711 def standby_3(): pass
713 def standby_4(): pass
715 def standby_5(): pass
717 def standby_6(): pass
719 def standby_7(): pass
721 def standby_8(): pass
723 def standby_9(): pass
725 def standby_10(): pass
727 def standby_11(): pass
729 def standby_12(): pass
731 def standby_13(): pass
733 def standby_14(): pass
735 def standby_15(): pass
737 def standby_16(): pass
739 def standby_17(): pass
741 def standby_18(): pass
743 def standby_19(): pass
745 def standby_20(): pass