1 # Thierry Parmentelat <thierry.parmentelat@inria.fr>
2 # Copyright (C) 2010 INRIA
9 from datetime import datetime, timedelta
10 from types import StringTypes
13 from Completer import Completer, CompleterTask
14 from TestSite import TestSite
15 from TestNode import TestNode, CompleterTaskNodeSsh
16 from TestUser import TestUser
17 from TestKey import TestKey
18 from TestSlice import TestSlice
19 from TestSliver import TestSliver
20 from TestBoxQemu import TestBoxQemu
21 from TestSsh import TestSsh
22 from TestApiserver import TestApiserver
23 from TestAuthSfa import TestAuthSfa
24 from PlcapiUrlScanner import PlcapiUrlScanner
26 has_sfa_cache_filename="sfa-cache"
28 # step methods must take (self) and return a boolean (options is a member of the class)
30 def standby(minutes, dry_run):
31 utils.header('Entering StandBy for %d mn'%minutes)
35 time.sleep(60*minutes)
38 def standby_generic(func):
40 minutes = int(func.__name__.split("_")[1])
41 return standby(minutes, self.options.dry_run)
44 def node_mapper(method):
45 def map_on_nodes(self, *args, **kwds):
47 node_method = TestNode.__dict__[method.__name__]
48 for test_node in self.all_nodes():
49 if not node_method(test_node, *args, **kwds):
52 # maintain __name__ for ignore_result
53 map_on_nodes.__name__ = method.__name__
54 # restore the doc text
55 map_on_nodes.__doc__ = TestNode.__dict__[method.__name__].__doc__
58 def slice_mapper(method):
59 def map_on_slices(self):
61 slice_method = TestSlice.__dict__[method.__name__]
62 for slice_spec in self.plc_spec['slices']:
63 site_spec = self.locate_site (slice_spec['sitename'])
64 test_site = TestSite(self,site_spec)
65 test_slice = TestSlice(self,test_site,slice_spec)
66 if not slice_method(test_slice, self.options):
69 # maintain __name__ for ignore_result
70 map_on_slices.__name__ = method.__name__
71 # restore the doc text
72 map_on_slices.__doc__ = TestSlice.__dict__[method.__name__].__doc__
75 # run a step but return True so that we can go on
76 def ignore_result(method):
78 # ssh_slice_ignore->ssh_slice
79 ref_name = method.__name__.replace('_ignore', '').replace('force_', '')
80 ref_method = TestPlc.__dict__[ref_name]
81 result = ref_method(self)
82 print "Actual (but ignored) result for %(ref_name)s is %(result)s" % locals()
83 return Ignored(result)
84 name = method.__name__.replace('_ignore', '').replace('force_', '')
85 ignoring.__name__ = name
86 ignoring.__doc__ = "ignored version of " + name
89 # a variant that expects the TestSlice method to return a list of CompleterTasks that
90 # are then merged into a single Completer run to avoid wating for all the slices
91 # esp. useful when a test fails of course
92 # because we need to pass arguments we use a class instead..
93 class slice_mapper__tasks(object):
94 # could not get this to work with named arguments
95 def __init__(self, timeout_minutes, silent_minutes, period_seconds):
96 self.timeout = timedelta(minutes = timeout_minutes)
97 self.silent = timedelta(minutes = silent_minutes)
98 self.period = timedelta(seconds = period_seconds)
99 def __call__(self, method):
101 # compute augmented method name
102 method_name = method.__name__ + "__tasks"
103 # locate in TestSlice
104 slice_method = TestSlice.__dict__[ method_name ]
107 for slice_spec in self.plc_spec['slices']:
108 site_spec = self.locate_site (slice_spec['sitename'])
109 test_site = TestSite(self, site_spec)
110 test_slice = TestSlice(self, test_site, slice_spec)
111 tasks += slice_method (test_slice, self.options)
112 return Completer (tasks, message=method.__name__).\
113 run(decorator_self.timeout, decorator_self.silent, decorator_self.period)
114 # restore the doc text from the TestSlice method even if a bit odd
115 wrappee.__name__ = method.__name__
116 wrappee.__doc__ = slice_method.__doc__
119 def auth_sfa_mapper(method):
122 auth_method = TestAuthSfa.__dict__[method.__name__]
123 for auth_spec in self.plc_spec['sfa']['auth_sfa_specs']:
124 test_auth = TestAuthSfa(self, auth_spec)
125 if not auth_method(test_auth, self.options):
128 # restore the doc text
129 actual.__doc__ = TestAuthSfa.__dict__[method.__name__].__doc__
133 def __init__(self, result):
143 'plcvm_delete','plcvm_timestamp','plcvm_create', SEP,
144 'plc_install', 'plc_configure', 'plc_start', SEP,
145 'keys_fetch', 'keys_store', 'keys_clear_known_hosts', SEP,
146 'plcapi_urls','speed_up_slices', SEP,
147 'initscripts', 'sites', 'nodes', 'slices', 'nodegroups', 'leases', SEP,
148 # slices created under plcsh interactively seem to be fine but these ones don't have the tags
149 # keep this our of the way for now
150 'check_vsys_defaults_ignore', SEP,
151 # run this first off so it's easier to re-run on another qemu box
152 'qemu_kill_mine', SEP,
153 'nodestate_reinstall', 'qemu_local_init','bootcd', 'qemu_local_config', SEP,
154 'qemu_clean_mine', 'qemu_export', 'qemu_start', 'qemu_timestamp', SEP,
155 'sfa_install_all', 'sfa_configure', 'cross_sfa_configure', 'sfa_start', 'sfa_import', SEPSFA,
156 'sfi_configure@1', 'sfa_register_site@1','sfa_register_pi@1', SEPSFA,
157 'sfa_register_user@1', 'sfa_update_user@1', 'sfa_register_slice@1', 'sfa_renew_slice@1', SEPSFA,
158 'sfa_remove_user_from_slice@1','sfi_show_slice_researchers@1',
159 'sfa_insert_user_in_slice@1','sfi_show_slice_researchers@1', SEPSFA,
160 'sfa_discover@1', 'sfa_rspec@1', 'sfa_allocate@1', 'sfa_provision@1', SEPSFA,
161 'sfa_check_slice_plc@1', 'sfa_update_slice@1', SEPSFA,
162 'sfi_list@1', 'sfi_show_site@1', 'sfa_utest@1', SEPSFA,
163 # we used to run plcsh_stress_test, and then ssh_node_debug and ssh_node_boot
164 # but as the stress test might take a while, we sometimes missed the debug mode..
165 'probe_kvm_iptables',
166 'ping_node', 'ssh_node_debug', 'plcsh_stress_test@1', SEP,
167 'ssh_node_boot', 'node_bmlogs', 'ssh_slice', 'ssh_slice_basics', 'check_initscripts', SEP,
168 'ssh_slice_sfa@1', SEPSFA,
169 'sfa_rspec_empty@1', 'sfa_allocate_empty@1', 'sfa_provision_empty@1','sfa_check_slice_plc_empty@1', SEPSFA,
170 'sfa_delete_slice@1', 'sfa_delete_user@1', SEPSFA,
171 'cross_check_tcp@1', 'check_system_slice', SEP,
172 # for inspecting the slice while it runs the first time
174 # check slices are turned off properly
175 'empty_slices', 'ssh_slice_off', 'slice_fs_deleted_ignore', SEP,
176 # check they are properly re-created with the same name
177 'fill_slices', 'ssh_slice_again', SEP,
178 'gather_logs_force', SEP,
181 'export', 'show_boxes', 'super_speed_up_slices', SEP,
182 'check_hooks', 'plc_stop', 'plcvm_start', 'plcvm_stop', SEP,
183 'delete_initscripts', 'delete_nodegroups','delete_all_sites', SEP,
184 'delete_sites', 'delete_nodes', 'delete_slices', 'keys_clean', SEP,
185 'delete_leases', 'list_leases', SEP,
187 'nodestate_show','nodestate_safeboot','nodestate_boot', SEP,
188 'qemu_list_all', 'qemu_list_mine', 'qemu_kill_all', SEP,
189 'sfa_install_core', 'sfa_install_sfatables', 'sfa_install_plc', 'sfa_install_client', SEPSFA,
190 'sfa_plcclean', 'sfa_dbclean', 'sfa_stop','sfa_uninstall', 'sfi_clean', SEPSFA,
191 'sfa_get_expires', SEPSFA,
192 'plc_db_dump' , 'plc_db_restore', SEP,
193 'check_netflow','check_drl', SEP,
194 'debug_nodemanager', 'slice_fs_present', SEP,
195 'standby_1_through_20','yes','no',SEP,
199 def printable_steps(list):
200 single_line = " ".join(list) + " "
201 return single_line.replace(" "+SEP+" ", " \\\n").replace(" "+SEPSFA+" ", " \\\n")
203 def valid_step(step):
204 return step != SEP and step != SEPSFA
206 # turn off the sfa-related steps when build has skipped SFA
207 # this was originally for centos5 but is still valid
208 # for up to f12 as recent SFAs with sqlalchemy won't build before f14
210 def _has_sfa_cached(rpms_url):
211 if os.path.isfile(has_sfa_cache_filename):
212 cached = file(has_sfa_cache_filename).read() == "yes"
213 utils.header("build provides SFA (cached):%s" % cached)
215 # warning, we're now building 'sface' so let's be a bit more picky
216 # full builds are expected to return with 0 here
217 utils.header("Checking if build provides SFA package...")
218 retcod = os.system("curl --silent %s/ | grep -q sfa-"%rpms_url) == 0
219 encoded = 'yes' if retcod else 'no'
220 with open(has_sfa_cache_filename,'w')as out:
225 def check_whether_build_has_sfa(rpms_url):
226 has_sfa = TestPlc._has_sfa_cached(rpms_url)
228 utils.header("build does provide SFA")
230 # move all steps containing 'sfa' from default_steps to other_steps
231 utils.header("SFA package not found - removing steps with sfa or sfi")
232 sfa_steps = [ step for step in TestPlc.default_steps
233 if step.find('sfa') >= 0 or step.find("sfi") >= 0 ]
234 TestPlc.other_steps += sfa_steps
235 for step in sfa_steps:
236 TestPlc.default_steps.remove(step)
238 def __init__(self, plc_spec, options):
239 self.plc_spec = plc_spec
240 self.options = options
241 self.test_ssh = TestSsh(self.plc_spec['host_box'], self.options.buildname)
242 self.vserverip = plc_spec['vserverip']
243 self.vservername = plc_spec['vservername']
244 self.url = "https://%s:443/PLCAPI/" % plc_spec['vserverip']
245 self.apiserver = TestApiserver(self.url, options.dry_run)
246 (self.ssh_node_boot_timeout, self.ssh_node_boot_silent) = plc_spec['ssh_node_boot_timers']
247 (self.ssh_node_debug_timeout, self.ssh_node_debug_silent) = plc_spec['ssh_node_debug_timers']
249 def has_addresses_api(self):
250 return self.apiserver.has_method('AddIpAddress')
253 name = self.plc_spec['name']
254 return "%s.%s" % (name,self.vservername)
257 return self.plc_spec['host_box']
260 return self.test_ssh.is_local()
262 # define the API methods on this object through xmlrpc
263 # would help, but not strictly necessary
267 def actual_command_in_guest(self,command, backslash=False):
268 raw1 = self.host_to_guest(command)
269 raw2 = self.test_ssh.actual_command(raw1, dry_run=self.options.dry_run, backslash=backslash)
272 def start_guest(self):
273 return utils.system(self.test_ssh.actual_command(self.start_guest_in_host(),
274 dry_run=self.options.dry_run))
276 def stop_guest(self):
277 return utils.system(self.test_ssh.actual_command(self.stop_guest_in_host(),
278 dry_run=self.options.dry_run))
280 def run_in_guest(self, command, backslash=False):
281 raw = self.actual_command_in_guest(command, backslash)
282 return utils.system(raw)
284 def run_in_host(self,command):
285 return self.test_ssh.run_in_buildname(command, dry_run=self.options.dry_run)
287 # backslashing turned out so awful at some point that I've turned off auto-backslashing
288 # see e.g. plc_start esp. the version for f14
289 #command gets run in the plc's vm
290 def host_to_guest(self, command):
291 vservername = self.vservername
292 personality = self.options.personality
293 raw = "%(personality)s virsh -c lxc:/// lxc-enter-namespace %(vservername)s" % locals()
294 # f14 still needs some extra help
295 if self.options.fcdistro == 'f14':
296 raw +=" -- /usr/bin/env PATH=/bin:/sbin:/usr/bin:/usr/sbin %(command)s" % locals()
298 raw +=" -- /usr/bin/env %(command)s" % locals()
301 # this /vservers thing is legacy...
302 def vm_root_in_host(self):
303 return "/vservers/%s/" % (self.vservername)
305 def vm_timestamp_path(self):
306 return "/vservers/%s/%s.timestamp" % (self.vservername,self.vservername)
308 #start/stop the vserver
309 def start_guest_in_host(self):
310 return "virsh -c lxc:/// start %s" % (self.vservername)
312 def stop_guest_in_host(self):
313 return "virsh -c lxc:/// destroy %s" % (self.vservername)
316 def run_in_guest_piped(self,local,remote):
317 return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote),
320 def yum_check_installed(self, rpms):
321 if isinstance(rpms, list):
323 return self.run_in_guest("rpm -q %s"%rpms) == 0
325 # does a yum install in the vs, ignore yum retcod, check with rpm
326 def yum_install(self, rpms):
327 if isinstance(rpms, list):
329 self.run_in_guest("yum -y install %s" % rpms)
330 # yum-complete-transaction comes with yum-utils, that is in vtest.pkgs
331 self.run_in_guest("yum-complete-transaction -y")
332 return self.yum_check_installed(rpms)
335 return {'Username' : self.plc_spec['settings']['PLC_ROOT_USER'],
336 'AuthMethod' : 'password',
337 'AuthString' : self.plc_spec['settings']['PLC_ROOT_PASSWORD'],
338 'Role' : self.plc_spec['role'],
341 def locate_site(self,sitename):
342 for site in self.plc_spec['sites']:
343 if site['site_fields']['name'] == sitename:
345 if site['site_fields']['login_base'] == sitename:
347 raise Exception,"Cannot locate site %s" % sitename
349 def locate_node(self, nodename):
350 for site in self.plc_spec['sites']:
351 for node in site['nodes']:
352 if node['name'] == nodename:
354 raise Exception, "Cannot locate node %s" % nodename
356 def locate_hostname(self, hostname):
357 for site in self.plc_spec['sites']:
358 for node in site['nodes']:
359 if node['node_fields']['hostname'] == hostname:
361 raise Exception,"Cannot locate hostname %s" % hostname
363 def locate_key(self, key_name):
364 for key in self.plc_spec['keys']:
365 if key['key_name'] == key_name:
367 raise Exception,"Cannot locate key %s" % key_name
369 def locate_private_key_from_key_names(self, key_names):
370 # locate the first avail. key
372 for key_name in key_names:
373 key_spec = self.locate_key(key_name)
374 test_key = TestKey(self,key_spec)
375 publickey = test_key.publicpath()
376 privatekey = test_key.privatepath()
377 if os.path.isfile(publickey) and os.path.isfile(privatekey):
384 def locate_slice(self, slicename):
385 for slice in self.plc_spec['slices']:
386 if slice['slice_fields']['name'] == slicename:
388 raise Exception,"Cannot locate slice %s" % slicename
390 def all_sliver_objs(self):
392 for slice_spec in self.plc_spec['slices']:
393 slicename = slice_spec['slice_fields']['name']
394 for nodename in slice_spec['nodenames']:
395 result.append(self.locate_sliver_obj(nodename, slicename))
398 def locate_sliver_obj(self, nodename, slicename):
399 site,node = self.locate_node(nodename)
400 slice = self.locate_slice(slicename)
402 test_site = TestSite(self, site)
403 test_node = TestNode(self, test_site, node)
404 # xxx the slice site is assumed to be the node site - mhh - probably harmless
405 test_slice = TestSlice(self, test_site, slice)
406 return TestSliver(self, test_node, test_slice)
408 def locate_first_node(self):
409 nodename = self.plc_spec['slices'][0]['nodenames'][0]
410 site,node = self.locate_node(nodename)
411 test_site = TestSite(self, site)
412 test_node = TestNode(self, test_site, node)
415 def locate_first_sliver(self):
416 slice_spec = self.plc_spec['slices'][0]
417 slicename = slice_spec['slice_fields']['name']
418 nodename = slice_spec['nodenames'][0]
419 return self.locate_sliver_obj(nodename,slicename)
421 # all different hostboxes used in this plc
422 def get_BoxNodes(self):
423 # maps on sites and nodes, return [ (host_box,test_node) ]
425 for site_spec in self.plc_spec['sites']:
426 test_site = TestSite(self,site_spec)
427 for node_spec in site_spec['nodes']:
428 test_node = TestNode(self, test_site, node_spec)
429 if not test_node.is_real():
430 tuples.append( (test_node.host_box(),test_node) )
431 # transform into a dict { 'host_box' -> [ test_node .. ] }
433 for (box,node) in tuples:
434 if not result.has_key(box):
437 result[box].append(node)
440 # a step for checking this stuff
441 def show_boxes(self):
442 'print summary of nodes location'
443 for box,nodes in self.get_BoxNodes().iteritems():
444 print box,":"," + ".join( [ node.name() for node in nodes ] )
447 # make this a valid step
448 def qemu_kill_all(self):
449 'kill all qemu instances on the qemu boxes involved by this setup'
450 # this is the brute force version, kill all qemus on that host box
451 for (box,nodes) in self.get_BoxNodes().iteritems():
452 # pass the first nodename, as we don't push template-qemu on testboxes
453 nodedir = nodes[0].nodedir()
454 TestBoxQemu(box, self.options.buildname).qemu_kill_all(nodedir)
457 # make this a valid step
458 def qemu_list_all(self):
459 'list all qemu instances on the qemu boxes involved by this setup'
460 for box,nodes in self.get_BoxNodes().iteritems():
461 # this is the brute force version, kill all qemus on that host box
462 TestBoxQemu(box, self.options.buildname).qemu_list_all()
465 # kill only the qemus related to this test
466 def qemu_list_mine(self):
467 'list qemu instances for our nodes'
468 for (box,nodes) in self.get_BoxNodes().iteritems():
469 # the fine-grain version
474 # kill only the qemus related to this test
475 def qemu_clean_mine(self):
476 'cleanup (rm -rf) qemu instances for our nodes'
477 for box,nodes in self.get_BoxNodes().iteritems():
478 # the fine-grain version
483 # kill only the right qemus
484 def qemu_kill_mine(self):
485 'kill the qemu instances for our nodes'
486 for box,nodes in self.get_BoxNodes().iteritems():
487 # the fine-grain version
492 #################### display config
494 "show test configuration after localization"
499 # uggly hack to make sure 'run export' only reports about the 1st plc
500 # to avoid confusion - also we use 'inri_slice1' in various aliases..
503 "print cut'n paste-able stuff to export env variables to your shell"
504 # guess local domain from hostname
505 if TestPlc.exported_id > 1:
506 print "export GUESTHOSTNAME%d=%s" % (TestPlc.exported_id, self.plc_spec['vservername'])
508 TestPlc.exported_id += 1
509 domain = socket.gethostname().split('.',1)[1]
510 fqdn = "%s.%s" % (self.plc_spec['host_box'],domain)
511 print "export BUILD=%s" % self.options.buildname
512 print "export PLCHOSTLXC=%s" % fqdn
513 print "export GUESTNAME=%s" % self.plc_spec['vservername']
514 vplcname = self.plc_spec['vservername'].split('-')[-1]
515 print "export GUESTHOSTNAME=%s.%s"%(vplcname,domain)
516 # find hostname of first node
517 hostname,qemubox = self.all_node_infos()[0]
518 print "export KVMHOST=%s.%s" % (qemubox,domain)
519 print "export NODE=%s" % (hostname)
523 always_display_keys=['PLC_WWW_HOST','nodes','sites',]
524 def show_pass(self, passno):
525 for (key,val) in self.plc_spec.iteritems():
526 if not self.options.verbose and key not in TestPlc.always_display_keys:
531 self.display_site_spec(site)
532 for node in site['nodes']:
533 self.display_node_spec(node)
534 elif key=='initscripts':
535 for initscript in val:
536 self.display_initscript_spec(initscript)
539 self.display_slice_spec(slice)
542 self.display_key_spec(key)
544 if key not in ['sites', 'initscripts', 'slices', 'keys']:
545 print '+ ',key,':',val
547 def display_site_spec(self, site):
548 print '+ ======== site', site['site_fields']['name']
549 for k,v in site.iteritems():
550 if not self.options.verbose and k not in TestPlc.always_display_keys:
554 print '+ ','nodes : ',
556 print node['node_fields']['hostname'],'',
562 print user['name'],'',
564 elif k == 'site_fields':
565 print '+ login_base',':',v['login_base']
566 elif k == 'address_fields':
572 def display_initscript_spec(self, initscript):
573 print '+ ======== initscript', initscript['initscript_fields']['name']
575 def display_key_spec(self, key):
576 print '+ ======== key', key['key_name']
578 def display_slice_spec(self, slice):
579 print '+ ======== slice', slice['slice_fields']['name']
580 for k,v in slice.iteritems():
587 elif k == 'usernames':
593 elif k == 'slice_fields':
594 print '+ fields',':',
595 print 'max_nodes=',v['max_nodes'],
600 def display_node_spec(self, node):
601 print "+ node=%s host_box=%s" % (node['name'],node['host_box']),
602 print "hostname=", node['node_fields']['hostname'],
603 print "ip=", node['interface_fields']['ip']
604 if self.options.verbose:
605 utils.pprint("node details", node, depth=3)
607 # another entry point for just showing the boxes involved
608 def display_mapping(self):
609 TestPlc.display_mapping_plc(self.plc_spec)
613 def display_mapping_plc(plc_spec):
614 print '+ MyPLC',plc_spec['name']
615 # WARNING this would not be right for lxc-based PLC's - should be harmless though
616 print '+\tvserver address = root@%s:/vservers/%s' % (plc_spec['host_box'], plc_spec['vservername'])
617 print '+\tIP = %s/%s' % (plc_spec['settings']['PLC_API_HOST'], plc_spec['vserverip'])
618 for site_spec in plc_spec['sites']:
619 for node_spec in site_spec['nodes']:
620 TestPlc.display_mapping_node(node_spec)
623 def display_mapping_node(node_spec):
624 print '+ NODE %s' % (node_spec['name'])
625 print '+\tqemu box %s' % node_spec['host_box']
626 print '+\thostname=%s' % node_spec['node_fields']['hostname']
628 # write a timestamp in /vservers/<>.timestamp
629 # cannot be inside the vserver, that causes vserver .. build to cough
630 def plcvm_timestamp(self):
631 "Create a timestamp to remember creation date for this plc"
632 now = int(time.time())
633 # TODO-lxc check this one
634 # a first approx. is to store the timestamp close to the VM root like vs does
635 stamp_path = self.vm_timestamp_path()
636 stamp_dir = os.path.dirname(stamp_path)
637 utils.system(self.test_ssh.actual_command("mkdir -p %s" % stamp_dir))
638 return utils.system(self.test_ssh.actual_command("echo %d > %s" % (now, stamp_path))) == 0
640 # this is called inconditionnally at the beginning of the test sequence
641 # just in case this is a rerun, so if the vm is not running it's fine
642 def plcvm_delete(self):
643 "vserver delete the test myplc"
644 stamp_path = self.vm_timestamp_path()
645 self.run_in_host("rm -f %s" % stamp_path)
646 self.run_in_host("virsh -c lxc:// destroy %s" % self.vservername)
647 self.run_in_host("virsh -c lxc:// undefine %s" % self.vservername)
648 self.run_in_host("rm -fr /vservers/%s" % self.vservername)
652 # historically the build was being fetched by the tests
653 # now the build pushes itself as a subdir of the tests workdir
654 # so that the tests do not have to worry about extracting the build (svn, git, or whatever)
655 def plcvm_create(self):
656 "vserver creation (no install done)"
657 # push the local build/ dir to the testplc box
659 # a full path for the local calls
660 build_dir = os.path.dirname(sys.argv[0])
661 # sometimes this is empty - set to "." in such a case
664 build_dir += "/build"
666 # use a standard name - will be relative to remote buildname
668 # remove for safety; do *not* mkdir first, otherwise we end up with build/build/
669 self.test_ssh.rmdir(build_dir)
670 self.test_ssh.copy(build_dir, recursive=True)
671 # the repo url is taken from arch-rpms-url
672 # with the last step (i386) removed
673 repo_url = self.options.arch_rpms_url
674 for level in [ 'arch' ]:
675 repo_url = os.path.dirname(repo_url)
677 # invoke initvm (drop support for vs)
678 script = "lbuild-initvm.sh"
680 # pass the vbuild-nightly options to [lv]test-initvm
681 script_options += " -p %s" % self.options.personality
682 script_options += " -d %s" % self.options.pldistro
683 script_options += " -f %s" % self.options.fcdistro
684 script_options += " -r %s" % repo_url
685 vserver_name = self.vservername
687 vserver_hostname = socket.gethostbyaddr(self.vserverip)[0]
688 script_options += " -n %s" % vserver_hostname
690 print "Cannot reverse lookup %s" % self.vserverip
691 print "This is considered fatal, as this might pollute the test results"
693 create_vserver="%(build_dir)s/%(script)s %(script_options)s %(vserver_name)s" % locals()
694 return self.run_in_host(create_vserver) == 0
697 def plc_install(self):
698 "yum install myplc, noderepo, and the plain bootstrapfs"
700 # workaround for getting pgsql8.2 on centos5
701 if self.options.fcdistro == "centos5":
702 self.run_in_guest("rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-3.noarch.rpm")
705 if self.options.personality == "linux32":
707 elif self.options.personality == "linux64":
710 raise Exception, "Unsupported personality %r"%self.options.personality
711 nodefamily = "%s-%s-%s" % (self.options.pldistro, self.options.fcdistro, arch)
714 pkgs_list.append("slicerepo-%s" % nodefamily)
715 pkgs_list.append("myplc")
716 pkgs_list.append("noderepo-%s" % nodefamily)
717 pkgs_list.append("nodeimage-%s-plain" % nodefamily)
718 pkgs_string=" ".join(pkgs_list)
719 return self.yum_install(pkgs_list)
722 def mod_python(self):
723 """yum install mod_python, useful on f18 and above so as to avoid broken wsgi"""
724 return self.yum_install( ['mod_python'] )
727 def plc_configure(self):
729 tmpname = '%s.plc-config-tty' % self.name()
730 with open(tmpname,'w') as fileconf:
731 for (var,value) in self.plc_spec['settings'].iteritems():
732 fileconf.write('e %s\n%s\n'%(var,value))
733 fileconf.write('w\n')
734 fileconf.write('q\n')
735 utils.system('cat %s' % tmpname)
736 self.run_in_guest_piped('cat %s' % tmpname, 'plc-config-tty')
737 utils.system('rm %s' % tmpname)
740 # f14 is a bit odd in this respect, although this worked fine in guests up to f18
741 # however using a vplc guest under f20 requires this trick
742 # the symptom is this: service plc start
743 # Starting plc (via systemctl): Failed to get D-Bus connection: \
744 # Failed to connect to socket /org/freedesktop/systemd1/private: Connection refused
745 # weird thing is the doc says f14 uses upstart by default and not systemd
746 # so this sounds kind of harmless
747 def start_service(self, service):
748 return self.start_stop_service(service, 'start')
749 def stop_service(self, service):
750 return self.start_stop_service(service, 'stop')
752 def start_stop_service(self, service, start_or_stop):
753 "utility to start/stop a service with the special trick for f14"
754 if self.options.fcdistro != 'f14':
755 return self.run_in_guest("service %s %s" % (service, start_or_stop)) == 0
757 # patch /sbin/service so it does not reset environment
758 self.run_in_guest('sed -i -e \\"s,env -i,env,\\" /sbin/service')
759 # this is because our own scripts in turn call service
760 return self.run_in_guest("SYSTEMCTL_SKIP_REDIRECT=true service %s %s" % \
761 (service, start_or_stop)) == 0
765 return self.start_service('plc')
769 return self.stop_service('plc')
771 def plcvm_start(self):
772 "start the PLC vserver"
776 def plcvm_stop(self):
777 "stop the PLC vserver"
781 # stores the keys from the config for further use
782 def keys_store(self):
783 "stores test users ssh keys in keys/"
784 for key_spec in self.plc_spec['keys']:
785 TestKey(self,key_spec).store_key()
788 def keys_clean(self):
789 "removes keys cached in keys/"
790 utils.system("rm -rf ./keys")
793 # fetches the ssh keys in the plc's /etc/planetlab and stores them in keys/
794 # for later direct access to the nodes
795 def keys_fetch(self):
796 "gets ssh keys in /etc/planetlab/ and stores them locally in keys/"
798 if not os.path.isdir(dir):
800 vservername = self.vservername
801 vm_root = self.vm_root_in_host()
803 prefix = 'debug_ssh_key'
804 for ext in ['pub', 'rsa'] :
805 src = "%(vm_root)s/etc/planetlab/%(prefix)s.%(ext)s" % locals()
806 dst = "keys/%(vservername)s-debug.%(ext)s" % locals()
807 if self.test_ssh.fetch(src, dst) != 0:
812 "create sites with PLCAPI"
813 return self.do_sites()
815 def delete_sites(self):
816 "delete sites with PLCAPI"
817 return self.do_sites(action="delete")
819 def do_sites(self, action="add"):
820 for site_spec in self.plc_spec['sites']:
821 test_site = TestSite(self,site_spec)
822 if (action != "add"):
823 utils.header("Deleting site %s in %s" % (test_site.name(), self.name()))
824 test_site.delete_site()
825 # deleted with the site
826 #test_site.delete_users()
829 utils.header("Creating site %s & users in %s" % (test_site.name(), self.name()))
830 test_site.create_site()
831 test_site.create_users()
834 def delete_all_sites(self):
835 "Delete all sites in PLC, and related objects"
836 print 'auth_root', self.auth_root()
837 sites = self.apiserver.GetSites(self.auth_root(), {}, ['site_id','login_base'])
839 # keep automatic site - otherwise we shoot in our own foot, root_auth is not valid anymore
840 if site['login_base'] == self.plc_spec['settings']['PLC_SLICE_PREFIX']:
842 site_id = site['site_id']
843 print 'Deleting site_id', site_id
844 self.apiserver.DeleteSite(self.auth_root(), site_id)
848 "create nodes with PLCAPI"
849 return self.do_nodes()
850 def delete_nodes(self):
851 "delete nodes with PLCAPI"
852 return self.do_nodes(action="delete")
854 def do_nodes(self, action="add"):
855 for site_spec in self.plc_spec['sites']:
856 test_site = TestSite(self, site_spec)
858 utils.header("Deleting nodes in site %s" % test_site.name())
859 for node_spec in site_spec['nodes']:
860 test_node = TestNode(self, test_site, node_spec)
861 utils.header("Deleting %s" % test_node.name())
862 test_node.delete_node()
864 utils.header("Creating nodes for site %s in %s" % (test_site.name(), self.name()))
865 for node_spec in site_spec['nodes']:
866 utils.pprint('Creating node %s' % node_spec, node_spec)
867 test_node = TestNode(self, test_site, node_spec)
868 test_node.create_node()
871 def nodegroups(self):
872 "create nodegroups with PLCAPI"
873 return self.do_nodegroups("add")
874 def delete_nodegroups(self):
875 "delete nodegroups with PLCAPI"
876 return self.do_nodegroups("delete")
880 def translate_timestamp(start, grain, timestamp):
881 if timestamp < TestPlc.YEAR:
882 return start+timestamp*grain
887 def timestamp_printable(timestamp):
888 return time.strftime('%m-%d %H:%M:%S UTC', time.gmtime(timestamp))
891 "create leases (on reservable nodes only, use e.g. run -c default -c resa)"
892 now = int(time.time())
893 grain = self.apiserver.GetLeaseGranularity(self.auth_root())
894 print 'API answered grain=', grain
895 start = (now/grain)*grain
897 # find out all nodes that are reservable
898 nodes = self.all_reservable_nodenames()
900 utils.header("No reservable node found - proceeding without leases")
903 # attach them to the leases as specified in plc_specs
904 # this is where the 'leases' field gets interpreted as relative of absolute
905 for lease_spec in self.plc_spec['leases']:
906 # skip the ones that come with a null slice id
907 if not lease_spec['slice']:
909 lease_spec['t_from'] = TestPlc.translate_timestamp(start, grain, lease_spec['t_from'])
910 lease_spec['t_until'] = TestPlc.translate_timestamp(start, grain, lease_spec['t_until'])
911 lease_addition = self.apiserver.AddLeases(self.auth_root(), nodes, lease_spec['slice'],
912 lease_spec['t_from'],lease_spec['t_until'])
913 if lease_addition['errors']:
914 utils.header("Cannot create leases, %s"%lease_addition['errors'])
917 utils.header('Leases on nodes %r for %s from %d (%s) until %d (%s)' % \
918 (nodes, lease_spec['slice'],
919 lease_spec['t_from'], TestPlc.timestamp_printable(lease_spec['t_from']),
920 lease_spec['t_until'], TestPlc.timestamp_printable(lease_spec['t_until'])))
924 def delete_leases(self):
925 "remove all leases in the myplc side"
926 lease_ids = [ l['lease_id'] for l in self.apiserver.GetLeases(self.auth_root())]
927 utils.header("Cleaning leases %r" % lease_ids)
928 self.apiserver.DeleteLeases(self.auth_root(), lease_ids)
931 def list_leases(self):
932 "list all leases known to the myplc"
933 leases = self.apiserver.GetLeases(self.auth_root())
934 now = int(time.time())
936 current = l['t_until'] >= now
937 if self.options.verbose or current:
938 utils.header("%s %s from %s until %s" % \
939 (l['hostname'], l['name'],
940 TestPlc.timestamp_printable(l['t_from']),
941 TestPlc.timestamp_printable(l['t_until'])))
944 # create nodegroups if needed, and populate
945 def do_nodegroups(self, action="add"):
946 # 1st pass to scan contents
948 for site_spec in self.plc_spec['sites']:
949 test_site = TestSite(self,site_spec)
950 for node_spec in site_spec['nodes']:
951 test_node = TestNode(self, test_site, node_spec)
952 if node_spec.has_key('nodegroups'):
953 nodegroupnames = node_spec['nodegroups']
954 if isinstance(nodegroupnames, StringTypes):
955 nodegroupnames = [ nodegroupnames ]
956 for nodegroupname in nodegroupnames:
957 if not groups_dict.has_key(nodegroupname):
958 groups_dict[nodegroupname] = []
959 groups_dict[nodegroupname].append(test_node.name())
960 auth = self.auth_root()
962 for (nodegroupname,group_nodes) in groups_dict.iteritems():
964 print 'nodegroups:', 'dealing with nodegroup',\
965 nodegroupname, 'on nodes', group_nodes
966 # first, check if the nodetagtype is here
967 tag_types = self.apiserver.GetTagTypes(auth, {'tagname':nodegroupname})
969 tag_type_id = tag_types[0]['tag_type_id']
971 tag_type_id = self.apiserver.AddTagType(auth,
972 {'tagname' : nodegroupname,
973 'description' : 'for nodegroup %s' % nodegroupname,
974 'category' : 'test'})
975 print 'located tag (type)', nodegroupname, 'as', tag_type_id
977 nodegroups = self.apiserver.GetNodeGroups(auth, {'groupname' : nodegroupname})
979 self.apiserver.AddNodeGroup(auth, nodegroupname, tag_type_id, 'yes')
980 print 'created nodegroup', nodegroupname, \
981 'from tagname', nodegroupname, 'and value', 'yes'
982 # set node tag on all nodes, value='yes'
983 for nodename in group_nodes:
985 self.apiserver.AddNodeTag(auth, nodename, nodegroupname, "yes")
987 traceback.print_exc()
988 print 'node', nodename, 'seems to already have tag', nodegroupname
991 expect_yes = self.apiserver.GetNodeTags(auth,
992 {'hostname' : nodename,
993 'tagname' : nodegroupname},
994 ['value'])[0]['value']
995 if expect_yes != "yes":
996 print 'Mismatch node tag on node',nodename,'got',expect_yes
999 if not self.options.dry_run:
1000 print 'Cannot find tag', nodegroupname, 'on node', nodename
1004 print 'cleaning nodegroup', nodegroupname
1005 self.apiserver.DeleteNodeGroup(auth, nodegroupname)
1007 traceback.print_exc()
1011 # a list of TestNode objs
1012 def all_nodes(self):
1014 for site_spec in self.plc_spec['sites']:
1015 test_site = TestSite(self,site_spec)
1016 for node_spec in site_spec['nodes']:
1017 nodes.append(TestNode(self, test_site, node_spec))
1020 # return a list of tuples (nodename,qemuname)
1021 def all_node_infos(self) :
1023 for site_spec in self.plc_spec['sites']:
1024 node_infos += [ (node_spec['node_fields']['hostname'], node_spec['host_box']) \
1025 for node_spec in site_spec['nodes'] ]
1028 def all_nodenames(self):
1029 return [ x[0] for x in self.all_node_infos() ]
1030 def all_reservable_nodenames(self):
1032 for site_spec in self.plc_spec['sites']:
1033 for node_spec in site_spec['nodes']:
1034 node_fields = node_spec['node_fields']
1035 if 'node_type' in node_fields and node_fields['node_type'] == 'reservable':
1036 res.append(node_fields['hostname'])
1039 # silent_minutes : during the first <silent_minutes> minutes nothing gets printed
1040 def nodes_check_boot_state(self, target_boot_state, timeout_minutes,
1041 silent_minutes, period_seconds = 15):
1042 if self.options.dry_run:
1046 class CompleterTaskBootState(CompleterTask):
1047 def __init__(self, test_plc, hostname):
1048 self.test_plc = test_plc
1049 self.hostname = hostname
1050 self.last_boot_state = 'undef'
1051 def actual_run(self):
1053 node = self.test_plc.apiserver.GetNodes(self.test_plc.auth_root(),
1056 self.last_boot_state = node['boot_state']
1057 return self.last_boot_state == target_boot_state
1061 return "CompleterTaskBootState with node %s" % self.hostname
1062 def failure_epilogue(self):
1063 print "node %s in state %s - expected %s" %\
1064 (self.hostname, self.last_boot_state, target_boot_state)
1066 timeout = timedelta(minutes=timeout_minutes)
1067 graceout = timedelta(minutes=silent_minutes)
1068 period = timedelta(seconds=period_seconds)
1069 # the nodes that haven't checked yet - start with a full list and shrink over time
1070 utils.header("checking nodes boot state (expected %s)" % target_boot_state)
1071 tasks = [ CompleterTaskBootState(self,hostname) \
1072 for (hostname,_) in self.all_node_infos() ]
1073 message = 'check_boot_state={}'.format(target_boot_state)
1074 return Completer(tasks, message=message).run(timeout, graceout, period)
1076 def nodes_booted(self):
1077 return self.nodes_check_boot_state('boot', timeout_minutes=30, silent_minutes=28)
1079 def probe_kvm_iptables(self):
1080 (_,kvmbox) = self.all_node_infos()[0]
1081 TestSsh(kvmbox).run("iptables-save")
1085 def check_nodes_ping(self, timeout_seconds=30, period_seconds=10):
1086 class CompleterTaskPingNode(CompleterTask):
1087 def __init__(self, hostname):
1088 self.hostname = hostname
1089 def run(self, silent):
1090 command="ping -c 1 -w 1 %s >& /dev/null" % self.hostname
1091 return utils.system(command, silent=silent) == 0
1092 def failure_epilogue(self):
1093 print "Cannot ping node with name %s" % self.hostname
1094 timeout = timedelta(seconds = timeout_seconds)
1096 period = timedelta(seconds = period_seconds)
1097 node_infos = self.all_node_infos()
1098 tasks = [ CompleterTaskPingNode(h) for (h,_) in node_infos ]
1099 return Completer(tasks, message='ping_node').run(timeout, graceout, period)
1101 # ping node before we try to reach ssh, helpful for troubleshooting failing bootCDs
1102 def ping_node(self):
1104 return self.check_nodes_ping()
1106 def check_nodes_ssh(self, debug, timeout_minutes, silent_minutes, period_seconds=15):
1108 timeout = timedelta(minutes=timeout_minutes)
1109 graceout = timedelta(minutes=silent_minutes)
1110 period = timedelta(seconds=period_seconds)
1111 vservername = self.vservername
1114 completer_message = 'ssh_node_debug'
1115 local_key = "keys/%(vservername)s-debug.rsa" % locals()
1118 completer_message = 'ssh_node_boot'
1119 local_key = "keys/key_admin.rsa"
1120 utils.header("checking ssh access to nodes (expected in %s mode)" % message)
1121 node_infos = self.all_node_infos()
1122 tasks = [ CompleterTaskNodeSsh(nodename, qemuname, local_key,
1123 boot_state=message, dry_run=self.options.dry_run) \
1124 for (nodename, qemuname) in node_infos ]
1125 return Completer(tasks, message=completer_message).run(timeout, graceout, period)
1127 def ssh_node_debug(self):
1128 "Tries to ssh into nodes in debug mode with the debug ssh key"
1129 return self.check_nodes_ssh(debug = True,
1130 timeout_minutes = self.ssh_node_debug_timeout,
1131 silent_minutes = self.ssh_node_debug_silent)
1133 def ssh_node_boot(self):
1134 "Tries to ssh into nodes in production mode with the root ssh key"
1135 return self.check_nodes_ssh(debug = False,
1136 timeout_minutes = self.ssh_node_boot_timeout,
1137 silent_minutes = self.ssh_node_boot_silent)
1139 def node_bmlogs(self):
1140 "Checks that there's a non-empty dir. /var/log/bm/raw"
1141 return utils.system(self.actual_command_in_guest("ls /var/log/bm/raw")) == 0
1144 def qemu_local_init(self): pass
1146 def bootcd(self): pass
1148 def qemu_local_config(self): pass
1150 def nodestate_reinstall(self): pass
1152 def nodestate_safeboot(self): pass
1154 def nodestate_boot(self): pass
1156 def nodestate_show(self): pass
1158 def qemu_export(self): pass
1160 ### check hooks : invoke scripts from hooks/{node,slice}
1161 def check_hooks_node(self):
1162 return self.locate_first_node().check_hooks()
1163 def check_hooks_sliver(self) :
1164 return self.locate_first_sliver().check_hooks()
1166 def check_hooks(self):
1167 "runs unit tests in the node and slice contexts - see hooks/{node,slice}"
1168 return self.check_hooks_node() and self.check_hooks_sliver()
1171 def do_check_initscripts(self):
1172 class CompleterTaskInitscript(CompleterTask):
1173 def __init__(self, test_sliver, stamp):
1174 self.test_sliver = test_sliver
1176 def actual_run(self):
1177 return self.test_sliver.check_initscript_stamp(self.stamp)
1179 return "initscript checker for %s" % self.test_sliver.name()
1180 def failure_epilogue(self):
1181 print "initscript stamp %s not found in sliver %s"%\
1182 (self.stamp, self.test_sliver.name())
1185 for slice_spec in self.plc_spec['slices']:
1186 if not slice_spec.has_key('initscriptstamp'):
1188 stamp = slice_spec['initscriptstamp']
1189 slicename = slice_spec['slice_fields']['name']
1190 for nodename in slice_spec['nodenames']:
1191 print 'nodename', nodename, 'slicename', slicename, 'stamp', stamp
1192 site,node = self.locate_node(nodename)
1193 # xxx - passing the wrong site - probably harmless
1194 test_site = TestSite(self, site)
1195 test_slice = TestSlice(self, test_site, slice_spec)
1196 test_node = TestNode(self, test_site, node)
1197 test_sliver = TestSliver(self, test_node, test_slice)
1198 tasks.append(CompleterTaskInitscript(test_sliver, stamp))
1199 return Completer(tasks, message='check_initscripts').\
1200 run (timedelta(minutes=5), timedelta(minutes=4), timedelta(seconds=10))
1202 def check_initscripts(self):
1203 "check that the initscripts have triggered"
1204 return self.do_check_initscripts()
1206 def initscripts(self):
1207 "create initscripts with PLCAPI"
1208 for initscript in self.plc_spec['initscripts']:
1209 utils.pprint('Adding Initscript in plc %s' % self.plc_spec['name'], initscript)
1210 self.apiserver.AddInitScript(self.auth_root(), initscript['initscript_fields'])
1213 def delete_initscripts(self):
1214 "delete initscripts with PLCAPI"
1215 for initscript in self.plc_spec['initscripts']:
1216 initscript_name = initscript['initscript_fields']['name']
1217 print('Attempting to delete %s in plc %s' % (initscript_name, self.plc_spec['name']))
1219 self.apiserver.DeleteInitScript(self.auth_root(), initscript_name)
1220 print initscript_name, 'deleted'
1222 print 'deletion went wrong - probably did not exist'
1227 "create slices with PLCAPI"
1228 return self.do_slices(action="add")
1230 def delete_slices(self):
1231 "delete slices with PLCAPI"
1232 return self.do_slices(action="delete")
1234 def fill_slices(self):
1235 "add nodes in slices with PLCAPI"
1236 return self.do_slices(action="fill")
1238 def empty_slices(self):
1239 "remove nodes from slices with PLCAPI"
1240 return self.do_slices(action="empty")
1242 def do_slices(self, action="add"):
1243 for slice in self.plc_spec['slices']:
1244 site_spec = self.locate_site(slice['sitename'])
1245 test_site = TestSite(self,site_spec)
1246 test_slice=TestSlice(self,test_site,slice)
1247 if action == "delete":
1248 test_slice.delete_slice()
1249 elif action == "fill":
1250 test_slice.add_nodes()
1251 elif action == "empty":
1252 test_slice.delete_nodes()
1254 test_slice.create_slice()
1257 @slice_mapper__tasks(20, 10, 15)
1258 def ssh_slice(self): pass
1259 @slice_mapper__tasks(20, 19, 15)
1260 def ssh_slice_off(self): pass
1261 @slice_mapper__tasks(1, 1, 15)
1262 def slice_fs_present(self): pass
1263 @slice_mapper__tasks(1, 1, 15)
1264 def slice_fs_deleted(self): pass
1266 # use another name so we can exclude/ignore it from the tests on the nightly command line
1267 def ssh_slice_again(self): return self.ssh_slice()
1268 # note that simply doing ssh_slice_again=ssh_slice would kind of work too
1269 # but for some reason the ignore-wrapping thing would not
1272 def ssh_slice_basics(self): pass
1274 def check_vsys_defaults(self): pass
1277 def keys_clear_known_hosts(self): pass
1279 def plcapi_urls(self):
1280 return PlcapiUrlScanner(self.auth_root(), ip=self.vserverip).scan()
1282 def speed_up_slices(self):
1283 "tweak nodemanager cycle (wait time) to 30+/-10 s"
1284 return self._speed_up_slices (30, 10)
1285 def super_speed_up_slices(self):
1286 "dev mode: tweak nodemanager cycle (wait time) to 5+/-1 s"
1287 return self._speed_up_slices(5, 1)
1289 def _speed_up_slices(self, p, r):
1290 # create the template on the server-side
1291 template = "%s.nodemanager" % self.name()
1292 with open(template,"w") as template_file:
1293 template_file.write('OPTIONS="-p %s -r %s -d"\n'%(p,r))
1294 in_vm = "/var/www/html/PlanetLabConf/nodemanager"
1295 remote = "%s/%s" % (self.vm_root_in_host(), in_vm)
1296 self.test_ssh.copy_abs(template, remote)
1298 if not self.apiserver.GetConfFiles(self.auth_root(),
1299 {'dest' : '/etc/sysconfig/nodemanager'}):
1300 self.apiserver.AddConfFile(self.auth_root(),
1301 {'dest' : '/etc/sysconfig/nodemanager',
1302 'source' : 'PlanetLabConf/nodemanager',
1303 'postinstall_cmd' : 'service nm restart',})
1306 def debug_nodemanager(self):
1307 "sets verbose mode for nodemanager, and speeds up cycle even more (needs speed_up_slices first)"
1308 template = "%s.nodemanager" % self.name()
1309 with open(template,"w") as template_file:
1310 template_file.write('OPTIONS="-p 10 -r 6 -v -d"\n')
1311 in_vm = "/var/www/html/PlanetLabConf/nodemanager"
1312 remote = "%s/%s" % (self.vm_root_in_host(), in_vm)
1313 self.test_ssh.copy_abs(template, remote)
1317 def qemu_start(self) : pass
1320 def qemu_timestamp(self) : pass
1322 # when a spec refers to a node possibly on another plc
1323 def locate_sliver_obj_cross(self, nodename, slicename, other_plcs):
1324 for plc in [ self ] + other_plcs:
1326 return plc.locate_sliver_obj(nodename, slicename)
1329 raise Exception, "Cannot locate sliver %s@%s among all PLCs" % (nodename, slicename)
1331 # implement this one as a cross step so that we can take advantage of different nodes
1332 # in multi-plcs mode
1333 def cross_check_tcp(self, other_plcs):
1334 "check TCP connectivity between 2 slices (or in loopback if only one is defined)"
1335 if 'tcp_specs' not in self.plc_spec or not self.plc_spec['tcp_specs']:
1336 utils.header("check_tcp: no/empty config found")
1338 specs = self.plc_spec['tcp_specs']
1341 # first wait for the network to be up and ready from the slices
1342 class CompleterTaskNetworkReadyInSliver(CompleterTask):
1343 def __init__(self, test_sliver):
1344 self.test_sliver = test_sliver
1345 def actual_run(self):
1346 return self.test_sliver.check_tcp_ready(port = 9999)
1348 return "network ready checker for %s" % self.test_sliver.name()
1349 def failure_epilogue(self):
1350 print "could not bind port from sliver %s" % self.test_sliver.name()
1354 managed_sliver_names = set()
1356 # locate the TestSliver instances involved, and cache them in the spec instance
1357 spec['s_sliver'] = self.locate_sliver_obj_cross(spec['server_node'], spec['server_slice'], other_plcs)
1358 spec['c_sliver'] = self.locate_sliver_obj_cross(spec['client_node'], spec['client_slice'], other_plcs)
1359 message = "Will check TCP between s=%s and c=%s" % \
1360 (spec['s_sliver'].name(), spec['c_sliver'].name())
1361 if 'client_connect' in spec:
1362 message += " (using %s)" % spec['client_connect']
1363 utils.header(message)
1364 # we need to check network presence in both slivers, but also
1365 # avoid to insert a sliver several times
1366 for sliver in [ spec['s_sliver'], spec['c_sliver'] ]:
1367 if sliver.name() not in managed_sliver_names:
1368 tasks.append(CompleterTaskNetworkReadyInSliver(sliver))
1369 # add this sliver's name in the set
1370 managed_sliver_names .update( {sliver.name()} )
1372 # wait for the netork to be OK in all server sides
1373 if not Completer(tasks, message='check for network readiness in slivers').\
1374 run(timedelta(seconds=30), timedelta(seconds=24), period=timedelta(seconds=5)):
1377 # run server and client
1381 # the issue here is that we have the server run in background
1382 # and so we have no clue if it took off properly or not
1383 # looks like in some cases it does not
1384 if not spec['s_sliver'].run_tcp_server(port, timeout=20):
1388 # idem for the client side
1389 # use nodename from located sliver, unless 'client_connect' is set
1390 if 'client_connect' in spec:
1391 destination = spec['client_connect']
1393 destination = spec['s_sliver'].test_node.name()
1394 if not spec['c_sliver'].run_tcp_client(destination, port):
1398 # painfully enough, we need to allow for some time as netflow might show up last
1399 def check_system_slice(self):
1400 "all nodes: check that a system slice is alive"
1401 # netflow currently not working in the lxc distro
1402 # drl not built at all in the wtx distro
1403 # if we find either of them we're happy
1404 return self.check_netflow() or self.check_drl()
1407 def check_netflow(self): return self._check_system_slice('netflow')
1408 def check_drl(self): return self._check_system_slice('drl')
1410 # we have the slices up already here, so it should not take too long
1411 def _check_system_slice(self, slicename, timeout_minutes=5, period_seconds=15):
1412 class CompleterTaskSystemSlice(CompleterTask):
1413 def __init__(self, test_node, dry_run):
1414 self.test_node = test_node
1415 self.dry_run = dry_run
1416 def actual_run(self):
1417 return self.test_node._check_system_slice(slicename, dry_run=self.dry_run)
1419 return "System slice %s @ %s" % (slicename, self.test_node.name())
1420 def failure_epilogue(self):
1421 print "COULD not find system slice %s @ %s"%(slicename, self.test_node.name())
1422 timeout = timedelta(minutes=timeout_minutes)
1423 silent = timedelta(0)
1424 period = timedelta(seconds=period_seconds)
1425 tasks = [ CompleterTaskSystemSlice(test_node, self.options.dry_run) \
1426 for test_node in self.all_nodes() ]
1427 return Completer(tasks, message='_check_system_slice').run(timeout, silent, period)
1429 def plcsh_stress_test(self):
1430 "runs PLCAPI stress test, that checks Add/Update/Delete on all types - preserves contents"
1431 # install the stress-test in the plc image
1432 location = "/usr/share/plc_api/plcsh_stress_test.py"
1433 remote = "%s/%s" % (self.vm_root_in_host(), location)
1434 self.test_ssh.copy_abs("plcsh_stress_test.py", remote)
1436 command += " -- --check"
1437 if self.options.size == 1:
1438 command += " --tiny"
1439 return self.run_in_guest(command) == 0
1441 # populate runs the same utility without slightly different options
1442 # in particular runs with --preserve (dont cleanup) and without --check
1443 # also it gets run twice, once with the --foreign option for creating fake foreign entries
1445 def sfa_install_all(self):
1446 "yum install sfa sfa-plc sfa-sfatables sfa-client"
1447 return self.yum_install("sfa sfa-plc sfa-sfatables sfa-client")
1449 def sfa_install_core(self):
1451 return self.yum_install("sfa")
1453 def sfa_install_plc(self):
1454 "yum install sfa-plc"
1455 return self.yum_install("sfa-plc")
1457 def sfa_install_sfatables(self):
1458 "yum install sfa-sfatables"
1459 return self.yum_install("sfa-sfatables")
1461 # for some very odd reason, this sometimes fails with the following symptom
1462 # # yum install sfa-client
1463 # Setting up Install Process
1465 # Downloading Packages:
1466 # Running rpm_check_debug
1467 # Running Transaction Test
1468 # Transaction Test Succeeded
1469 # Running Transaction
1470 # Transaction couldn't start:
1471 # installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem
1472 # [('installing package sfa-client-2.1-7.onelab.2012.05.23.i686 needs 68KB on the / filesystem', (9, '/', 69632L))]
1473 # even though in the same context I have
1474 # [2012.05.23--f14-32-sfastd1-1-vplc07] / # df -h
1475 # Filesystem Size Used Avail Use% Mounted on
1476 # /dev/hdv1 806G 264G 501G 35% /
1477 # none 16M 36K 16M 1% /tmp
1479 # so as a workaround, we first try yum install, and then invoke rpm on the cached rpm...
1480 def sfa_install_client(self):
1481 "yum install sfa-client"
1482 first_try = self.yum_install("sfa-client")
1485 utils.header("********** Regular yum failed - special workaround in place, 2nd chance")
1486 code, cached_rpm_path = \
1487 utils.output_of(self.actual_command_in_guest('find /var/cache/yum -name sfa-client\*.rpm'))
1488 utils.header("rpm_path=<<%s>>" % rpm_path)
1490 self.run_in_guest("rpm -i %s" % cached_rpm_path)
1491 return self.yum_check_installed("sfa-client")
1493 def sfa_dbclean(self):
1494 "thoroughly wipes off the SFA database"
1495 return self.run_in_guest("sfaadmin reg nuke") == 0 or \
1496 self.run_in_guest("sfa-nuke.py") == 0 or \
1497 self.run_in_guest("sfa-nuke-plc.py") == 0
1499 def sfa_fsclean(self):
1500 "cleanup /etc/sfa/trusted_roots and /var/lib/sfa"
1501 self.run_in_guest("rm -rf /etc/sfa/trusted_roots /var/lib/sfa/authorities")
1504 def sfa_plcclean(self):
1505 "cleans the PLC entries that were created as a side effect of running the script"
1507 sfa_spec = self.plc_spec['sfa']
1509 for auth_sfa_spec in sfa_spec['auth_sfa_specs']:
1510 login_base = auth_sfa_spec['login_base']
1512 self.apiserver.DeleteSite(self.auth_root(),login_base)
1514 print "Site %s already absent from PLC db"%login_base
1516 for spec_name in ['pi_spec','user_spec']:
1517 user_spec = auth_sfa_spec[spec_name]
1518 username = user_spec['email']
1520 self.apiserver.DeletePerson(self.auth_root(),username)
1522 # this in fact is expected as sites delete their members
1523 #print "User %s already absent from PLC db"%username
1526 print "REMEMBER TO RUN sfa_import AGAIN"
1529 def sfa_uninstall(self):
1530 "uses rpm to uninstall sfa - ignore result"
1531 self.run_in_guest("rpm -e sfa sfa-sfatables sfa-client sfa-plc")
1532 self.run_in_guest("rm -rf /var/lib/sfa")
1533 self.run_in_guest("rm -rf /etc/sfa")
1534 self.run_in_guest("rm -rf /var/log/sfa_access.log /var/log/sfa_import_plc.log /var/log/sfa.daemon")
1536 self.run_in_guest("rpm -e --noscripts sfa-plc")
1539 ### run unit tests for SFA
1540 # NOTE: for some reason on f14/i386, yum install sfa-tests fails for no reason
1541 # Running Transaction
1542 # Transaction couldn't start:
1543 # installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem
1544 # [('installing package sfa-tests-1.0-21.onelab.i686 needs 204KB on the / filesystem', (9, '/', 208896L))]
1545 # no matter how many Gbs are available on the testplc
1546 # could not figure out what's wrong, so...
1547 # if the yum install phase fails, consider the test is successful
1548 # other combinations will eventually run it hopefully
1549 def sfa_utest(self):
1550 "yum install sfa-tests and run SFA unittests"
1551 self.run_in_guest("yum -y install sfa-tests")
1552 # failed to install - forget it
1553 if self.run_in_guest("rpm -q sfa-tests") != 0:
1554 utils.header("WARNING: SFA unit tests failed to install, ignoring")
1556 return self.run_in_guest("/usr/share/sfa/tests/testAll.py") == 0
1560 dirname = "conf.%s" % self.plc_spec['name']
1561 if not os.path.isdir(dirname):
1562 utils.system("mkdir -p %s" % dirname)
1563 if not os.path.isdir(dirname):
1564 raise Exception,"Cannot create config dir for plc %s" % self.name()
1567 def conffile(self, filename):
1568 return "%s/%s" % (self.confdir(),filename)
1569 def confsubdir(self, dirname, clean, dry_run=False):
1570 subdirname = "%s/%s" % (self.confdir(),dirname)
1572 utils.system("rm -rf %s" % subdirname)
1573 if not os.path.isdir(subdirname):
1574 utils.system("mkdir -p %s" % subdirname)
1575 if not dry_run and not os.path.isdir(subdirname):
1576 raise "Cannot create config subdir %s for plc %s" % (dirname,self.name())
1579 def conffile_clean(self, filename):
1580 filename=self.conffile(filename)
1581 return utils.system("rm -rf %s" % filename)==0
1584 def sfa_configure(self):
1585 "run sfa-config-tty"
1586 tmpname = self.conffile("sfa-config-tty")
1587 with open(tmpname,'w') as fileconf:
1588 for (var,value) in self.plc_spec['sfa']['settings'].iteritems():
1589 fileconf.write('e %s\n%s\n'%(var,value))
1590 fileconf.write('w\n')
1591 fileconf.write('R\n')
1592 fileconf.write('q\n')
1593 utils.system('cat %s' % tmpname)
1594 self.run_in_guest_piped('cat %s' % tmpname, 'sfa-config-tty')
1597 def aggregate_xml_line(self):
1598 port = self.plc_spec['sfa']['neighbours-port']
1599 return '<aggregate addr="%s" hrn="%s" port="%r"/>' % \
1600 (self.vserverip, self.plc_spec['sfa']['settings']['SFA_REGISTRY_ROOT_AUTH'], port)
1602 def registry_xml_line(self):
1603 return '<registry addr="%s" hrn="%s" port="12345"/>' % \
1604 (self.vserverip, self.plc_spec['sfa']['settings']['SFA_REGISTRY_ROOT_AUTH'])
1607 # a cross step that takes all other plcs in argument
1608 def cross_sfa_configure(self, other_plcs):
1609 "writes aggregates.xml and registries.xml that point to all other PLCs in the test"
1610 # of course with a single plc, other_plcs is an empty list
1613 agg_fname = self.conffile("agg.xml")
1614 with open(agg_fname,"w") as out:
1615 out.write("<aggregates>%s</aggregates>\n" % \
1616 " ".join([ plc.aggregate_xml_line() for plc in other_plcs ]))
1617 utils.header("(Over)wrote %s" % agg_fname)
1618 reg_fname=self.conffile("reg.xml")
1619 with open(reg_fname,"w") as out:
1620 out.write("<registries>%s</registries>\n" % \
1621 " ".join([ plc.registry_xml_line() for plc in other_plcs ]))
1622 utils.header("(Over)wrote %s" % reg_fname)
1623 return self.test_ssh.copy_abs(agg_fname,
1624 '/%s/etc/sfa/aggregates.xml' % self.vm_root_in_host()) == 0 \
1625 and self.test_ssh.copy_abs(reg_fname,
1626 '/%s/etc/sfa/registries.xml' % self.vm_root_in_host()) == 0
1628 def sfa_import(self):
1629 "use sfaadmin to import from plc"
1630 auth = self.plc_spec['sfa']['settings']['SFA_REGISTRY_ROOT_AUTH']
1631 return self.run_in_guest('sfaadmin reg import_registry') == 0
1633 def sfa_start(self):
1635 return self.start_service('sfa')
1638 def sfi_configure(self):
1639 "Create /root/sfi on the plc side for sfi client configuration"
1640 if self.options.dry_run:
1641 utils.header("DRY RUN - skipping step")
1643 sfa_spec = self.plc_spec['sfa']
1644 # cannot use auth_sfa_mapper to pass dir_name
1645 for slice_spec in self.plc_spec['sfa']['auth_sfa_specs']:
1646 test_slice = TestAuthSfa(self, slice_spec)
1647 dir_basename = os.path.basename(test_slice.sfi_path())
1648 dir_name = self.confsubdir("dot-sfi/%s" % dir_basename,
1649 clean=True, dry_run=self.options.dry_run)
1650 test_slice.sfi_configure(dir_name)
1651 # push into the remote /root/sfi area
1652 location = test_slice.sfi_path()
1653 remote = "%s/%s" % (self.vm_root_in_host(), location)
1654 self.test_ssh.mkdir(remote, abs=True)
1655 # need to strip last level or remote otherwise we get an extra dir level
1656 self.test_ssh.copy_abs(dir_name, os.path.dirname(remote), recursive=True)
1660 def sfi_clean(self):
1661 "clean up /root/sfi on the plc side"
1662 self.run_in_guest("rm -rf /root/sfi")
1665 def sfa_rspec_empty(self):
1666 "expose a static empty rspec (ships with the tests module) in the sfi directory"
1667 filename = "empty-rspec.xml"
1669 for slice_spec in self.plc_spec['sfa']['auth_sfa_specs']:
1670 test_slice = TestAuthSfa(self, slice_spec)
1671 in_vm = test_slice.sfi_path()
1672 remote = "%s/%s" % (self.vm_root_in_host(), in_vm)
1673 if self.test_ssh.copy_abs(filename, remote) !=0:
1678 def sfa_register_site(self): pass
1680 def sfa_register_pi(self): pass
1682 def sfa_register_user(self): pass
1684 def sfa_update_user(self): pass
1686 def sfa_register_slice(self): pass
1688 def sfa_renew_slice(self): pass
1690 def sfa_get_expires(self): pass
1692 def sfa_discover(self): pass
1694 def sfa_rspec(self): pass
1696 def sfa_allocate(self): pass
1698 def sfa_allocate_empty(self): pass
1700 def sfa_provision(self): pass
1702 def sfa_provision_empty(self): pass
1704 def sfa_check_slice_plc(self): pass
1706 def sfa_check_slice_plc_empty(self): pass
1708 def sfa_update_slice(self): pass
1710 def sfa_remove_user_from_slice(self): pass
1712 def sfa_insert_user_in_slice(self): pass
1714 def sfi_list(self): pass
1716 def sfi_show_site(self): pass
1718 def sfi_show_slice(self): pass
1720 def sfi_show_slice_researchers(self): pass
1722 def ssh_slice_sfa(self): pass
1724 def sfa_delete_user(self): pass
1726 def sfa_delete_slice(self): pass
1730 return self.stop_service('sfa')
1733 "creates random entries in the PLCAPI"
1734 # install the stress-test in the plc image
1735 location = "/usr/share/plc_api/plcsh_stress_test.py"
1736 remote = "%s/%s" % (self.vm_root_in_host(), location)
1737 self.test_ssh.copy_abs("plcsh_stress_test.py", remote)
1739 command += " -- --preserve --short-names"
1740 local = (self.run_in_guest(command) == 0);
1741 # second run with --foreign
1742 command += ' --foreign'
1743 remote = (self.run_in_guest(command) == 0);
1744 return local and remote
1746 def gather_logs(self):
1747 "gets all possible logs from plc's/qemu node's/slice's for future reference"
1748 # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log.<plcname>/*
1749 # (1.b) get the plc's /var/lib/pgsql/data/pg_log/ -> logs/myplc.pgsql-log.<plcname>/*
1750 # (1.c) get the plc's /root/sfi -> logs/sfi.<plcname>/
1751 # (2) get all the nodes qemu log and store it as logs/node.qemu.<node>.log
1752 # (3) get the nodes /var/log and store is as logs/node.var-log.<node>/*
1753 # (4) as far as possible get the slice's /var/log as logs/sliver.var-log.<sliver>/*
1755 print "-------------------- TestPlc.gather_logs : PLC's /var/log"
1756 self.gather_var_logs()
1758 print "-------------------- TestPlc.gather_logs : PLC's /var/lib/psql/data/pg_log/"
1759 self.gather_pgsql_logs()
1761 print "-------------------- TestPlc.gather_logs : PLC's /root/sfi/"
1762 self.gather_root_sfi()
1764 print "-------------------- TestPlc.gather_logs : nodes's QEMU logs"
1765 for site_spec in self.plc_spec['sites']:
1766 test_site = TestSite(self,site_spec)
1767 for node_spec in site_spec['nodes']:
1768 test_node = TestNode(self, test_site, node_spec)
1769 test_node.gather_qemu_logs()
1771 print "-------------------- TestPlc.gather_logs : nodes's /var/log"
1772 self.gather_nodes_var_logs()
1774 print "-------------------- TestPlc.gather_logs : sample sliver's /var/log"
1775 self.gather_slivers_var_logs()
1778 def gather_slivers_var_logs(self):
1779 for test_sliver in self.all_sliver_objs():
1780 remote = test_sliver.tar_var_logs()
1781 utils.system("mkdir -p logs/sliver.var-log.%s" % test_sliver.name())
1782 command = remote + " | tar -C logs/sliver.var-log.%s -xf -" % test_sliver.name()
1783 utils.system(command)
1786 def gather_var_logs(self):
1787 utils.system("mkdir -p logs/myplc.var-log.%s" % self.name())
1788 to_plc = self.actual_command_in_guest("tar -C /var/log/ -cf - .")
1789 command = to_plc + "| tar -C logs/myplc.var-log.%s -xf -" % self.name()
1790 utils.system(command)
1791 command = "chmod a+r,a+x logs/myplc.var-log.%s/httpd" % self.name()
1792 utils.system(command)
1794 def gather_pgsql_logs(self):
1795 utils.system("mkdir -p logs/myplc.pgsql-log.%s" % self.name())
1796 to_plc = self.actual_command_in_guest("tar -C /var/lib/pgsql/data/pg_log/ -cf - .")
1797 command = to_plc + "| tar -C logs/myplc.pgsql-log.%s -xf -" % self.name()
1798 utils.system(command)
1800 def gather_root_sfi(self):
1801 utils.system("mkdir -p logs/sfi.%s"%self.name())
1802 to_plc = self.actual_command_in_guest("tar -C /root/sfi/ -cf - .")
1803 command = to_plc + "| tar -C logs/sfi.%s -xf -"%self.name()
1804 utils.system(command)
1806 def gather_nodes_var_logs(self):
1807 for site_spec in self.plc_spec['sites']:
1808 test_site = TestSite(self, site_spec)
1809 for node_spec in site_spec['nodes']:
1810 test_node = TestNode(self, test_site, node_spec)
1811 test_ssh = TestSsh(test_node.name(), key="keys/key_admin.rsa")
1812 command = test_ssh.actual_command("tar -C /var/log -cf - .")
1813 command = command + "| tar -C logs/node.var-log.%s -xf -" % test_node.name()
1814 utils.system("mkdir -p logs/node.var-log.%s" % test_node.name())
1815 utils.system(command)
1818 # returns the filename to use for sql dump/restore, using options.dbname if set
1819 def dbfile(self, database):
1820 # uses options.dbname if it is found
1822 name = self.options.dbname
1823 if not isinstance(name, StringTypes):
1829 return "/root/%s-%s.sql" % (database, name)
1831 def plc_db_dump(self):
1832 'dump the planetlab5 DB in /root in the PLC - filename has time'
1833 dump=self.dbfile("planetab5")
1834 self.run_in_guest('pg_dump -U pgsqluser planetlab5 -f '+ dump)
1835 utils.header('Dumped planetlab5 database in %s' % dump)
1838 def plc_db_restore(self):
1839 'restore the planetlab5 DB - looks broken, but run -n might help'
1840 dump = self.dbfile("planetab5")
1841 ##stop httpd service
1842 self.run_in_guest('service httpd stop')
1843 # xxx - need another wrapper
1844 self.run_in_guest_piped('echo drop database planetlab5', 'psql --user=pgsqluser template1')
1845 self.run_in_guest('createdb -U postgres --encoding=UNICODE --owner=pgsqluser planetlab5')
1846 self.run_in_guest('psql -U pgsqluser planetlab5 -f ' + dump)
1847 ##starting httpd service
1848 self.run_in_guest('service httpd start')
1850 utils.header('Database restored from ' + dump)
1853 def create_ignore_steps():
1854 for step in TestPlc.default_steps + TestPlc.other_steps:
1855 # default step can have a plc qualifier
1857 step, qualifier = step.split('@')
1858 # or be defined as forced or ignored by default
1859 for keyword in ['_ignore','_force']:
1860 if step.endswith(keyword):
1861 step=step.replace(keyword,'')
1862 if step == SEP or step == SEPSFA :
1864 method = getattr(TestPlc,step)
1865 name = step + '_ignore'
1866 wrapped = ignore_result(method)
1867 # wrapped.__doc__ = method.__doc__ + " (run in ignore-result mode)"
1868 setattr(TestPlc, name, wrapped)
1871 # def ssh_slice_again_ignore (self): pass
1873 # def check_initscripts_ignore (self): pass
1875 def standby_1_through_20(self):
1876 """convenience function to wait for a specified number of minutes"""
1879 def standby_1(): pass
1881 def standby_2(): pass
1883 def standby_3(): pass
1885 def standby_4(): pass
1887 def standby_5(): pass
1889 def standby_6(): pass
1891 def standby_7(): pass
1893 def standby_8(): pass
1895 def standby_9(): pass
1897 def standby_10(): pass
1899 def standby_11(): pass
1901 def standby_12(): pass
1903 def standby_13(): pass
1905 def standby_14(): pass
1907 def standby_15(): pass
1909 def standby_16(): pass
1911 def standby_17(): pass
1913 def standby_18(): pass
1915 def standby_19(): pass
1917 def standby_20(): pass
1919 # convenience for debugging the test logic
1920 def yes(self): return True
1921 def no(self): return False
1922 def fail(self): return False