From: Thierry Parmentelat Date: Tue, 17 Mar 2015 09:49:39 +0000 (+0100) Subject: a first rough implementation for testing 'bonding' myplc's X-Git-Tag: tests-5.3-10~3 X-Git-Url: http://git.onelab.eu/?p=tests.git;a=commitdiff_plain;h=65c93a296f2e481a312e7846f518eb2eaa2fc080 a first rough implementation for testing 'bonding' myplc's typicallly, you would have 2 complete builds b1 and b2 then on testmaster in b1 you can do $ run --bonding b2 or simply $ run -g b2 in order to expose the nodeflavour and slicefamily provided by b2 to build b1 for now it only does the partial-rpms & yum install bootcd nodeimage noderepo & packages reindexing need some more work to bring a actual nodes and slices online --- diff --git a/system/TestBonding.py b/system/TestBonding.py new file mode 100644 index 0000000..4066e44 --- /dev/null +++ b/system/TestBonding.py @@ -0,0 +1,182 @@ +""" +Utilities to create a setup made of 2 different builds +read : 2 different node flavours +so that each myplc knows about the nodeflavour/slicefamily supported +by the other one +--- +This would be the basics for running tests on multi-node myplc, +in particular for node upgrades +""" + +#################### WARNING + +# this feature relies on a few assumptions that need to be taken care of +# more or less manually; this is based on the onelab.eu setup + +# (*) the build host is expected to have /root/git-build.sh reasonably up-to-date +# with our build module, so we can locate partial-repo.sh +# this utility needs to be run on the build host so we can point at a PARTIAL-RPMS +# sub-repo that exposes the +# bootcd/bootstraps/ and the like rpms from one flavour to another + +# a utility to create a bonding_plc_spec from +# a plc_spec and just a buildname + +def onelab_bonding_spec (buildname): + + # essentially generic .. + buildname = buildname + + with open ("../{}/arg-fcdistro".format(buildname)) as input: + fcdistro = input.read().strip() + with open ("../{}/arg-pldistro".format(buildname)) as input: + pldistro = input.read().strip() + with open ("../{}/arg-ips-bplc".format(buildname)) as input: + plc_box = input.read().strip().split()[0] + # e.g. http://build.onelab.eu/onelab//2015.03.15--f14/RPMS/x86_64 + with open ("../{}/arg-arch-rpms-url".format(buildname)) as input: + arch_rpms_url = input.read().strip() + arch = arch_rpms_url.split('/')[-1] + build_www_host = arch_rpms_url.split('/')[2] + base_url = arch_rpms_url.replace("RPMS/{}".format(arch), "PARTIAL-RPMS") + + # onelab specifics + build_www_git = '/root/git-build/' + build_www_dir = '/build/{}/{}'.format(pldistro, buildname) + + return locals() + +#################### +import os, os.path + +import utils +from TestSsh import TestSsh + +#################### +class TestBonding(object): + + """ + Holds details about a 'bonding' build + so we can configure the local myplc (test_plc) + for multi-flavour nodes and slices + options is a TestMain options + """ + + def __init__(self, test_plc, bonding_spec, options): + """ + test_plc is one local TestPlc instance + bonding_spec is a dictionary that gives details on + the build we want to be bonding with + """ + self.test_plc = test_plc + self.bonding_spec = bonding_spec + self.options = options + # the local build & plc is described in options + # the bonding build is described in bonding_spec + + def nodefamily(self): + return "{pldistro}-{fcdistro}-{arch}".format(**self.bonding_spec) + + def init_partial(self): + """ + runs partial-repo.sh for the bonding build + this action takes place on the build host + """ + test_ssh = TestSsh (self.bonding_spec['build_www_host']) + command = "{build_www_git}/partial-repo.sh -i {build_www_dir}".\ + format(**self.bonding_spec) + + return test_ssh.run (command, dry_run = self.options.dry_run) == 0 + + + def add_yum(self): + """ + creates a separate yum.repo file in the myplc box + where our own build runs, and that points at the partial + repo for the bonding build + """ + + # create a .repo file locally + yumrepo_contents = """ +[{buildname}] +name=Partial repo from bonding build {buildname} +baseurl={base_url} +enabled=1 +gpgcheck=0 +""".format(**self.bonding_spec) + + yumrepo_local = '{buildname}-partial.repo'.\ + format(**self.bonding_spec) + with open(yumrepo_local, 'w') as yumrepo_file: + yumrepo_file.write(yumrepo_contents) + utils.header("(Over)wrote {}".format(yumrepo_local)) + + # push onto our myplc instance + test_ssh = TestSsh (self.test_plc.vserverip) + + yumrepo_remote = '/etc/yum.repos.d/{bonding_buildname}-partial.repo'.\ + format(bonding_buildname = self.bonding_spec['buildname']) + + if test_ssh.copy_abs (yumrepo_local, yumrepo_remote, + dry_run=self.options.dry_run) != 0: + return False + + # xxx TODO looks like drupal also needs to be excluded + # from the 2 entries in building.repo + # otherwise subsequent yum update calls will fail + + return True + + def install_rpms(self): + """ + once the 2 operations above have been performed, we can + actually install the various rpms that provide support for the + nodeflavour/slicefamily offered byt the bonding build to our own build + """ + + test_ssh = TestSsh (self.test_plc.vserverip) + + command1 = "yum -y update" + if test_ssh.run (command1, dry_run = self.options.dry_run) != 0: + return False + + nodefamily = self.nodefamily() + extra_list = [ 'bootcd', 'nodeimage', 'noderepo' ] + + extra_rpms = [ "{}-{}".format(rpm, nodefamily) for rpm in extra_list] + + command2 = "yum -y install " + " ".join(extra_rpms) + if test_ssh.run (command2, dry_run = self.options.dry_run) != 0: + return False + + command3 = "/etc/plc.d/packages force" + if test_ssh.run (command3, dry_run = self.options.dry_run) != 0: + return False + + return True + +### probably obsolete already +if __name__ == '__main__': + + from TestPlc import TestPlc + + from config_default import sample_test_plc_spec + test_plc_spec = sample_test_plc_spec() + test_plc = TestPlc (test_plc_spec) + test_plc.show() + + print test_plc.host_box + + from argparse import ArgumentParser + parser = ArgumentParser() + parser.add_argument ("-n", "--dry-run", dest='dry_run', default=False, + action='store_true', help="dry run") + parser.add_argument ("build_name") + args = parser.parse_args() + + test_bonding = TestBonding (test_plc, + onelab_bonding_spec(args.build_name), + dry_run = args.dry_run) + + test_bonding.bond () + diff --git a/system/TestMain.py b/system/TestMain.py index 1f470f6..01bf6cd 100755 --- a/system/TestMain.py +++ b/system/TestMain.py @@ -12,6 +12,7 @@ from datetime import datetime import utils from TestPlc import TestPlc, Ignored +from TestBonding import TestBonding, onelab_bonding_spec from TestSite import TestSite from TestNode import TestNode from macros import sequences @@ -106,12 +107,17 @@ class TestMain: utils.show_options("main options", options) def init_steps(self): - self.steps_message = 20*'x' + " Defaut steps are\n" + \ - TestPlc.printable_steps(TestPlc.default_steps) - self.steps_message += 20*'x' + " Other useful steps are\n" + \ - TestPlc.printable_steps(TestPlc.other_steps) - self.steps_message += 20*'x' + " Macro steps are\n" + \ - " ".join(Step.list_macros()) + self.steps_message = "" + if not self.options.bonding: + self.steps_message += 20*'x' + " Defaut steps are\n" + \ + TestPlc.printable_steps(TestPlc.default_steps) + self.steps_message += 20*'x' + " Other useful steps are\n" + \ + TestPlc.printable_steps(TestPlc.other_steps) + self.steps_message += 20*'x' + " Macro steps are\n" + \ + " ".join(Step.list_macros()) + else: + self.steps_message += 20*'x' + " Default steps with bonding are\n" + \ + TestPlc.printable_steps(TestPlc.bonding_steps) def list_steps(self): if not self.options.verbose: @@ -139,7 +145,6 @@ class TestMain: Step(stepname).print_doc() def run (self): - self.init_steps() usage = """usage: %%prog [options] steps arch-rpms-url defaults to the last value used, as stored in arg-arch-rpms-url, no default @@ -148,9 +153,10 @@ config defaults to the last value used, as stored in arg-config, ips_vnode, ips_vplc and ips_qemu defaults to the last value used, as stored in arg-ips-{bplc,vplc,bnode,vnode}, default is to use IP scanning steps refer to a method in TestPlc or to a step_* module + +run with -l to see a list of available steps === """%(TestMain.default_config) - usage += self.steps_message parser = ArgumentParser(usage = usage) parser.add_argument("-u", "--url", action="store", dest="arch_rpms_url", @@ -201,8 +207,8 @@ steps refer to a method in TestPlc or to a step_* module help="Show environment and exits") parser.add_argument("-t", "--trace", action="store", dest="trace_file", default=None, help="Trace file location") -# parser.add_argument("-g", "--bonding", action='store', dest='bonding', default=None, -# help="specify build to bond with") + parser.add_argument("-g", "--bonding", action='store', dest='bonding', default=None, + help="specify build to bond with") parser.add_argument("steps", nargs='*') self.options = parser.parse_args() @@ -282,21 +288,19 @@ steps refer to a method in TestPlc or to a step_* module # hack : if sfa is not among the published rpms, skip these tests TestPlc.check_whether_build_has_sfa(self.options.arch_rpms_url) - # use the default list of steps if unspecified - if len(self.options.steps) == 0: - self.options.steps = TestPlc.default_steps + # initialize steps + if not self.options.steps: + # defaults, depends on using bonding or not + if self.options.bonding: + self.options.steps = TestPlc.bonding_steps + else: + self.options.steps = TestPlc.default_steps if self.options.list_steps: self.init_steps() self.list_steps() return 'SUCCESS' - # steps - if not self.options.steps: - #default (all) steps - #self.options.steps=['dump','clean','install','populate'] - self.options.steps = TestPlc.default_steps - # rewrite '-' into '_' in step names self.options.steps = [ step.replace('-', '_') for step in self.options.steps ] self.options.exclude = [ step.replace('-', '_') for step in self.options.exclude ] @@ -376,6 +380,17 @@ steps refer to a method in TestPlc or to a step_* module # pass options to utils as well utils.init_options(self.options) + # populate TestBonding objects + # need to wait until here as we need all_plcs + if self.options.bonding: + ## allow to pass -g ../2015.03.15--f18 so we can use bash completion + self.options.bonding = os.path.basename(self.options.bonding) + # this will fail if ../{bonding} has not the right arg- files + for spec, test_plc in all_plcs: + test_plc.test_bonding = TestBonding (test_plc, + onelab_bonding_spec(self.options.bonding), + self.options) + overall_result = 'SUCCESS' all_step_infos = [] for step in self.options.steps: diff --git a/system/TestPlc.py b/system/TestPlc.py index 61ae1df..a60f619 100644 --- a/system/TestPlc.py +++ b/system/TestPlc.py @@ -23,6 +23,8 @@ from TestApiserver import TestApiserver from TestAuthSfa import TestAuthSfa from PlcapiUrlScanner import PlcapiUrlScanner +from TestBonding import TestBonding + has_sfa_cache_filename="sfa-cache" # step methods must take (self) and return a boolean (options is a member of the class) @@ -72,6 +74,17 @@ def slice_mapper(method): map_on_slices.__doc__ = TestSlice.__dict__[method.__name__].__doc__ return map_on_slices +def bonding_redirector(method): + bonding_name = method.__name__.replace('bonding_', '') + def redirect(self): + bonding_method = TestBonding.__dict__[bonding_name] + return bonding_method(self.test_bonding) + # maintain __name__ for ignore_result + redirect.__name__ = method.__name__ + # restore the doc text + redirect.__doc__ = TestBonding.__dict__[bonding_name].__doc__ + return redirect + # run a step but return True so that we can go on def ignore_result(method): def ignoring(self): @@ -194,6 +207,11 @@ class TestPlc: 'debug_nodemanager', 'slice_fs_present', SEP, 'standby_1_through_20','yes','no',SEP, ] + bonding_steps = [ + 'bonding_init_partial', + 'bonding_add_yum', + 'bonding_install_rpms', SEP, + ] @staticmethod def printable_steps(list): @@ -507,20 +525,20 @@ class TestPlc: return True TestPlc.exported_id += 1 domain = socket.gethostname().split('.',1)[1] - fqdn = "%s.%s" % (self.plc_spec['host_box'],domain) + fqdn = "%s.%s" % (self.plc_spec['host_box'], domain) print "export BUILD=%s" % self.options.buildname print "export PLCHOSTLXC=%s" % fqdn print "export GUESTNAME=%s" % self.plc_spec['vservername'] vplcname = self.plc_spec['vservername'].split('-')[-1] - print "export GUESTHOSTNAME=%s.%s"%(vplcname,domain) + print "export GUESTHOSTNAME=%s.%s"%(vplcname, domain) # find hostname of first node - hostname,qemubox = self.all_node_infos()[0] - print "export KVMHOST=%s.%s" % (qemubox,domain) + hostname, qemubox = self.all_node_infos()[0] + print "export KVMHOST=%s.%s" % (qemubox, domain) print "export NODE=%s" % (hostname) return True # entry point - always_display_keys=['PLC_WWW_HOST','nodes','sites',] + always_display_keys=['PLC_WWW_HOST', 'nodes', 'sites'] def show_pass(self, passno): for (key,val) in self.plc_spec.iteritems(): if not self.options.verbose and key not in TestPlc.always_display_keys: @@ -531,18 +549,18 @@ class TestPlc: self.display_site_spec(site) for node in site['nodes']: self.display_node_spec(node) - elif key=='initscripts': + elif key == 'initscripts': for initscript in val: self.display_initscript_spec(initscript) - elif key=='slices': + elif key == 'slices': for slice in val: self.display_slice_spec(slice) - elif key=='keys': + elif key == 'keys': for key in val: self.display_key_spec(key) elif passno == 1: if key not in ['sites', 'initscripts', 'slices', 'keys']: - print '+ ',key,':',val + print '+ ', key, ':', val def display_site_spec(self, site): print '+ ======== site', site['site_fields']['name'] @@ -562,7 +580,7 @@ class TestPlc: print user['name'],'', print '' elif k == 'site_fields': - print '+ login_base',':',v['login_base'] + print '+ login_base', ':', v['login_base'] elif k == 'address_fields': pass else: @@ -1743,6 +1761,19 @@ class TestPlc: remote = (self.run_in_guest(command) == 0); return local and remote + + #################### + @bonding_redirector + def bonding_init_partial(self): pass + + @bonding_redirector + def bonding_add_yum(self): pass + + @bonding_redirector + def bonding_install_rpms(self): pass + + #################### + def gather_logs(self): "gets all possible logs from plc's/qemu node's/slice's for future reference" # (1.a) get the plc's /var/log/ and store it locally in logs/myplc.var-log./*