From 560938db174121e93854fd39b66d29be86769864 Mon Sep 17 00:00:00 2001 From: Thierry Parmentelat Date: Fri, 4 Apr 2008 12:42:54 +0000 Subject: [PATCH] another round of rework --- system/Makefile | 16 +- system/TestApiserver.py | 59 +++++ system/TestBox.py | 13 +- system/TestMain.py | 43 +-- system/TestNode.py | 58 ++-- system/TestPlc.py | 248 ++++++++++-------- system/TestSite.py | 6 +- system/TestSlice.py | 32 +-- system/TestSliver.py | 112 ++------ system/TestSsh.py | 13 +- system/TestUser.py | 12 +- system/config_main.py | 52 ++-- system/template-qemu/env-qemu | 167 ------------ system/template-qemu/qemu-bridge-init | 139 ++++++++++ system/template-qemu/qemu-ifup | 18 +- .../{kill-qemu-node => qemu-kill-node} | 2 + .../{start-qemu-node => qemu-start-node} | 20 +- system/utils.py | 24 +- 18 files changed, 548 insertions(+), 486 deletions(-) create mode 100644 system/TestApiserver.py delete mode 100755 system/template-qemu/env-qemu create mode 100755 system/template-qemu/qemu-bridge-init rename system/template-qemu/{kill-qemu-node => qemu-kill-node} (94%) rename system/template-qemu/{start-qemu-node => qemu-start-node} (60%) diff --git a/system/Makefile b/system/Makefile index 37209ab..10f2063 100644 --- a/system/Makefile +++ b/system/Makefile @@ -1,2 +1,16 @@ tags: - find . -name '*.py' | xargs etags + find . '(' -name '*.py' -o -name 'qemu-*' ')' | grep -v '.svn/' | xargs etags + + +LOCAL_RSYNC_EXCLUDES := --exclude '*.pyc' --exclude 'arg*' +RSYNC_EXCLUDES := --exclude .svn --exclude CVS --exclude '*~' --exclude TAGS $(LOCAL_RSYNC_EXCLUDES) +RSYNC_COND_DRY_RUN := $(if $(findstring n,$(MAKEFLAGS)),--dry-run,) +RSYNC := rsync -a -v $(RSYNC_COND_DRY_RUN) $(RSYNC_EXCLUDES) + +sync: +ifeq (,$(RSYNC_URL)) + echo "sync: You must define target host as RSYNC_URL on the command line" + echo " e.g. make sync RSYNC_URL=root@testbox1:workdir" ; exit 1 +else + +$(RSYNC) ./ $(RSYNC_URL)/ +endif diff --git a/system/TestApiserver.py b/system/TestApiserver.py new file mode 100644 index 0000000..2590829 --- /dev/null +++ b/system/TestApiserver.py @@ -0,0 +1,59 @@ +# $Id$ +# wrapper to xmlrpc server, that support dry-run commands +# we dont want to have to depend on PLCAPI, so: +import xmlrpclib + +server_methods = [ ('GetNodes' , []), + ('AddNode' , True), + ('UpdateNode' , True), + ('AddNodeNetwork' , True), + ('GetNodeNetworks' , True), + ('GetNodeNetworkSettingTypes' , True), + ('AddNodeNetworkSettingType' , True), + ('AddNodeNetworkSetting' , True), + ('GetBootMedium' , True), + ('GetNodeGroups' , True), + ('AddNodeGroup' , True), + ('AddNodeToNodeGroup' , True), + ('AddInitScript' , True), + ('AddSite' , True), + ('AddSiteAddress' , True), + ('DeleteSite' , True), + ('DeleteSlice' , True), + ('AddSlice' , True), + ('AddPersonToSlice' , True), + ('AddSliceToNodes' , True), + ('AddSliceAttribute' , True), + ('AddPerson' , True), + ('UpdatePerson' , True), + ('AddRoleToPerson' , True), + ('AddPersonToSite' , True), + ('DeletePerson' , True), + ('AddPersonKey' , True), + ('GetPlcRelease', {'build': {'target-arch':'i386'}}), + ] + +class TestApiserver: + class Callable: + def __init__(self,server,dry_run,method,defaults): + self.server=server + self.dry_run=dry_run + self.method=method + self.defaults=defaults + def __call__ (self, *args): + if self.dry_run: + print "dry_run:",self.method, + if len(args)>0 and type(args[0])==type({}) and args[0].has_key('AuthMethod'): + print '', + args=args[1:] + print '(',args,')' + return self.defaults + else: + actual_method=getattr(self.server,self.method) + return actual_method(*args) + + def __init__(self,url,dry_run=False): + self.apiserver = xmlrpclib.Server(url,allow_none=True) + self.dry_run=dry_run + for (method,defaults) in server_methods: + setattr(self,method,TestApiserver.Callable(self.apiserver,dry_run,method,defaults)) diff --git a/system/TestBox.py b/system/TestBox.py index 5c25fcd..978344c 100644 --- a/system/TestBox.py +++ b/system/TestBox.py @@ -20,18 +20,11 @@ class TestBox: def is_local(self): return self.test_ssh.is_local() - def tar_logs(self): - if os.path.isdir("nodeslogs"): - tar_command="tar cvf nodeslogs.tar nodeslogs/" - self.run_in_buildname (tar_command) - return True - return False - def run_in_buildname (self,command,background=False): utils.header("Running command %s on testbox %s"%(command,self.hostname())) return self.test_ssh.run_in_buildname (command,background) - # should use rsync instead + # xxx could/should use rsync instead def copy (self,local_file,recursive=False): return self.test_ssh.copy (local_file,recursive) @@ -42,10 +35,10 @@ class TestBox: return self.test_ssh.mkdir(dirname) def kill_all_qemus(self): - self.run_in_buildname("template-qemu/kill-qemu-node") + self.run_in_buildname("template-qemu/qemu-kill-node") return True def list_all_qemus(self): - self.run_in_buildname("template-qemu/kill-qemu-node -l") + self.run_in_buildname("template-qemu/qemu-kill-node -l") return True diff --git a/system/TestMain.py b/system/TestMain.py index 09c0f8e..e4dda38 100755 --- a/system/TestMain.py +++ b/system/TestMain.py @@ -10,26 +10,29 @@ from TestPlc import TestPlc from TestSite import TestSite from TestNode import TestNode +SEP='' + class TestMain: subversion_id = "$Id$" default_config = [ 'main' , '1vnodes' , '1testbox64' ] - default_steps = ['uninstall','install','install_rpm', - 'configure', 'start', + default_steps = ['uninstall','install','install_rpm', + 'configure', 'start', SEP, 'store_keys', 'initscripts', - 'sites', 'nodes', 'slices', - 'bootcd', 'nodegroups', - 'kill_all_qemus', 'start_nodes', - 'standby_20', - 'nodes_booted', 'nodes_ssh', 'check_slices','check_initscripts', - 'check_tcp','force_kill_qemus', ] - other_steps = [ 'stop_all_vservers','fresh_install', 'stop', - 'clean_sites', 'clean_nodes', 'clean_slices', 'clean_keys', - 'show_boxes', 'list_all_qemus', 'list_qemus', + 'sites', 'nodes', 'slices', 'nodegroups', SEP, + 'init_node','bootcd', 'configure_qemu', SEP, + 'kill_all_qemus', 'start_nodes', SEP, + 'standby_20', SEP, + 'nodes_booted', 'nodes_ssh', 'check_slices', + 'check_initscripts', 'check_tcp',SEP, + 'force_kill_qemus', 'force_gather_logs' ] + other_steps = [ 'stop_all_vservers','fresh_install', 'cache_rpm', 'stop', SEP, + 'clean_sites', 'clean_nodes', 'clean_slices', 'clean_keys', SEP, + 'show_boxes', 'list_all_qemus', 'list_qemus', SEP, 'db_dump' , 'db_restore', - 'standby_1 through 20', + 'standby_1 through 20', SEP, ] default_build_url = "http://svn.planet-lab.org/svn/build/trunk" @@ -40,7 +43,7 @@ class TestMain: @staticmethod def show_env (options, message): utils.header (message) - utils.pprint("main options",options) + utils.show_options("main options",options) @staticmethod def optparse_list (option, opt, value, parser): @@ -49,9 +52,13 @@ class TestMain: except: setattr(parser.values,option.dest,value.split()) + @staticmethod + def printable_steps (list): + return " ".join(list).replace(" "+SEP+" ","\n") + def run (self): - steps_message="Defaut steps are\n\t%s"%(" ".join(TestMain.default_steps)) - steps_message += "\nOther useful steps are\n\t %s"%(" ".join(TestMain.other_steps)) + steps_message=20*'x'+" Defaut steps are\n"+TestMain.printable_steps(TestMain.default_steps) + steps_message += "\n"+20*'x'+" Other useful steps are\n"+TestMain.printable_steps(TestMain.other_steps) usage = """usage: %%prog [options] steps myplc-url defaults to the last value used, as stored in arg-myplc-url, no default @@ -194,11 +201,14 @@ steps refer to a method in TestPlc or to a step_* module # show config if not self.options.quiet: utils.show_test_spec("Test specifications",all_plc_specs) - # build a TestPlc object from the result + # build a TestPlc object from the result, passing options for spec in all_plc_specs: spec['disabled'] = False all_plcs = [ (x, TestPlc(x,self.options)) for x in all_plc_specs] + # pass options to utils as well + utils.init_options(self.options) + overall_result = True testplc_method_dict = __import__("TestPlc").__dict__['TestPlc'].__dict__ all_step_infos=[] @@ -229,7 +239,6 @@ steps refer to a method in TestPlc or to a step_* module if self.options.dry_run: self.show_env(self.options,"Dry run") - return 0 # do all steps on all plcs for (stepname,method,force) in all_step_infos: diff --git a/system/TestNode.py b/system/TestNode.py index f9f3979..be3faa3 100644 --- a/system/TestNode.py +++ b/system/TestNode.py @@ -62,7 +62,7 @@ class TestNode: userauth = test_user.auth() utils.header("node %s created by user %s"%(self.name(),test_user.name())) rootauth=self.test_plc.auth_root() - server = self.test_plc.server + server = self.test_plc.apiserver server.AddNode(userauth, self.test_site.site_spec['site_fields']['login_base'], self.node_spec['node_fields']) @@ -99,50 +99,58 @@ class TestNode: auth = test_user.auth() except: auth=self.test_plc.auth_root() - self.test_plc.server.DeleteNode(auth,self.name()) + self.test_plc.apiserver.DeleteNode(auth,self.name()) # Do most of the stuff locally - will be pushed on host_box - *not* the plc - later if needed - def prepare_area(self): + def init_node(self): utils.system("rm -rf %s"%self.nodedir()) utils.system("mkdir %s"%self.nodedir()) - #create the tar log file - utils.system("rm -rf nodeslogs && mkdir nodeslogs") - if self.is_qemu(): - utils.system("rsync -v -a --exclude .svn template-qemu/ %s/"%self.nodedir()) + if not self.is_qemu(): + return True + return utils.system("rsync -v -a --exclude .svn template-qemu/ %s/"%self.nodedir())==0 - def create_boot_cd(self): + def bootcd(self): utils.header("Calling GetBootMedium for %s"%self.name()) options = [] if self.is_qemu(): options=['serial'] - encoded=self.test_plc.server.GetBootMedium(self.test_plc.auth_root(), self.name(), 'node-iso', '', options) + encoded=self.test_plc.apiserver.GetBootMedium(self.test_plc.auth_root(), self.name(), 'node-iso', '', options) if (encoded == ''): raise Exception, 'GetBootmedium failed' filename="%s/%s.iso"%(self.nodedir(),self.name()) utils.header('Storing boot medium into %s'%filename) - file(filename,'w').write(base64.b64decode(encoded)) + if self.test_plc.options.dry_run: + print "Dry_run: skipped writing of iso image" + return True + else: + file(filename,'w').write(base64.b64decode(encoded)) + return True def configure_qemu(self): if not self.is_qemu(): return mac=self.node_spec['network_fields']['mac'] hostname=self.node_spec['node_fields']['hostname'] + auth=self.test_plc.auth_root() + target_arch=self.test_plc.apiserver.GetPlcRelease(auth)['build']['target-arch'] conf_filename="%s/qemu.conf"%(self.nodedir()) + if self.test_plc.options.dry_run: + print "dry_run: skipped actual storage of qemu.conf" + return utils.header('Storing qemu config for %s in %s'%(self.name(),conf_filename)) file=open(conf_filename,'w') file.write('MACADDR=%s\n'%mac) file.write('NODE_ISO=%s.iso\n'%self.name()) file.write('HOSTNAME=%s\n'%hostname) + file.write('TARGET_ARCH=%s\n'%target_arch) file.close() # if relevant, push the qemu area onto the host box - if ( not self.test_box().is_local()): - utils.header ("Transferring configuration files for node %s onto %s"%(self.name(),self.host_box())) -# self.test_box().clean_dir(self.buildname()) - self.test_box().mkdir("nodeslogs") - self.test_box().copy(self.nodedir(),recursive=True) - + if self.test_box().is_local(): + return True + utils.header ("Transferring configuration files for node %s onto %s"%(self.name(),self.host_box())) + return self.test_box().copy(self.nodedir(),recursive=True)==0 def start_node (self,options): model=self.node_spec['node_fields']['model'] @@ -156,13 +164,13 @@ class TestNode: test_box = self.test_box() utils.header("Starting qemu node %s on %s"%(self.name(),test_box.hostname())) - test_box.run_in_buildname("qemu-%s/env-qemu start >> nodeslogs/%s.log"%(self.name(),self.name())) + test_box.run_in_buildname("%s/qemu-bridge-init start >> %s/qemu.log"%(self.nodedir(),self.nodedir())) # kick it off in background, as it would otherwise hang - test_box.run_in_buildname("qemu-%s/start-qemu-node 2>&1 >> nodeslogs/%s.log &"%(self.name(),self.name()),True) + test_box.run_in_buildname("%s/qemu-start-node 2>&1 >> %s/qemu.log"%(self.nodedir(),self.nodedir()),True) def list_qemu (self): utils.header("Listing qemu for host %s on box %s"%(self.name(),self.test_box().hostname())) - command="qemu-%s/kill-qemu-node -l %s"%(self.name(),self.name()) + command="qemu-%s/qemu-kill-node -l %s"%(self.name(),self.name()) self.test_box().run_in_buildname(command) return True @@ -173,13 +181,13 @@ class TestNode: utils.header("Failed to get the nodes log files") # kill the right processes utils.header("Stopping qemu for host %s on box %s"%(self.name(),self.test_box().hostname())) - command="qemu-%s/kill-qemu-node %s"%(self.name(),self.name()) + command="qemu-%s/qemu-kill-node %s"%(self.name(),self.name()) self.test_box().run_in_buildname(command) return True def gather_qemu_logs (self): - utils.header("WARNING - Incomplete logs gathering TestNodes.gather_qemu_logs") - - def gather_var_logs (self): - utils.header("WARNING - Incomplete logs gathering TestNodes.gather_var_logs") - + if not self.is_qemu(): + return True + remote_log="%s/qemu.log"%self.nodedir() + local_log="logs/%s-qemu.log"%self.name() + self.test_box().test_ssh.fetch(remote_log,local_log) diff --git a/system/TestPlc.py b/system/TestPlc.py index f2a0dc4..cdcd8d8 100644 --- a/system/TestPlc.py +++ b/system/TestPlc.py @@ -3,7 +3,6 @@ import os, os.path import datetime import time import sys -import xmlrpclib import datetime import traceback from types import StringTypes @@ -17,18 +16,34 @@ from TestSlice import TestSlice from TestSliver import TestSliver from TestBox import TestBox from TestSsh import TestSsh +from TestApiserver import TestApiserver # step methods must take (self) and return a boolean (options is a member of the class) -def standby(minutes): +def standby(minutes,dry_run): utils.header('Entering StandBy for %d mn'%minutes) - time.sleep(60*minutes) + if dry_run: + print 'dry_run' + else: + time.sleep(60*minutes) return True def standby_generic (func): def actual(self): minutes=int(func.__name__.split("_")[1]) - return standby(minutes) + return standby(minutes,self.options.dry_run) + return actual + +def node_mapper (method): + def actual(self): + overall=True + for site_spec in self.plc_spec['sites']: + test_site = TestSite (self,site_spec) + for node_spec in site_spec['nodes']: + test_node = TestNode (self,test_site,node_spec) + node_method = TestNode.__dict__[method.__name__] + if not node_method(test_node): overall=False + return overall return actual class TestPlc: @@ -46,14 +61,14 @@ class TestPlc: self.vserver=False self.url="https://%s:443/PLCAPI/"%plc_spec['hostname'] # utils.header('Using API url %s'%self.url) - self.server=xmlrpclib.Server(self.url,allow_none=True) + self.apiserver=TestApiserver(self.url,options.dry_run) def name(self): name=self.plc_spec['name'] if self.vserver: - return name+"[%s]"%self.vservername + return name+".vserver.%s"%self.vservername else: - return name+"[chroot]" + return name+".chroot" def hostname(self): return self.plc_spec['hostname'] @@ -66,8 +81,11 @@ class TestPlc: def connect (self): pass + def actual_command_in_guest (self,command): + return self.test_ssh.actual_command(self.host_to_guest(command)) + def run_in_guest (self,command): - return self.test_ssh.run(self.host_to_guest(command)) + return utils.system(self.actual_command_in_guest(command)) def run_in_host (self,command): return self.test_ssh.run_in_buildname(command) @@ -97,7 +115,7 @@ class TestPlc: utils.system("scp %s %s@/vservers/%s/%s"%(localfile,self.hostname(),self.vservername,remotefile)) - # xxx quick n dirty + # xxx quick n dirty def run_in_guest_piped (self,local,remote): return utils.system(local+" | "+self.test_ssh.actual_command(self.host_to_guest(remote))) @@ -240,7 +258,7 @@ class TestPlc: # run checkout in any case - would do an update if already exists build_checkout = "svn checkout %s %s"%(self.options.build_url,build_dir) if self.run_in_host(build_checkout) != 0: - raise Exception,"Cannot checkout build dir" + return False # the repo url is taken from myplc-url # with the last two steps (i386/myplc...) removed repo_url = self.options.myplc_url @@ -248,9 +266,7 @@ class TestPlc: repo_url = os.path.dirname(repo_url) create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\ (build_dir,self.vservername,repo_url,self.vserverip) - if self.run_in_host(create_vserver) != 0: - raise Exception,"Could not create vserver for %s"%self.vservername - return True + return self.run_in_host(create_vserver) == 0 def install(self): if self.vserver: @@ -258,25 +274,23 @@ class TestPlc: else: return self.install_chroot() - ### install_rpm - def cache_rpm(self,url,rpm): + ### install_rpm - make this an optional step + def cache_rpm(self): + url = self.options.myplc_url + rpm = os.path.basename(url) cache_fetch="pwd;if [ -f %(rpm)s ] ; then echo Using cached rpm %(rpm)s ; else echo Fetching %(url)s ; curl -O %(url)s; fi"%locals() - id = self.run_in_host(cache_fetch) - if (id != 0): - raise Exception,"Could not get rpm from %s"%url + return self.run_in_host(cache_fetch)==0 def install_rpm_chroot(self): url = self.options.myplc_url rpm = os.path.basename(url) - self.cache_rpm(url,rpm) + if not self.cache_rpm(): + return False utils.header('Installing the : %s'%rpm) - self.run_in_host('rpm -Uvh '+rpm) - self.run_in_host('service plc mount') - return True + return self.run_in_host('rpm -Uvh '+rpm)==0 and self.run_in_host('service plc mount')==0 def install_rpm_vserver(self): - self.run_in_guest("yum -y install myplc-native") - return True + return self.run_in_guest("yum -y install myplc-native")==0 def install_rpm(self): if self.vserver: @@ -395,11 +409,11 @@ class TestPlc: auth=self.auth_root() for (nodegroupname,group_nodes) in groups_dict.iteritems(): try: - self.server.GetNodeGroups(auth,{'name':nodegroupname})[0] + self.apiserver.GetNodeGroups(auth,{'name':nodegroupname})[0] except: - self.server.AddNodeGroup(auth,{'name':nodegroupname}) + self.apiserver.AddNodeGroup(auth,{'name':nodegroupname}) for node in group_nodes: - self.server.AddNodeToNodeGroup(auth,node,nodegroupname) + self.apiserver.AddNodeToNodeGroup(auth,node,nodegroupname) return True def all_hostnames (self) : @@ -411,6 +425,9 @@ class TestPlc: # gracetime : during the first minutes nothing gets printed def do_nodes_booted (self, minutes, gracetime=2): + if self.options.dry_run: + print 'dry_run' + return True # compute timeout timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes) graceout = datetime.datetime.now()+datetime.timedelta(minutes=gracetime) @@ -421,7 +438,7 @@ class TestPlc: status = dict ( [ (hostname,'undef') for hostname in tocheck ] ) while tocheck: # get their status - tocheck_status=self.server.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] ) + tocheck_status=self.apiserver.GetNodes(self.auth_root(), tocheck, ['hostname','boot_state' ] ) # update status for array in tocheck_status: hostname=array['hostname'] @@ -455,23 +472,8 @@ class TestPlc: def nodes_booted(self): return self.do_nodes_booted(minutes=0) -# #to scan and store the nodes's public keys and avoid to ask for confirmation when ssh -# def scan_publicKeys(self,hostnames): -# try: -# temp_knownhosts="/root/known_hosts" -# remote_knownhosts="/root/.ssh/known_hosts" -# self.run_in_host("touch %s"%temp_knownhosts ) -# for hostname in hostnames: -# utils.header("Scan Public %s key and store it in the known_host file(under the root image) "%hostname) -# scan=self.run_in_host('ssh-keyscan -t rsa %s >> %s '%(hostname,temp_knownhosts)) -# #Store the public keys in the right root image -# self.copy_in_guest(temp_knownhosts,remote_knownhosts,True) -# #clean the temp keys file used -# self.run_in_host('rm -f %s '%temp_knownhosts ) -# except Exception, err: -# print err - - def do_check_nodesSsh(self,minutes): + + def do_nodes_ssh(self,minutes): # compute timeout timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes) tocheck = self.all_hostnames() @@ -480,7 +482,7 @@ class TestPlc: while tocheck: for hostname in tocheck: # try to ssh in nodes - node_test_ssh = TestSsh (hostname) + node_test_ssh = TestSsh (hostname,key="/etc/planetlab/root_ssh_key.rsa") access=self.run_in_guest(node_test_ssh.actual_command("date")) if not access: utils.header('The node %s is sshable -->'%hostname) @@ -504,29 +506,31 @@ class TestPlc: return True def nodes_ssh(self): - return self.do_check_nodesSsh(minutes=2) + return self.do_nodes_ssh(minutes=2) - def bootcd (self): - for site_spec in self.plc_spec['sites']: - test_site = TestSite (self,site_spec) - for node_spec in site_spec['nodes']: - test_node=TestNode (self,test_site,node_spec) - test_node.prepare_area() - test_node.create_boot_cd() - test_node.configure_qemu() - return True - + @node_mapper + def init_node (self): pass + @node_mapper + def bootcd (self): pass + @node_mapper + def configure_qemu (self): pass + def do_check_initscripts(self): - for site_spec in self.plc_spec['sites']: - test_site = TestSite (self,site_spec) - test_node = TestNode (self,test_site,site_spec['nodes']) - for slice_spec in self.plc_spec['slices']: - test_slice=TestSlice (self,test_site,slice_spec) - test_sliver=TestSliver(self,test_node,test_slice) - init_status=test_sliver.get_initscript(slice_spec) - if (not init_status): - return False - return init_status + overall = True + for slice_spec in self.plc_spec['slices']: + if not slice_spec.has_key('initscriptname'): + continue + initscript=slice_spec['initscriptname'] + for nodename in slice_spec['nodenames']: + (site,node) = self.locate_node (nodename) + # xxx - passing the wrong site - probably harmless + test_site = TestSite (self,site) + test_slice = TestSlice (self,test_site,slice_spec) + test_node = TestNode (self,test_site,node) + test_sliver = TestSliver (self, test_node, test_slice) + if not test_sliver.check_initscript(initscript): + overall = False + return overall def check_initscripts(self): return self.do_check_initscripts() @@ -534,7 +538,7 @@ class TestPlc: def initscripts (self): for initscript in self.plc_spec['initscripts']: utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript) - self.server.AddInitScript(self.auth_root(),initscript['initscript_fields']) + self.apiserver.AddInitScript(self.auth_root(),initscript['initscript_fields']) return True def slices (self): @@ -573,59 +577,93 @@ class TestPlc: TestSite(self,site_spec).start_nodes (self.options) return True - def gather_all_logs (self): - # (1) get the plc's /var/log and store it locally in logs/-var-log/* - # (2) get all the nodes qemu log and store it as logs/-qemu.log - # (3) get the nodes /var/log and store is as logs/-var-log/* - # (4) as far as possible get the slice's /var/log as logs/--var-log/* - # (1) - self.gather_logs () - # (2) and (3) - for site_spec in self.plc_spec['sites']: - test_site = TestSite (self,site_spec) - for node_spec in site_spec['nodes']: - TestNode(self,test_site,node_spec).gather_qemu_logs() - TestNode(self,test_site,node_spec).gather_var_logs() - return True - def gather_logs (self): - utils.header("WARNING - Incomplete logs gathering TestPlc.gather_logs") + def locate_first_sliver (self): + slice_spec = self.plc_spec['slices'][0] + slicename = slice_spec['slice_fields']['name'] + nodename = slice_spec['nodenames'][0] + return self.locate_sliver_obj(nodename,slicename) + def locate_sliver_obj (self,nodename,slicename): + (site,node) = self.locate_node(nodename) + slice = self.locate_slice (slicename) + # build objects + test_site = TestSite (self, site) + test_node = TestNode (self, test_site,node) + # xxx the slice site is assumed to be the node site - mhh - probably harmless + test_slice = TestSlice (self, test_site, slice) + return TestSliver (self, test_node, test_slice) def check_tcp (self): specs = self.plc_spec['tcp_test'] overall=True for spec in specs: - utils.header ("WARNING : xxx check_tcp is underway, spec=%r"%spec) port = spec['port'] - # locate specs - (s_site,s_node) = self.locate_node(spec['server_node']) - s_slice = self.locate_slice (spec['server_slice']) - # build objects - s_test_site = TestSite (self, s_site) - s_test_node = TestNode (self, s_test_site,s_node) - # xxx the slice site is assumed to be the node site - mhh - s_test_slice = TestSlice (self, s_test_site, s_slice) - s_test_sliver = TestSliver (self, s_test_node, s_test_slice) - if not s_test_sliver.run_tcp_server(port): + # server side + s_test_sliver = self.locate_sliver_obj (spec['server_node'],spec['server_slice']) + if not s_test_sliver.run_tcp_server(port,timeout=10): overall=False break # idem for the client side - (c_site,c_node) = self.locate_node(spec['server_node']) - c_slice = self.locate_slice (spec['server_slice']) - # build objects - c_test_site = TestSite (self, c_site) - c_test_node = TestNode (self, c_test_site,c_node) - # xxx the slice site is assumed to be the node site - mhh - c_test_slice = TestSlice (self, c_test_site, c_slice) - c_test_sliver = TestSliver (self, c_test_node, c_test_slice) + c_test_sliver = self.locate_sliver_obj(spec['server_node'],spec['server_slice']) if not c_test_sliver.run_tcp_client(s_test_node.name(),port): overall=False - return overall - + + def gather_logs (self): + # (1) get the plc's /var/log and store it locally in logs/-var-log/* + # (2) get all the nodes qemu log and store it as logs/-qemu.log + # (3) get the nodes /var/log and store is as logs/-var-log/* + # (4) as far as possible get the slice's /var/log as logs/--var-log/* + # (1) + print "-------------------- TestPlc.gather_logs : PLC's /var/log" + self.gather_var_logs () + # (2) + print "-------------------- TestPlc.gather_logs : nodes's QEMU logs" + for site_spec in self.plc_spec['sites']: + test_site = TestSite (self,site_spec) + for node_spec in site_spec['nodes']: + test_node=TestNode(self,test_site,node_spec) + test_node.gather_qemu_logs() + # (3) + print "-------------------- TestPlc.gather_logs : nodes's /var/log" + self.gather_nodes_var_logs() + # (4) + print "-------------------- TestPlc.gather_logs : sample sliver's /var/log" + self.gather_first_sliver_logs() + return True + + def gather_first_sliver_logs(self): + try: + test_sliver = self.locate_first_sliver() + remote = test_sliver.tar_var_logs() + utils.system("mkdir -p logs/%s-var-log"%test_sliver.name()) + command = remote + " | tar -C logs/%s-var-log -xf -"%test_sliver.name() + utils.system(command) + except Exception,e: + print 'Cannot locate first sliver - giving up',e + return True + + def gather_var_logs (self): + to_plc = self.actual_command_in_guest("tar -cf - /var/log") + command = to_plc + "| tar -C logs/%s-var-log -xf -"%self.name() + utils.system("mkdir -p logs/%s-var-log"%self.name()) + utils.system(command) + + def gather_nodes_var_logs (self): + for site_spec in self.plc_spec['sites']: + test_site = TestSite (self,site_spec) + for node_spec in site_spec['nodes']: + test_node=TestNode(self,test_site,node_spec) + test_ssh = TestSsh (test_node.name(),key="/etc/planetlab/root_ssh_key.rsa") + to_plc = self.actual_command_in_guest ( test_ssh.actual_command("tar -C /var/log -cf - .")) + command = to_plc + "| tar -C logs/%s-var-log -xf -"%test_node.name() + utils.system("mkdir -p logs/%s-var-log"%test_node.name()) + utils.system(command) + + # returns the filename to use for sql dump/restore, using options.dbname if set def dbfile (self, database): # uses options.dbname if it is found diff --git a/system/TestSite.py b/system/TestSite.py index 9a9aa3b..c774908 100644 --- a/system/TestSite.py +++ b/system/TestSite.py @@ -19,9 +19,9 @@ class TestSite: def create_site (self): print self.test_plc.auth_root() - self.test_plc.server.AddSite(self.test_plc.auth_root(), + self.test_plc.apiserver.AddSite(self.test_plc.auth_root(), self.site_spec['site_fields']) - self.test_plc.server.AddSiteAddress(self.test_plc.auth_root(),self.name(), + self.test_plc.apiserver.AddSiteAddress(self.test_plc.auth_root(),self.name(), self.site_spec['address_fields']) def create_users (self): @@ -32,7 +32,7 @@ class TestSite: def delete_site (self): print self.test_plc.auth_root() - self.test_plc.server.DeleteSite(self.test_plc.auth_root(),self.name()) + self.test_plc.apiserver.DeleteSite(self.test_plc.auth_root(),self.name()) return True def delete_users(self): diff --git a/system/TestSlice.py b/system/TestSlice.py index 02d6caa..dc822cc 100644 --- a/system/TestSlice.py +++ b/system/TestSlice.py @@ -29,7 +29,7 @@ class TestSlice: auth = TestUser(self,self.test_site,owner_spec).auth() slice_fields = self.slice_spec['slice_fields'] slice_name = slice_fields['name'] - self.test_plc.server.DeleteSlice(auth,slice_fields['name']) + self.test_plc.apiserver.DeleteSlice(auth,slice_fields['name']) utils.header("Deleted slice %s"%slice_fields['name']) @@ -39,11 +39,11 @@ class TestSlice: slice_fields = self.slice_spec['slice_fields'] slice_name = slice_fields['name'] - self.test_plc.server.AddSlice(auth,slice_fields) + self.test_plc.apiserver.AddSlice(auth,slice_fields) for username in self.slice_spec['usernames']: user_spec=self.test_site.locate_user(username) test_user=TestUser(self,self.test_site,user_spec) - self.test_plc.server.AddPersonToSlice(auth, test_user.name(), slice_name) + self.test_plc.apiserver.AddPersonToSlice(auth, test_user.name(), slice_name) hostnames=[] for nodename in self.slice_spec['nodenames']: @@ -51,11 +51,11 @@ class TestSlice: test_node=TestNode(self,self.test_site,node_spec) hostnames += [test_node.name()] utils.header("Adding %r in %s"%(hostnames,slice_name)) - self.test_plc.server.AddSliceToNodes(auth, slice_name, hostnames) + self.test_plc.apiserver.AddSliceToNodes(auth, slice_name, hostnames) if self.slice_spec.has_key('initscriptname'): isname=self.slice_spec['initscriptname'] utils.header("Adding initscript %s in %s"%(isname,slice_name)) - self.test_plc.server.AddSliceAttribute(self.test_plc.auth_root(), slice_name,'initscript',isname) + self.test_plc.apiserver.AddSliceAttribute(self.test_plc.auth_root(), slice_name,'initscript',isname) # def clear_known_hosts (self): # utils.header("Messing with known_hosts for slice %s"%self.name()) @@ -68,10 +68,10 @@ class TestSlice: # #scan public key and update the known_host file in the root image # self.test_plc.scan_publicKeys(hostnames) - def locate_key(self,slice_spec): + def locate_key(self): # locate the first avail. key found=False - for username in slice_spec['usernames']: + for username in self.slice_spec['usernames']: user_spec=self.test_site.locate_user(username) for keyname in user_spec['keynames']: key_spec=self.test_plc.locate_key(keyname) @@ -81,26 +81,20 @@ class TestSlice: keyname=test_key.name() if os.path.isfile(publickey) and os.path.isfile(privatekey): found=True - #create dir in plc root image - remote_privatekey="/root/keys/%s.rsa"%keyname - if not os.path.isfile(remote_privatekey): - self.test_plc.run_in_guest("mkdir -p /root/keys" ) - self.test_plc.copy_in_guest(privatekey,remote_privatekey,True) - - return (found,remote_privatekey) + return (found,privatekey) - def do_check_slice(self,minutes,options): + def do_check_slice(self,options,minutes=3): # self.clear_known_hosts() timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes) # locate a key - slice_spec = self.slice_spec - (found,remote_privatekey)=self.locate_key(slice_spec) + (found,remote_privatekey)=self.locate_key() if not found : utils.header("WARNING: Cannot find a valid key for slice %s"%self.name()) return False # convert nodenames to real hostnames + slice_spec = self.slice_spec restarted=[] tocheck=[] for nodename in slice_spec['nodenames']: @@ -109,13 +103,13 @@ class TestSlice: while tocheck: for hostname in tocheck: - (site_spec,node_spec) = self.test_plc.locate_host(hostname) + (site_spec,node_spec) = self.test_plc.locate_hostname(hostname) date_test_ssh = TestSsh (hostname,key=remote_privatekey,username=self.name()) # this can be ran locally as we have the key utils.header('Trying to enter into slice %s@%s'%(self.name(),hostname)) date = date_test_ssh.run("date") if not date: - utils.header("Successfuly entered slice %s on %s"%self.name(),hostname) + utils.header("Successfuly entered slice %s on %s"%(self.name(),hostname)) tocheck.remove(hostname) else: # real nodes will have been checked once in case they're up - skip if not diff --git a/system/TestSliver.py b/system/TestSliver.py index 5fd181a..f4d4be6 100644 --- a/system/TestSliver.py +++ b/system/TestSliver.py @@ -10,95 +10,41 @@ class TestSliver: self.test_plc=test_plc self.test_node=test_node self.test_slice=test_slice - self.test_ssh =TestSsh(self.test_plc.test_ssh) + self.test_ssh = self.create_test_ssh() - def is_local(self): - return self.test_ssh.is_local() - - def host_to_guest(self,command): - return self.test_plc.host_to_guest(self.test_plc.vserver,self.test_plc.vservername,command) - - def get_privateKey(self,slice_spec): + def get_privateKey(self): + slice_spec=self.test_slice.slice_spec try: - (found,remote_privatekey)=self.test_slice.locate_key(slice_spec) - return (found,remote_privatekey) + (found,privatekey)=self.test_slice.locate_key() + return (found,privatekey) except Exception,e: print str(e) - def get_initscript(self,slice_spec): - (found,remote_privatekey)=self.get_privateKey(slice_spec) - if not found : + def create_test_ssh(self): + (found,privatekey) = self.get_privateKey() + if not found: raise Exception,"Cannot find a valid key for slice %s"%self.test_slice.name() - for hostname in slice_spec['nodenames']: - utils.header("Checking initiscript %s on the slice %s@%s" - %(slice_spec['initscriptname'],self.test_slice.name(),hostname)) - init_file=self.test_plc.run_in_guest('ssh -i %s %s@%s ls -l /tmp/init* ' - %(remote_privatekey,self.test_slice.name(),hostname)) - if ( init_file): - return False - - return True + return TestSsh (self.test_node.name(),key=privatekey,username=self.test_slice.name(), + # so that copies end up in the home dir + buildname=".") + + def name (self): + return "%s@%s"%(self.test_slice.name(),self.test_node.name()) + + def check_initscript(self,initscript): + slice_spec=self.test_slice.slice_spec + initscript = slice_spec['initscriptname'] + utils.header("Checking initscript %s on sliver %s"%(initscript,self.name())) + return self.test_ssh.run("ls -l /tmp/%s.stamp"%initscript)==0 - def run_tcp_server (self,port): - print ("XXX run_tcp_server not implemented") - return True - def run_tcp_client (self,hostname,port): - print ("XXX run_tcp_client not implemented") - return True - - def run_tcpcheck(self,peer_spec,remote_privatekey): - if peer_spec['peer_name']=="server": - slice_ssh = TestSsh (peer_spec['server_name'], - key=remote_privatekey, - username=peer_spec['slice_name']) - remote_server_command=slice_ssh.actual_command("./tcptest.py server -t 10") - return self.test_plc.run_in_guest(remote_server_command) - - else: - slice_ssh = TestSsh (peer_spec['client_name'], - key=remote_privatekey, - username=peer_spec['slice_name']) - client_command="/tcptest.py client -a %s -p %d"%(peer_spec['peer_server'], - peer_spec['server_port']) - remote_client_command=slice_ssh.actual_command(client_command) - return self.test_plc.run_in_guest(remote_client_command) - - def do_check_tcp(self,tcp_param,options): - for tcp_spec in tcp_param: - #copy the tcptest file under the chroot - localfile=remotefile="tcptest.py" - self.test_plc.copy_in_guest(localfile, remotefile, False) - peer_param=tcp_spec['tcp_fields'] - if (tcp_spec['tcp_fields']['peer_name']=='server'): - #server instruction - utils.header("Transfert the tcp script to the server at %s@%s"%(peer_param['slice_name'], - peer_param['server_name'])) - slice_spec=self.test_slice.get_slice(peer_param['slice_name']) - (found,remote_privatekey)=self.get_privateKey(slice_spec) - cp_server_command="scp -i %s ./tcptest.py %s@%s:"%(remote_privatekey,peer_param['slice_name'], - peer_param['server_name']) - self.test_plc.run_in_guest(cp_server_command) - serv_status=self.run_tcpcheck(peer_param,remote_privatekey) - if (serv_status): - utils.header("FAILED to check loop Connexion on the %s server side"%peer_param['server_name']) - return False - else: - #Client instruction - utils.header("Transfert the tcp script to the client at %s@%s" %(peer_param['slice_name'], - peer_param['client_name'])) - slice_spec=self.test_slice.get_slice(peer_param['slice_name']) - (found,remote_privatekey)=self.get_privateKey(slice_spec) - cp_client_command="scp -i %s ./tcptest.py %s@%s:"%(remote_privatekey, peer_param['slice_name'], - peer_param['client_name']) - self.test_plc.run_in_guest(cp_client_command) - client_status=self.run_tcpcheck(peer_param,remote_privatekey) - if ( serv_status): - utils.header("FAILED to Contact the server %s from the client side %s"%(peer_param['peer_server'], - peer_param['client_name'])) - return False - - - self.test_plc.run_in_guest("rm -rf tcptest.py") - return True + def run_tcp_server (self,port,timeout=10): + server_command = "tcptest.py server -p %d -t %d"%(port,timeout) + return self.test_ssh.copy("tcptest.py")==0 and self.test_ssh.run(server_command)==0 + def run_tcp_client (self,servername,port): + client_command="tcptest.py client -a %s -p %d"%(servername,port) + return self.test_ssh.copy("tcptest.py")==0 and self.test_ssh.run(client_command)==0 + def tar_var_logs (self): + return self.test_ssh.actual_command("sudo tar -C /var/log -cf - .") + diff --git a/system/TestSsh.py b/system/TestSsh.py index 396d4b6..b4cc398 100644 --- a/system/TestSsh.py +++ b/system/TestSsh.py @@ -126,4 +126,15 @@ class TestSsh: scp_command += "%s %s:%s/%s"%(local_file,self.hostname_part(), self.buildname,os.path.basename(local_file) or ".") return utils.system(scp_command) - + + def fetch (self, remote_file, local_file, recursive=False): + if self.is_local(): + command="cp " + if recursive: command += "-r " + command += "%s %s"%(remote_file,local_file) + else: + command="scp " + if recursive: command += "-r " + command += self.key_part() + command += "%s:%s/%s %s"%(self.hostname_part(),self.buildname,remote_file,local_file) + utils.system(command) diff --git a/system/TestUser.py b/system/TestUser.py index 9cf7886..8170601 100644 --- a/system/TestUser.py +++ b/system/TestUser.py @@ -26,24 +26,24 @@ class TestUser: fields=user_spec['user_fields'] auth=self.test_plc.auth_root() utils.header('Adding user %s - roles %r'%(fields['email'],user_spec['roles'])) - self.test_plc.server.AddPerson(auth,fields) - self.test_plc.server.UpdatePerson(auth,fields['email'],{'enabled': True}) + self.test_plc.apiserver.AddPerson(auth,fields) + self.test_plc.apiserver.UpdatePerson(auth,fields['email'],{'enabled': True}) for role in user_spec['roles']: - self.test_plc.server.AddRoleToPerson(auth,role,fields['email']) - self.test_plc.server.AddPersonToSite(auth, + self.test_plc.apiserver.AddRoleToPerson(auth,role,fields['email']) + self.test_plc.apiserver.AddPersonToSite(auth, self.name(), self.test_site.name()) def delete_user(self): auth=self.test_plc.auth_root() - self.test_plc.server.DeletePerson(auth,self.name()) + self.test_plc.apiserver.DeletePerson(auth,self.name()) def add_keys (self): user_spec=self.user_spec for keyname in user_spec['keynames']: key=self.test_plc.locate_key(keyname) auth=self.auth() - self.test_plc.server.AddPersonKey(auth,self.name(), key['key_fields']) + self.test_plc.apiserver.AddPersonKey(auth,self.name(), key['key_fields']) diff --git a/system/config_main.py b/system/config_main.py index 307447b..e0236b5 100644 --- a/system/config_main.py +++ b/system/config_main.py @@ -15,13 +15,13 @@ def nodes(options): 'owner' : 'pi', 'network_fields': { 'method':'static', 'type':'ipv4', - 'ip':'xx-deferred-xxx', - 'gateway':'xx-deferred-xxx', - 'network':'xx-deferred-xxx', - 'broadcast':'xx-deferred-xxx', - 'netmask':'xx-deferred-xxx', - 'dns1': 'xx-deferred-xxx', - 'dns2': 'xx-deferred-xxx', + 'ip':'xxx-deferred-xxx', + 'gateway':'xxx-deferred-xxx', + 'network':'xxx-deferred-xxx', + 'broadcast':'xxx-deferred-xxx', + 'netmask':'xxx-deferred-xxx', + 'dns1': 'xxx-deferred-xxx', + 'dns2': 'xxx-deferred-xxx', }, }, {'name':'node2', @@ -31,13 +31,13 @@ def nodes(options): 'owner' : 'pi', 'network_fields': { 'method':'static', 'type':'ipv4', - 'ip':'xx-deferred-xxx', - 'gateway':'xx-deferred-xxx', - 'network':'xx-deferred-xxx', - 'broadcast':'xx-deferred-xxx', - 'netmask':'xx-deferred-xxx', - 'dns1': 'xx-deferred-xxx', - 'dns2': 'xx-deferred-xxx', + 'ip':'xxx-deferred-xxx', + 'gateway':'xxx-deferred-xxx', + 'network':'xxx-deferred-xxx', + 'broadcast':'xxx-deferred-xxx', + 'netmask':'xxx-deferred-xxx', + 'dns1': 'xxx-deferred-xxx', + 'dns2': 'xxx-deferred-xxx', }, }, ] @@ -144,14 +144,16 @@ def keys (options): ] def initscripts(options): - initscripts= [ { 'initscript_fields' : { 'enabled' : True, - 'name':'script1', - 'script' : '#! /bin/sh\n (echo Starting test initscript: Stage 1; date) > /tmp/initscript1.log \n ', - }}, - { 'initscript_fields' : { 'enabled' : True, - 'name':'script2', - 'script' : '#! /bin/sh\n (echo Starting test initscript: Stage 2; date) > /tmp/initscript2.log \n ', - }}, + initscripts= [ { 'initscript_fields' : + { 'enabled' : True, + 'name':'script1', + 'script' : '#! /bin/sh\n (echo Starting test initscript: Stage 1; date) > /tmp/script1.stamp \n ', + }}, + { 'initscript_fields' : + { 'enabled' : True, + 'name':'script2', + 'script' : '#! /bin/sh\n (echo Starting test initscript: Stage 2; date) > /tmp/script2.stamp \n ', + }}, ] if options.small_test: return [initscripts[0]] @@ -218,7 +220,7 @@ def plc (options) : return { 'name' : 'onetest', # as of yet, not sure we can handle foreign hosts, but this is required though - 'hostname' : 'xx-deferred-xxx', + 'hostname' : 'xxx-deferred-xxx', # set these two items to run within a vserver # 'vservername': '138.96.250.131' # 'vserverip': '138.96.250.131' @@ -232,8 +234,8 @@ def plc (options) : 'PLC_API_HOST' : 'test.one-lab.org', 'PLC_WWW_HOST' : 'test.one-lab.org', 'PLC_BOOT_HOST' : 'test.one-lab.org', - 'PLC_NET_DNS1' : 'xx-deferred-xxx', - 'PLC_NET_DNS2' : 'xx-deferred-xxx', + 'PLC_NET_DNS1' : 'xxx-deferred-xxx', + 'PLC_NET_DNS2' : 'xxx-deferred-xxx', 'sites' : sites(options), 'keys' : keys(options), 'initscripts': initscripts(options), diff --git a/system/template-qemu/env-qemu b/system/template-qemu/env-qemu deleted file mode 100755 index 18b8e72..0000000 --- a/system/template-qemu/env-qemu +++ /dev/null @@ -1,167 +0,0 @@ -#!/bin/bash - - -# Establishment of a runtime environment for a -# virtual machine under QEMU, This script permits -# to the virtual machine to share the -# network connection with a host machine under FC6. - -#Author: Amine chaoui - -# Default Value -IP_GATEWAY=0.0.0.0 -IP_HOST=0.0.0.0 -IP_BROADCAST=0.0.0.0 -IP_NETMASK=0.0.0.0 - -INTERFACE_LAN=eth0 -INTERFACE_BRIDGE=br0 -brctl=/usr/sbin/brctl - -# Fonction de mise en place du pont -start () { - #if we have already configured the same host_box no need to do it again - set $(/sbin/ifconfig | grep $INTERFACE_BRIDGE) >/dev/null - if [ -n "$1" ]; then - echo "Interface bridge $INTERFACE_BRIDGE already exist." - exit 0 - fi - if [ -n "$1" ]; then - INTERFACE_LAN=$1 - shift - fi - set $(/sbin/ifconfig | grep $INTERFACE_LAN) >/dev/null - if [ -z "$1" ]; then - echo "Interface réseau $IF_HOTE non trouvée." - exit 1 - fi - shift $(($# - 1)) - echo "Using the interface" $INTERFACE_LAN - - #Restarting the udev - echo "Starting the udev ..." - /sbin/udevd restart - echo "Starting the kqemu patch module ..." - modprobe kqemu - #Loding the tun/tap model - echo "Loading the kqemu patch module ..." - modprobe kqemu - #Loding the tun/tap model - echo "Loading the tun module ..." - modprobe tun - set $(lsmod | grep tun) >/dev/null - if [ -z "$1" ]; then - echo "Module tun/tap not activated" - exit 1 - fi - shift $(($# - 1)) - - #Giving acces in Read/Write to the tun module - echo "Granting the Read/Write acces to the tun module..." - chmod 666 /dev/net/tun - - - ##Get The BROADCAST ip @ - set $(/sbin/ip addr show $INTERFACE_LAN | grep inet) >/dev/null 2>&1 - if [ -n "$2" ]; then - IP_BROADCAST=$4 - fi - shift $(($# - 1)) - - #Getting the GATEWAY IP @ - set $(netstat -rn | grep UG ) >/dev/null 2>&1 - if [ -n "$2" ]; then - IP_GATEWAY=$2 - fi - shift $(($# - 1)) - - #Getting the host IP - set $(ifconfig $INTERFACE_LAN 2> /dev/null | grep "inet addr:" | \ - sed -e "s/.*addr:\([^ ]*\).*/\1/") - if [ -n "$1" ]; then - IP_HOST=$1 - fi - shift $(($# - 1)) - - ##Getting the Netmask address - set $(ifconfig $INTERFACE_LAN 2> /dev/null | grep "inet addr:" | \ - sed -e "s/.*Mask:\([^ ]*\).*/\1/") - if [ -n "$1" ]; then - IP_NETMASK=$1 - fi - shift $(($# - 1)) - #check if the module brctl is there - if [ ! -e $brctl ] ; then - echo "Module Brctl '/usr/sbin/brctl' is not found" - exit 0 - fi - # Création et paramétrage du pont - echo "Configure $INTERFACE_BRIDGE bridge..." - $brctl addbr $INTERFACE_BRIDGE - #/usr/sbin/brctl stp $INTERFACE_BRIDGE yes - $brctl addif $INTERFACE_BRIDGE $INTERFACE_LAN - echo "Activating promiscuous mode $INTERFACE_LAN..." - /sbin/ifconfig $INTERFACE_LAN 0.0.0.0 promisc up - sleep 2 - echo "IP address on $INTERFACE_BRIDGE..." - # static - /sbin/ifconfig $INTERFACE_BRIDGE $IP_HOST broadcast $IP_BROADCAST netmask $IP_NETMASK up - sleep 1 - - #Reconfigure the Bridge IP @ in the host machine - echo "Configuring the IP Gateway @:" $IP_GATEWAY - route add default gw $IP_GATEWAY - - - #wipe the host firewall otherwise the guest qemu can't acces to the LAN - echo "Wiping the firewall..." - iptables -F - -} - - -#Adding a new interface to the bridge -add () { - /sbin/ifconfig $1 0.0.0.0 promisc up - /usr/sbin/brctl addif $INTERFACE_BRIDGE $1 -} - - -#Stop the actual bridged network and Restore the original network -stop () { - if [ -n "$1" ]; then - INTERFACE_LAN=$1 - fi - TESTPONT=$(/sbin/ifconfig | grep $INTERFACE_BRIDGE) - if [ -z "$TESTPONT" ]; then - echo "Attention : pont réseau non trouvé. Vérifier la config réseau ..." - exit 1 - fi - #check if the module brctl is there - if [ ! -e $brctl ] ;then - echo "Module Brctl '/usr/sbin/brctl' is not found" - exit 0 - fi - $brctl delif $INTERFACE_BRIDGE $INTERFACE_LAN - /sbin/ifconfig $INTERFACE_BRIDGE down - $brctl delbr $INTERFACE_BRIDGE - /sbin/service network restart -} - -# -case $1 in - start) - start $2 - ;; - stop) - stop $2 - ;; - add) - add $2 - ;; - *) - echo $"Use: env-qemu {start|add|stop} [interface]" - exit 1 -esac - -exit 0 diff --git a/system/template-qemu/qemu-bridge-init b/system/template-qemu/qemu-bridge-init new file mode 100755 index 0000000..879d1c2 --- /dev/null +++ b/system/template-qemu/qemu-bridge-init @@ -0,0 +1,139 @@ +#!/bin/bash + +# Establishment of a runtime environment for a +# virtual machine under QEMU, This script allows the host box +# to share its network connection with qemu-based guests +# +# Author: Amine chaoui +# + +COMMAND=$(basename $0) + +# constant +INTERFACE_BRIDGE=br0 +# Default Value +INTERFACE_LAN=eth0 + +# Fonction de mise en place du pont +start () { + + # take extra arg for ifname, if provided + [ -n "$1" ] && { INTERFACE_LAN=$1; shift ; } + + ### Checking + type -p brctl &> /dev/null || { echo "brctl not found, please install bridge-utils" ; exit 1 ; } + + #if we have already configured the same host_box no need to do it again + /sbin/ifconfig $INTERFACE_BRIDGE &> /dev/null && { + echo "Interface bridge $INTERFACE_BRIDGE already exist." + exit 0 + } + /sbin/ifconfig $INTERFACE_LAN &>/dev/null || { + echo "Cannot use interface $INTERFACE_LAN - exiting" + exit 1 + } + + #Getting host IP/masklen + address=$(/sbin/ip addr show $INTERFACE_LAN | grep -v inet6 | grep inet | awk '{print $2;}') + [ -z "$address" ] && { echo "ERROR: Could not determine IP address for $INTERFACE_LAN" ; exit 1 ; } + + broadcast=$(/sbin/ip addr show $INTERFACE_LAN | grep -v inet6 | grep inet | awk '{print $4;}') + [ -z "$broadcast" ] && echo "WARNING: Could not determine broadcast address for $INTERFACE_LAN" + + gateway=$(netstat -rn | grep '^0.0.0.0' | awk '{print $2;}') + [ -z "$gateway" ] && echo "WARNING: Could not determine gateway IP" + + ### do it + #Restarting udev + echo "Starting udev ..." + /sbin/udevd restart + if modprobe kqemu ; then + echo "kqemu loadded" + else + echo "WARNING : Could not modprobe kqemu" + fi + #Loding the tun/tap model + if modprobe tun ; then + echo "tun loaded" + # Giving read/write access + echo "Granting read/write acces to the tun device" + chmod 666 /dev/net/tun + else + echo "Could not modprobe tun - exiting" + exit 1 + fi + + # creating the bridge + echo "Creating bridge $INTERFACE_BRIDGE" + brctl addbr $INTERFACE_BRIDGE + #brctl stp $INTERFACE_BRIDGE yes + brctl addif $INTERFACE_BRIDGE $INTERFACE_LAN + echo "Activating promiscuous mode $INTERFACE_LAN..." + /sbin/ifconfig $INTERFACE_LAN 0.0.0.0 promisc up + sleep 2 + echo "Setting bridge $address $broadcast" + # static + /sbin/ifconfig $INTERFACE_BRIDGE $address broadcast $broadcast up + sleep 1 + + #Reconfigure the routing table + echo "Configuring the IP Gateway @:" $gateway + route add default gw $gateway + + # xxx this is *wrong* + # wipe the host firewall otherwise the guest qemu can't access the LAN + echo "Wiping the firewall..." + iptables -F + +} + +#Adding a new interface to the bridge: this is used by qemu-ifup +add () { + + [[ -z "$@" ]] && { echo "Usage: $COMMAND add ifname" ; exit 1 ; } + INTERFACE_LAN=$1; shift + + echo "Activating link for $INTERFACE_LAN..." + /sbin/ip link set $INTERFACE_LAN up + sleep 1 + echo "Adding $INTERFACE_BRIDGE to $INTERFACE_BRIDGE" + brctl addif $INTERFACE_BRIDGE $INTERFACE_LAN + # putting a virtual interface in promisc mode seems like an odd thing to do + # echo "Activating promiscuous mode on $INTERFACE_LAN ..." + # /sbin/ifconfig $INTERFACE_LAN 0.0.0.0 promisc up + # sleep 1 + echo "Done." + +} + +#Stop the bridge and restore the original setting +stop () { + # take extra arg for ifname, if provided + [ -n "$1" ] && { INTERFACE_LAN=$1; shift ; } + + ### Checking + type -p brctl &> /dev/null || { echo "brctl not found, please install bridge-utils" ; exit 1 ; } + + /sbin/ifconfig $INTERFACE_BRIDGE &> /dev/null || { + echo "Interface bridge $INTERFACE_BRIDGE does not exist." + exit 0 + } + brctl delif $INTERFACE_BRIDGE $INTERFACE_LAN + /sbin/ifconfig $INTERFACE_BRIDGE down + brctl delbr $INTERFACE_BRIDGE + /sbin/service network restart +} + +# +case "$1" in + start) + shift; start "$@" ;; + stop) + shift; stop "$@" ;; + add) + shift; add "$@" ;; + *) + echo $"Usage: env-qemu {start|add|stop} [interface]" ; exit 1 ;; +esac + +exit 0 diff --git a/system/template-qemu/qemu-ifup b/system/template-qemu/qemu-ifup index b14f2f3..877496e 100755 --- a/system/template-qemu/qemu-ifup +++ b/system/template-qemu/qemu-ifup @@ -1,19 +1,5 @@ #!/bin/bash -# Mise en place d'un environnement d'exécution d'une machine -# virtuelle sous QEMU pour partage de la connexion réseau -# avec une une machine hôte sous FC6. +# use the bridge as set up by qemu-bridge-init start -INTERFACE_LAN=eth0 -INTERFACE_BRIDGE=br0 - -echo "Configure $1 interface..." -echo "Activating link for $1..." -/sbin/ip link set $1 up -sleep 1 -echo "Configure $INTERFACE_BRIDGE bridge..." -/usr/sbin/brctl addif $INTERFACE_BRIDGE $1 -echo "Activating promiscuous mode on $1 ..." -/sbin/ifconfig $1 0.0.0.0 promisc up -sleep 1 -echo "Done." +qemu-bridge-init add $1 diff --git a/system/template-qemu/kill-qemu-node b/system/template-qemu/qemu-kill-node similarity index 94% rename from system/template-qemu/kill-qemu-node rename to system/template-qemu/qemu-kill-node index 0ac501e..830d33d 100755 --- a/system/template-qemu/kill-qemu-node +++ b/system/template-qemu/qemu-kill-node @@ -31,6 +31,8 @@ function kill_from_file () { echo Killing $pid kill $pid mv $file $file.killed + else + echo "Could not find file $file - that qemu was killed already" fi } diff --git a/system/template-qemu/start-qemu-node b/system/template-qemu/qemu-start-node similarity index 60% rename from system/template-qemu/start-qemu-node rename to system/template-qemu/qemu-start-node index b138531..8d8b079 100755 --- a/system/template-qemu/start-qemu-node +++ b/system/template-qemu/qemu-start-node @@ -3,7 +3,8 @@ # cd in this command's directory cd $(dirname $0) -#Getting the env. as passed by the test framework - mostly MACADDR and NODE_ISO +#Getting the env. as passed by the test framework +# expected vars are MACADDR, NODE_ISO, HOSTNAME and TARGET_ARCH CONFIG=qemu.conf if [ ! -e ${CONFIG} ];then echo "File for node_iso version not found" @@ -11,10 +12,17 @@ if [ ! -e ${CONFIG} ];then fi . $CONFIG -#default Value -#always use the 64 bit version of qemu, as this will work on both 32 & 64 bit host kernels +# the launcher, depending on local/target archs +archs="$(uname-i)+$(TARGET_ARCH)" +case $archs in + i386+i386) QEMU= qemu;; + i386+x86_64) QEMU= qemu-system-x86_64;; + x86_64+i386) QEMU= qemu;; + x86_64+x86_64) QEMU= qemu-system-x86_64;; +esac + QEMU=qemu -HDA=hda_5.img +HDA=hard_drive.img RAM=1024 SCRIPT=qemu-ifup @@ -25,8 +33,8 @@ TAP="tap,script=$SCRIPT" if [ -f $HDA ] ; then echo "Using $HDA" else - echo "Creating hard disk for Qemu install under $HDA" - img=$(qemu-img create -f qcow2 $HDA 5G) + echo "Creating hard disk (10G) for Qemu install under $HDA" + img=$(qemu-img create -f qcow2 $HDA 10G) if [ -z "$img" ];then echo "Can't Create disk image..." exit 1 diff --git a/system/utils.py b/system/utils.py index 85ad7ab..7aa670f 100644 --- a/system/utils.py +++ b/system/utils.py @@ -4,6 +4,12 @@ import os import re from pprint import PrettyPrinter +options={} + +def init_options(options_arg): + global options + options=options_arg + # how could this accept a list again ? def header(message): now=time.strftime("%H:%M:%S", time.localtime()) @@ -14,6 +20,17 @@ def pprint(message,spec,depth=2): print ">",now,"--",message PrettyPrinter(indent=8,depth=depth).pprint(spec) +# quick & dirty - should probably use the parseroption object instead +# and move to TestMain as well +exclude_options_keys = [ 'ensure_value' , 'read_file', 'read_module' ] +def show_options (message,options): + now=time.strftime("%H:%M:%S", time.localtime()) + print ">",now,"--",message + for k in dir(options): + if k.find("_")==0: continue + if k in exclude_options_keys: continue + print " ",k,":",getattr(options,k) + def show_site_spec (site): print '======== site',site['site_fields']['name'] for (k,v) in site.iteritems(): @@ -94,8 +111,11 @@ def show_test_spec_pass (plc_spec,passno): def system(command,background=False): now=time.strftime("%H:%M:%S", time.localtime()) if background: command += " &" - print "+",now,':',command - return os.system("set -x; " + command) + if options.dry_run: + print 'dry_run:',command + return 0 + else: + return os.system("set -x; " + command) def match (string, pattern): # tmp - there's probably much simpler -- 2.43.0