From d09cbd5b5d9d02ceadf6f09c446b17c8c1cb6f5e Mon Sep 17 00:00:00 2001 From: Thierry Parmentelat Date: Tue, 18 Nov 2008 12:24:25 +0000 Subject: [PATCH] * new interactive option * nodes IP addresses saved * config display now a separate step * default config has 2 nodes again * onelab config reviewed, using more testboxes --- system/TestMain.py | 45 ++++++++---- system/TestMapper.py | 12 +-- system/TestNode.py | 10 +-- system/TestPlc.py | 5 +- system/config_1testbox.py | 17 ++++- system/config_1vnodes.py | 26 +++---- system/config_1vservers.py | 12 +-- system/config_default.py | 8 +- system/config_main.py | 8 +- system/config_wifilab.py | 14 ++-- system/utils.py | 145 ++++++++++++++++++++----------------- 11 files changed, 169 insertions(+), 133 deletions(-) diff --git a/system/TestMain.py b/system/TestMain.py index c99fe85..b0db61f 100755 --- a/system/TestMain.py +++ b/system/TestMain.py @@ -5,6 +5,7 @@ import sys, os, os.path from optparse import OptionParser import traceback from time import strftime +import readline import utils from TestPlc import TestPlc @@ -45,7 +46,7 @@ build-url defaults to the last value used, as stored in arg-build-url, or %s config defaults to the last value used, as stored in arg-config, or %r -node-ips and plc-ips defaults to the last value used, as stored in arg-node-ips and arg-plc-ips, +node-ips and plc-ips defaults to the last value used, as stored in arg-ips-node and arg-ips-plc, default is to use IP scanning steps refer to a method in TestPlc or to a step_* module === @@ -72,10 +73,10 @@ steps refer to a method in TestPlc or to a step_* module help="Run all default steps") parser.add_option("-l","--list",action="store_true",dest="list_steps", default=False, help="List known steps") - parser.add_option("-N","--nodes",action="callback", callback=TestMain.optparse_list, dest="node_ips", + parser.add_option("-N","--nodes",action="callback", callback=TestMain.optparse_list, dest="ips_node", nargs=1,type="string", help="Specify the set of IP addresses to use for nodes (scanning disabled)") - parser.add_option("-P","--plcs",action="callback", callback=TestMain.optparse_list, dest="plc_ips", + parser.add_option("-P","--plcs",action="callback", callback=TestMain.optparse_list, dest="ips_plc", nargs=1,type="string", help="Specify the set of IP addresses to use for plcs (scanning disabled)") parser.add_option("-1","--small",action="store_true",dest="small_test",default=False, @@ -86,6 +87,8 @@ steps refer to a method in TestPlc or to a step_* module help="Run in verbose mode") parser.add_option("-q","--quiet", action="store_true", dest="quiet", default=False, help="Run in quiet mode") + parser.add_option("-i","--interactive",action="store_true",dest="interactive",default=False, + help="prompts before each step") parser.add_option("-n","--dry-run", action="store_true", dest="dry_run", default=False, help="Show environment and exits") parser.add_option("-r","--restart-nm", action="store_true", dest="forcenm", default=False, @@ -113,8 +116,8 @@ steps refer to a method in TestPlc or to a step_* module # handle defaults and option persistence for (recname,filename,default) in ( ('build_url','arg-build-url',TestMain.default_build_url) , - ('node_ips','arg-node-ips',[]) , - ('plc_ips','arg-plc-ips',[]) , + ('ips_node','arg-ips-node',[]) , + ('ips_plc','arg-ips-plc',[]) , ('config','arg-config',TestMain.default_config) , ('arch_rpms_url','arg-arch-rpms-url',"") , ('personality','arg-personality',"linux32"), @@ -195,16 +198,22 @@ steps refer to a method in TestPlc or to a step_* module traceback.print_exc() print 'Cannot load config %s -- ignored'%modulename raise - # show config - if not self.options.quiet: - utils.show_test_spec("Test specifications",all_plc_specs) # remember plc IP address(es) if not specified - current=file('arg-plc-ips').read() + current=file('arg-ips-plc').read() if not current: - plc_ips_file=open('arg-plc-ips','w') + ips_plc_file=open('arg-ips-plc','w') for plc_spec in all_plc_specs: - plc_ips_file.write("%s\n"%plc_spec['PLC_API_HOST']) - plc_ips_file.close() + ips_plc_file.write("%s\n"%plc_spec['PLC_API_HOST']) + ips_plc_file.close() + # ditto for nodes + current=file('arg-ips-node').read() + if not current: + ips_node_file=open('arg-ips-node','w') + for plc_spec in all_plc_specs: + for site_spec in plc_spec['sites']: + for node_spec in site_spec['nodes']: + ips_node_file.write("%s\n"%node_spec['node_fields']['hostname']) + ips_node_file.close() # build a TestPlc object from the result, passing options for spec in all_plc_specs: spec['disabled'] = False @@ -268,7 +277,17 @@ steps refer to a method in TestPlc or to a step_* module # run the step time=strftime("%Y-%m-%d-%H-%M") - if not spec['disabled'] or force: + if not spec['disabled'] or force or self.options.interactive: + if self.options.interactive: + msg="Run step %s on %s [y]/n/q ? "%(stepname,plcname) + answer=raw_input(msg).strip().lower() or "y" + answer=answer[0] + if answer in ['n']: + print '%s on %s skipped'%(stepname,plcname) + continue + elif answer in ['q','b']: + print 'Exiting' + return try: force_msg="" if force: force_msg=" (forced)" diff --git a/system/TestMapper.py b/system/TestMapper.py index d60cf05..13a0780 100644 --- a/system/TestMapper.py +++ b/system/TestMapper.py @@ -26,6 +26,8 @@ class TestMapper: def apply_first_map (self, type, name, obj, maplist): for (map_pattern,rename_dict) in maplist: + if self.options.verbose: + print 'dbg: TestMapper/',type,'name=',name,'& pattern=',map_pattern if utils.match (name,map_pattern): utils.header("TestMapper/%s : applying rules '%s' on %s"%(type,map_pattern,name)) for (k,v) in rename_dict.iteritems(): @@ -62,14 +64,8 @@ class TestMapper: def map (self,mapper): - try: - plc_maps = mapper['plc'] - except: - plc_maps = [] - try: - node_maps = mapper['node'] - except: - node_maps = [] + plc_maps = mapper.get('plc',[]) + node_maps = mapper.get('node',[]) for plc in self.plcs: name=TestMapper.plc_name(plc) diff --git a/system/TestNode.py b/system/TestNode.py index 7f682c0..08dd17c 100644 --- a/system/TestNode.py +++ b/system/TestNode.py @@ -71,16 +71,16 @@ class TestNode: server.UpdateNode(userauth, self.name(), {'boot_state':'reinstall'}) # populate network interfaces - primary server.AddInterface(userauth,self.name(), - self.node_spec['network_fields']) + self.node_spec['interface_fields']) # populate network interfaces - others if self.node_spec.has_key('extra_interfaces'): for interface in self.node_spec['extra_interfaces']: server.AddInterface(userauth,self.name(), - interface['network_fields']) + interface['interface_fields']) if interface.has_key('settings'): for (attribute,value) in interface['settings'].iteritems(): # locate node network - nn = server.GetInterfaces(userauth,{'ip':interface['network_fields']['ip']})[0] + nn = server.GetInterfaces(userauth,{'ip':interface['interface_fields']['ip']})[0] nnid=nn['interface_id'] # locate or create node network attribute type try: @@ -137,9 +137,9 @@ class TestNode: def configure_qemu(self): if not self.is_qemu(): return - mac=self.node_spec['network_fields']['mac'] + mac=self.node_spec['interface_fields']['mac'] hostname=self.node_spec['node_fields']['hostname'] - ip=self.node_spec['network_fields']['ip'] + ip=self.node_spec['interface_fields']['ip'] auth=self.test_plc.auth_root() target_arch=self.test_plc.apiserver.GetPlcRelease(auth)['build']['target-arch'] conf_filename="%s/qemu.conf"%(self.nodedir()) diff --git a/system/TestPlc.py b/system/TestPlc.py index 734ee7c..f5ea38e 100644 --- a/system/TestPlc.py +++ b/system/TestPlc.py @@ -62,7 +62,7 @@ SEP='' class TestPlc: - default_steps = ['uninstall','install','install_rpm', + default_steps = ['display','uninstall','install','install_rpm', 'configure', 'start', 'fetch_keys', SEP, 'store_keys', 'clear_known_hosts', 'initscripts', SEP, 'sites', 'nodes', 'slices', 'nodegroups', SEP, @@ -267,6 +267,9 @@ class TestPlc: node.kill_qemu() return True + def display (self): + utils.show_plc_spec (self.plc_spec) + return True ### utility methods for handling the pool of IP addresses allocated to plcs # Logic diff --git a/system/config_1testbox.py b/system/config_1testbox.py index aa44e7d..1a7f37e 100644 --- a/system/config_1testbox.py +++ b/system/config_1testbox.py @@ -7,9 +7,15 @@ from TestMapper import TestMapper def config (plcs, options): if options.arch == "i386": - target = 'testbox32.onelab.eu' + testbox1 = 'testbox32.onelab.eu' + testbox2 = 'testbox32.onelab.eu' + target=testbox1 elif options.arch == "x86_64": - target = 'testbox64.onelab.eu' +# testbox1 = 'testbox64-1.onelab.eu' +# testbox2 = 'testbox64-2.onelab.eu' + testbox1 = 'testbox64.onelab.eu' + testbox2 = 'estran.inria.fr' + target=testbox1 else: print 'Unsupported arch %s'%options.arch sys.exit(1) @@ -19,8 +25,11 @@ def config (plcs, options): 'PLC_API_HOST':target, 'PLC_BOOT_HOST':target, 'PLC_WWW_HOST':target, - 'name':'%s-'+options.arch } ) ], - 'node': [ ('*' , {'host_box': target } ) ], + 'name':'%s-'+options.arch } ) + ], + 'node': [ ('deferred01' , {'host_box': testbox1 } ), + ('deferred02' , {'host_box': testbox2 } ), + ], } return TestMapper(plcs,options).map(mapper) diff --git a/system/config_1vnodes.py b/system/config_1vnodes.py index 2474a75..48862a4 100644 --- a/system/config_1vnodes.py +++ b/system/config_1vnodes.py @@ -6,12 +6,12 @@ from TestPool import TestPool onelab_plcs_pool = [ ( 'vnode%02d.inria.fr'%i, '138.96.255.%d'%(220+i), '02:34:56:00:00:%02d'%i) for i in range(1,10) ] site_dict = { - 'network_fields:gateway':'138.96.248.250', - 'network_fields:network':'138.96.0.0', - 'network_fields:broadcast':'138.96.255.255', - 'network_fields:netmask':'255.255.0.0', - 'network_fields:dns1': '138.96.0.10', - 'network_fields:dns2': '138.96.0.11', + 'interface_fields:gateway':'138.96.248.250', + 'interface_fields:network':'138.96.0.0', + 'interface_fields:broadcast':'138.96.255.255', + 'interface_fields:netmask':'255.255.0.0', + 'interface_fields:dns1': '138.96.0.10', + 'interface_fields:dns2': '138.96.0.11', } def config (plcs, options): @@ -22,21 +22,21 @@ def config (plcs, options): all_nodenames = test_mapper.node_names() maps = [] for nodename in all_nodenames: - if len(options.node_ips) != 0: - ip=options.node_ips[0] - options.node_ips=options.node_ips[1:] + if len(options.ips_node) != 0: + ip=options.ips_node[0] + options.ips_node=options.ips_node[1:] (hostname,ip,mac)=test_pool.locate(ip) else: (hostname,ip,mac) = test_pool.next_free() node_dict= {'node_fields:hostname':hostname, - 'network_fields:ip':ip, - 'network_fields:mac':mac, + 'interface_fields:ip':ip, + 'interface_fields:mac':mac, } node_dict.update(site_dict) maps.append ( ( nodename, node_dict) ) - plc_map = [ ( '*' , { 'PLC_NET_DNS1' : site_dict [ 'network_fields:dns1' ], - 'PLC_NET_DNS2' : site_dict [ 'network_fields:dns2' ], } ) ] + plc_map = [ ( '*' , { 'PLC_NET_DNS1' : site_dict [ 'interface_fields:dns1' ], + 'PLC_NET_DNS2' : site_dict [ 'interface_fields:dns2' ], } ) ] return test_mapper.map ({'node': maps, 'plc' : plc_map } ) diff --git a/system/config_1vservers.py b/system/config_1vservers.py index 8713096..6e63fe3 100644 --- a/system/config_1vservers.py +++ b/system/config_1vservers.py @@ -15,16 +15,16 @@ def config (plcs,options): test_pool = TestPool (onelab_plcs_pool,options) - if len(options.plc_ips) != 0: - utils.header('Using user-provided IPS:\nplc_ips=%r'%options.plc_ips) - options.plc_ips.reverse() + if len(options.ips_plc) != 0: + utils.header('Using user-provided IPS:\nips_plc=%r'%options.ips_plc) + options.ips_plc.reverse() plc_counter=0 for plc in plcs: try: - if len(options.plc_ips) != 0: - ip=options.plc_ips[0] - options.plc_ips=options.plc_ips[1:] + if len(options.ips_plc) != 0: + ip=options.ips_plc[0] + options.ips_plc=options.ips_plc[1:] (hostname,ip,mac)=test_pool.locate(ip) utils.header("Using user-provided %s %s for plc %s"%( hostname,ip,plc['name'])) diff --git a/system/config_default.py b/system/config_default.py index e9082b4..08051e9 100644 --- a/system/config_default.py +++ b/system/config_default.py @@ -3,16 +3,12 @@ import utils def config (plc_specs, options): -# tmp : force small test - utils.header("XXX WARNING : forcing small tests in config_default") - options.small_test = True - import config_main plcs = config_main.config([],options) - import config_1vnodes - plcs = config_1vnodes.config(plcs,options) import config_1testbox plcs = config_1testbox.config (plcs,options) + import config_1vnodes + plcs = config_1vnodes.config(plcs,options) import config_1vservers plcs = config_1vservers.config (plcs,options) diff --git a/system/config_main.py b/system/config_main.py index b4e9aa2..41aeac4 100644 --- a/system/config_main.py +++ b/system/config_main.py @@ -11,10 +11,10 @@ def nodes(options): nodes= [{'name':'node1', 'node_fields': {'hostname': 'deferred01', 'model':'qemu/minhw', } , - 'host_box': 'testbox1.onelab.eu', + 'host_box': 'deferred-testbox01', 'owner' : 'pi', 'nodegroups' : 'mynodegroup', - 'network_fields': { 'method':'static', + 'interface_fields': { 'method':'static', 'type':'ipv4', 'ip':'xxx-deferred-xxx', 'gateway':'xxx-deferred-xxx', @@ -28,9 +28,9 @@ def nodes(options): {'name':'node2', 'node_fields': {'hostname': 'deferred02', 'model':'qemu/minhw', } , - 'host_box': 'testbox1.onelab.eu', + 'host_box': 'deferred-testbox02', 'owner' : 'pi', - 'network_fields': { 'method':'static', + 'interface_fields': { 'method':'static', 'type':'ipv4', 'ip':'xxx-deferred-xxx', 'gateway':'xxx-deferred-xxx', diff --git a/system/config_wifilab.py b/system/config_wifilab.py index 3fe7e0d..0aafdcf 100644 --- a/system/config_wifilab.py +++ b/system/config_wifilab.py @@ -38,8 +38,8 @@ def nodes(): 'node_fields': {'hostname': 'wlab02.inria.fr', 'model':'Dell Latitude 830'}, 'owner' : 'pi', 'nodegroups' : 'wifi', - 'network_fields': { 'method':'dhcp', 'type' : 'ipv4', 'ip':'138.96.250.162',}, - 'extra_interfaces' : [ { 'network_fields' : { 'method' : 'dhcp', + 'interface_fields': { 'method':'dhcp', 'type' : 'ipv4', 'ip':'138.96.250.162',}, + 'extra_interfaces' : [ { 'interface_fields' : { 'method' : 'dhcp', 'type' : 'ipv4', 'mac' : '00:1B:77:70:F4:C6', 'ip' : '138.96.250.192', }, @@ -52,8 +52,8 @@ def nodes(): 'node_fields': {'hostname': 'wlab17.inria.fr', 'model':'Dell Latitude 830'}, 'owner' : 'pi', 'nodegroups' : ['wifi','x86_64'] , - 'network_fields': { 'method':'dhcp', 'type' : 'ipv4', 'ip':'138.96.250.177',}, - 'extra_interfaces' : [ { 'network_fields' : { 'method' : 'dhcp', + 'interface_fields': { 'method':'dhcp', 'type' : 'ipv4', 'ip':'138.96.250.177',}, + 'extra_interfaces' : [ { 'interface_fields' : { 'method' : 'dhcp', 'type' : 'ipv4', 'mac' : '00:1c:bf:51:3c:19', 'ip' : '138.96.250.207',}, @@ -66,8 +66,8 @@ def nodes(): 'node_fields': {'hostname': 'wlab05.inria.fr', 'model':'Dell Latitude 830'}, 'owner' : 'pi', 'nodegroups' : 'wifi', - 'network_fields': { 'method':'dhcp', 'type' : 'ipv4', 'ip':'138.96.250.165',}, - 'extra_interfaces' : [ { 'network_fields' : { 'method' : 'static', + 'interface_fields': { 'method':'dhcp', 'type' : 'ipv4', 'ip':'138.96.250.165',}, + 'extra_interfaces' : [ { 'interface_fields' : { 'method' : 'static', 'type' : 'ipv4', 'mac' : '00:1B:77:70:FC:84', 'ip' : '138.96.250.215', @@ -80,7 +80,7 @@ def nodes(): 'settings' : { 'essid' : 'guest-inria-sophia', 'ifname' : 'wlan0',}, }, - { 'network_fields' : { 'method' : 'dhcp', + { 'interface_fields' : { 'method' : 'dhcp', 'type' : 'ipv4', 'mac' : '00:20:A6:4E:FF:E6', 'ip' : '138.96.250.50', diff --git a/system/utils.py b/system/utils.py index 42eb93b..75250ef 100644 --- a/system/utils.py +++ b/system/utils.py @@ -18,6 +18,43 @@ def pprint(message,spec,depth=2): print ">",now,"--",message PrettyPrinter(indent=8,depth=depth).pprint(spec) + + +def system(command,background=False): + if background: command += " &" + if options.dry_run: + print 'dry_run:',command + return 0 + else: + return os.system("set -x; " + command) + +### WARNING : this ALWAYS does its job, even in dry_run mode +def output_of (command): + import commands +# if options.dry_run: +# print 'dry_run',command +# return (0,'[[dry-run - fake output]]') +# else: + (code,string) = commands.getstatusoutput(command) + return (code,string) + + + +# convenience: translating shell-like pattern into regexp +def match (string, pattern): + # tmp - there's probably much simpler + # rewrite * into .*, ? into . + pattern=pattern.replace("*",".*") + pattern=pattern.replace("?",".") + return re.compile(pattern).match(string) + +def locate_sanity_scripts (message,path,extensions): + print message,'searching',path,'for extensions',extensions + scripts=[] + for ext in extensions: + scripts += glob.glob (path+'/*.'+ext) + return scripts + # quick & dirty - should probably use the parseroption object instead # and move to TestMain as well exclude_options_keys = [ 'ensure_value' , 'read_file', 'read_module' ] @@ -29,27 +66,56 @@ def show_options (message,options): if k in exclude_options_keys: continue print " ",k,":",getattr(options,k) + + +#################### display config +# entry point +def show_plc_spec (plc_spec): + show_plc_spec_pass (plc_spec,1) + show_plc_spec_pass (plc_spec,2) + +def show_plc_spec_pass (plc_spec,passno): + for (key,val) in plc_spec.iteritems(): + if passno == 2: + if key == 'sites': + for site in val: + show_site_spec(site) + for node in site['nodes']: + show_node_spec(node) + elif key=='initscripts': + for initscript in val: + show_initscript_spec (initscript) + elif key=='slices': + for slice in val: + show_slice_spec (slice) + elif key=='keys': + for key in val: + show_key_spec (key) + elif passno == 1: + if key not in ['sites','initscripts','slices','keys']: + print '* ',key,':',val + def show_site_spec (site): print '* ======== site',site['site_fields']['name'] for (k,v) in site.iteritems(): if k=='nodes': if v: - print '* \t\t','nodes : ', + print '* ','nodes : ', for node in v: print node['node_fields']['hostname'],'', print '' elif k=='users': if v: - print '* \t\tusers : ', + print '* users : ', for user in v: print user['name'],'', print '' elif k == 'site_fields': - print '* \t\tlogin_base',':',v['login_base'] + print '* login_base',':',v['login_base'] elif k == 'address_fields': pass else: - print '* \t\t',k, + print '* ',k, PrettyPrinter(indent=8,depth=2).pprint(v) def show_initscript_spec (initscript): @@ -63,79 +129,26 @@ def show_slice_spec (slice): for (k,v) in slice.iteritems(): if k=='nodenames': if v: - print '* \t\tnodes : ', + print '* nodes : ', for nodename in v: print nodename,'', print '' elif k=='usernames': if v: - print '* \t\tusers : ', + print '* users : ', for username in v: print username,'', print '' elif k=='slice_fields': - print '* \t\tfields',':', + print '* fields',':', print 'max_nodes=',v['max_nodes'], print '' else: - print '* \t\t',k,v - -def show_test_spec (message,all_plc_specs): - now=time.strftime("%H:%M:%S", time.localtime()) - print "*",now,"--",message - for plc_spec in all_plc_specs: - show_test_spec_pass (plc_spec,1) - show_test_spec_pass (plc_spec,2) - -def show_test_spec_pass (plc_spec,passno): - for (key,val) in plc_spec.iteritems(): - if passno == 2: - if key == 'sites': - for site in val: - show_site_spec(site) - elif key=='initscripts': - for initscript in val: - show_initscript_spec (initscript) - elif key=='slices': - for slice in val: - show_slice_spec (slice) - elif key=='keys': - for key in val: - show_key_spec (key) - elif passno == 1: - if key not in ['sites','initscripts','slices','keys']: - print '* \t',key,':',val - -def system(command,background=False): - if background: command += " &" - if options.dry_run: - print 'dry_run:',command - return 0 - else: - return os.system("set -x; " + command) - -### WARNING : this ALWAYS does its job, even in dry_run mode -def output_of (command): - import commands -# if options.dry_run: -# print 'dry_run',command -# return (0,'[[dry-run - fake output]]') -# else: - (code,string) = commands.getstatusoutput(command) - return (code,string) + print '* ',k,v -# convenience: translating shell-like pattern into regexp -def match (string, pattern): - # tmp - there's probably much simpler - # rewrite * into .*, ? into . - pattern=pattern.replace("*",".*") - pattern=pattern.replace("?",".") - return re.compile(pattern).match(string) - -def locate_sanity_scripts (message,path,extensions): - print message,'searching',path,'for extensions',extensions - scripts=[] - for ext in extensions: - scripts += glob.glob (path+'/*.'+ext) - return scripts +def show_node_spec (node): + print "* node",node['name'],"host_box=",node['host_box'], + print "hostname=",node['node_fields']['hostname'], + print "ip=",node['interface_fields']['ip'] + -- 2.43.0