From eb4d7e9f9e7f77d6f2e49aab70dd715a43708230 Mon Sep 17 00:00:00 2001 From: Thierry Parmentelat Date: Wed, 2 Apr 2008 19:27:42 +0000 Subject: [PATCH] various changes all over the place --- system/TestMain.py | 22 +++- system/TestMapper.py | 13 +-- system/TestNode.py | 7 ++ system/TestPlc.py | 238 ++++++++++++++++++++++++------------------ system/TestSlice.py | 112 ++++++++++---------- system/TestSliver.py | 28 +++-- system/TestSsh.py | 40 +++++-- system/config_main.py | 44 ++++---- system/ssh_config | 4 - system/utils.py | 11 +- 10 files changed, 305 insertions(+), 214 deletions(-) delete mode 100644 system/ssh_config diff --git a/system/TestMain.py b/system/TestMain.py index 93a61c8..09c0f8e 100755 --- a/system/TestMain.py +++ b/system/TestMain.py @@ -14,11 +14,11 @@ class TestMain: subversion_id = "$Id$" - default_config = [ 'onelab' ] + default_config = [ 'main' , '1vnodes' , '1testbox64' ] default_steps = ['uninstall','install','install_rpm', 'configure', 'start', - 'clear_ssh_config','store_keys', 'initscripts', + 'store_keys', 'initscripts', 'sites', 'nodes', 'slices', 'bootcd', 'nodegroups', 'kill_all_qemus', 'start_nodes', @@ -27,7 +27,7 @@ class TestMain: 'check_tcp','force_kill_qemus', ] other_steps = [ 'stop_all_vservers','fresh_install', 'stop', 'clean_sites', 'clean_nodes', 'clean_slices', 'clean_keys', - 'list_all_qemus', 'list_qemus', 'stop_nodes' , + 'show_boxes', 'list_all_qemus', 'list_qemus', 'db_dump' , 'db_restore', 'standby_1 through 20', ] @@ -73,6 +73,9 @@ steps refer to a method in TestPlc or to a step_* module parser.add_option("-c","--config",action="callback", callback=TestMain.optparse_list, dest="config", nargs=1,type="string", help="Config module - can be set multiple times, or use quotes") + parser.add_option("-x","--exclude",action="callback", callback=TestMain.optparse_list, dest="exclude", + nargs=1,type="string",default=[], + help="steps to exclude - can be set multiple times, or use quotes") parser.add_option("-a","--all",action="store_true",dest="all_steps", default=False, help="Run all default steps") parser.add_option("-l","--list",action="store_true",dest="list_steps", default=False, @@ -160,6 +163,17 @@ steps refer to a method in TestPlc or to a step_* module #self.options.steps=['dump','clean','install','populate'] self.options.steps=TestMain.default_steps + # exclude + selected=[] + for step in self.options.steps: + keep=True + for exclude in self.options.exclude: + if utils.match(step,exclude): + keep=False + break + if keep: selected.append(step) + self.options.steps=selected + # this is useful when propagating on host boxes, to avoid conflicts self.options.buildname = os.path.basename (os.path.abspath (self.path)) @@ -228,7 +242,7 @@ steps refer to a method in TestPlc or to a step_* module force_msg="" if force: force_msg=" (forced)" utils.header("********** RUNNING step %s%s on plc %s"%(stepname,force_msg,plcname)) - step_result = method(obj,self.options) + step_result = method(obj) if step_result: utils.header('********** SUCCESSFUL step %s on %s'%(stepname,plcname)) else: diff --git a/system/TestMapper.py b/system/TestMapper.py index 87ec6c4..f7933ba 100644 --- a/system/TestMapper.py +++ b/system/TestMapper.py @@ -8,7 +8,6 @@ # see an example in config_onelab_testbox32.py # -import re import utils class TestMapper: @@ -17,11 +16,6 @@ class TestMapper: self.plcs=plcs self.options=options - @staticmethod - def match (name,key): - key=key.replace("*",".*") - return re.compile(key).match(name) - @staticmethod def plc_name (plc): return plc['name'] @@ -32,8 +26,8 @@ class TestMapper: def apply_first_map (self, type, name, obj, maplist): for (map_pattern,rename_dict) in maplist: - if TestMapper.match (name,map_pattern): - utils.header("TestMapper/%s : applying match key %s on plc %s"%(type,map_pattern,name)) + if utils.match (name,map_pattern): + utils.header("TestMapper/%s : applying rules '%s' on %s"%(type,map_pattern,name)) for (k,v) in rename_dict.iteritems(): # apply : separator path=k.split(':') @@ -53,8 +47,7 @@ class TestMapper: # apply formatting if '%s' found in the value if v.find('%s')>=0: v=v%obj[k] - if self.options.verbose: - utils.header("mapping %s->%s towards %s"%(name,k,v)) + utils.header("TestMapper, rewriting %s: %s into %s"%(name,k,v)) o[step]=v # only apply first rule return diff --git a/system/TestNode.py b/system/TestNode.py index 2bc232a..f9f3979 100644 --- a/system/TestNode.py +++ b/system/TestNode.py @@ -176,3 +176,10 @@ class TestNode: command="qemu-%s/kill-qemu-node %s"%(self.name(),self.name()) self.test_box().run_in_buildname(command) return True + + def gather_qemu_logs (self): + utils.header("WARNING - Incomplete logs gathering TestNodes.gather_qemu_logs") + + def gather_var_logs (self): + utils.header("WARNING - Incomplete logs gathering TestNodes.gather_var_logs") + diff --git a/system/TestPlc.py b/system/TestPlc.py index b7da2f8..f2a0dc4 100644 --- a/system/TestPlc.py +++ b/system/TestPlc.py @@ -18,7 +18,7 @@ from TestSliver import TestSliver from TestBox import TestBox from TestSsh import TestSsh -# step methods must take (self, options) and return a boolean +# step methods must take (self) and return a boolean (options is a member of the class) def standby(minutes): utils.header('Entering StandBy for %d mn'%minutes) @@ -26,7 +26,7 @@ def standby(minutes): return True def standby_generic (func): - def actual(self,options): + def actual(self): minutes=int(func.__name__.split("_")[1]) return standby(minutes) return actual @@ -135,6 +135,12 @@ class TestPlc: return key raise Exception,"Cannot locate key %s"%keyname + def locate_slice (self, slicename): + for slice in self.plc_spec['slices']: + if slice['slice_fields']['name'] == slicename: + return slice + raise Exception,"Cannot locate slice %s"%slicename + # all different hostboxes used in this plc def gather_hostBoxes(self): # maps on sites and nodes, return [ (host_box,test_node) ] @@ -155,28 +161,27 @@ class TestPlc: return result # a step for checking this stuff - def showboxes (self,options): - print 'showboxes' + def show_boxes (self): for (box,nodes) in self.gather_hostBoxes().iteritems(): print box,":"," + ".join( [ node.name() for node in nodes ] ) return True # make this a valid step - def kill_all_qemus(self,options): + def kill_all_qemus(self): for (box,nodes) in self.gather_hostBoxes().iteritems(): # this is the brute force version, kill all qemus on that host box - TestBox(box,options.buildname).kill_all_qemus() + TestBox(box,self.options.buildname).kill_all_qemus() return True # make this a valid step - def list_all_qemus(self,options): + def list_all_qemus(self): for (box,nodes) in self.gather_hostBoxes().iteritems(): # this is the brute force version, kill all qemus on that host box - TestBox(box,options.buildname).list_all_qemus() + TestBox(box,self.options.buildname).list_all_qemus() return True # kill only the right qemus - def list_qemus(self,options): + def list_qemus(self): for (box,nodes) in self.gather_hostBoxes().iteritems(): # the fine-grain version for node in nodes: @@ -184,57 +189,47 @@ class TestPlc: return True # kill only the right qemus - def kill_qemus(self,options): + def kill_qemus(self): for (box,nodes) in self.gather_hostBoxes().iteritems(): # the fine-grain version for node in nodes: node.kill_qemu() return True - def clear_ssh_config (self,options): - # install local ssh_config file as root's .ssh/config - ssh should be quiet - # dir might need creation first - self.run_in_guest("mkdir /root/.ssh") - self.run_in_guest("chmod 700 /root/.ssh") - # this does not work - > redirection somehow makes it until an argument to cat - #self.run_in_guest_piped("cat ssh_config","cat > /root/.ssh/config") - self.copy_in_guest("ssh_config","/root/.ssh/config",True) - return True - #################### step methods ### uninstall - def uninstall_chroot(self,options): + def uninstall_chroot(self): self.run_in_host('service plc safestop') #####detecting the last myplc version installed and remove it self.run_in_host('rpm -e myplc') ##### Clean up the /plc directory - self.run_in_host('rm -rf /plc/data') + self.run_in_host('rm -rf /plc/data') ##### stop any running vservers self.run_in_host('for vserver in $(ls -d /vservers/* | sed -e s,/vservers/,,) ; do case $vserver in vtest*) echo Shutting down vserver $vserver ; vserver $vserver stop ;; esac ; done') return True - def uninstall_vserver(self,options): + def uninstall_vserver(self): self.run_in_host("vserver --silent %s delete"%self.vservername) return True - def uninstall(self,options): + def uninstall(self): # if there's a chroot-based myplc running, and then a native-based myplc is being deployed # it sounds safer to have the former uninstalled too # now the vserver method cannot be invoked for chroot instances as vservername is required if self.vserver: - self.uninstall_vserver(options) - self.uninstall_chroot(options) + self.uninstall_vserver() + self.uninstall_chroot() else: - self.uninstall_chroot(options) + self.uninstall_chroot() return True ### install - def install_chroot(self,options): + def install_chroot(self): # nothing to do return True - def install_vserver(self,options): + def install_vserver(self): # we need build dir for vtest-init-vserver if self.is_local(): # a full path for the local calls @@ -243,12 +238,12 @@ class TestPlc: # use a standard name - will be relative to HOME build_dir="options.buildname" # run checkout in any case - would do an update if already exists - build_checkout = "svn checkout %s %s"%(options.build_url,build_dir) + build_checkout = "svn checkout %s %s"%(self.options.build_url,build_dir) if self.run_in_host(build_checkout) != 0: raise Exception,"Cannot checkout build dir" # the repo url is taken from myplc-url # with the last two steps (i386/myplc...) removed - repo_url = options.myplc_url + repo_url = self.options.myplc_url for level in [ 'rpmname','arch' ]: repo_url = os.path.dirname(repo_url) create_vserver="%s/vtest-init-vserver.sh %s %s -- --interface eth0:%s"%\ @@ -257,11 +252,11 @@ class TestPlc: raise Exception,"Could not create vserver for %s"%self.vservername return True - def install(self,options): + def install(self): if self.vserver: - return self.install_vserver(options) + return self.install_vserver() else: - return self.install_chroot(options) + return self.install_chroot() ### install_rpm def cache_rpm(self,url,rpm): @@ -270,7 +265,7 @@ class TestPlc: if (id != 0): raise Exception,"Could not get rpm from %s"%url - def install_rpm_chroot(self,options): + def install_rpm_chroot(self): url = self.options.myplc_url rpm = os.path.basename(url) self.cache_rpm(url,rpm) @@ -279,18 +274,18 @@ class TestPlc: self.run_in_host('service plc mount') return True - def install_rpm_vserver(self,options): + def install_rpm_vserver(self): self.run_in_guest("yum -y install myplc-native") return True - def install_rpm(self,options): + def install_rpm(self): if self.vserver: - return self.install_rpm_vserver(options) + return self.install_rpm_vserver() else: - return self.install_rpm_chroot(options) + return self.install_rpm_chroot() ### - def configure(self,options): + def configure(self): tmpname='%s.plc-config-tty'%(self.name()) fileconf=open(tmpname,'w') for var in [ 'PLC_NAME', @@ -314,14 +309,14 @@ class TestPlc: return True # the chroot install is slightly different to this respect - def start(self, options): + def start(self): if self.vserver: self.run_in_guest('service plc start') else: self.run_in_host('service plc start') return True - def stop(self, options): + def stop(self): if self.vserver: self.run_in_guest('service plc stop') else: @@ -329,21 +324,21 @@ class TestPlc: return True # could use a TestKey class - def store_keys(self, options): + def store_keys(self): for key_spec in self.plc_spec['keys']: TestKey(self,key_spec).store_key() return True - def clean_keys(self, options): + def clean_keys(self): utils.system("rm -rf %s/keys/"%os.path(sys.argv[0])) - def sites (self,options): - return self.do_sites(options) + def sites (self): + return self.do_sites() - def clean_sites (self,options): - return self.do_sites(options,action="delete") + def clean_sites (self): + return self.do_sites(action="delete") - def do_sites (self,options,action="add"): + def do_sites (self,action="add"): for site_spec in self.plc_spec['sites']: test_site = TestSite (self,site_spec) if (action != "add"): @@ -358,12 +353,12 @@ class TestPlc: test_site.create_users() return True - def nodes (self, options): - return self.do_nodes(options) - def clean_nodes (self, options): - return self.do_nodes(options,action="delete") + def nodes (self): + return self.do_nodes() + def clean_nodes (self): + return self.do_nodes(action="delete") - def do_nodes (self, options,action="add"): + def do_nodes (self,action="add"): for site_spec in self.plc_spec['sites']: test_site = TestSite (self,site_spec) if action != "add": @@ -382,7 +377,7 @@ class TestPlc: # create nodegroups if needed, and populate # no need for a clean_nodegroups if we are careful enough - def nodegroups (self, options): + def nodegroups (self): # 1st pass to scan contents groups_dict = {} for site_spec in self.plc_spec['sites']: @@ -457,40 +452,42 @@ class TestPlc: # only useful in empty plcs return True - def nodes_booted(self,options): + def nodes_booted(self): return self.do_nodes_booted(minutes=0) - #to scan and store the nodes's public keys and avoid to ask for confirmation when ssh - def scan_publicKeys(self,hostnames): - try: - temp_knownhosts="/root/known_hosts" - remote_knownhosts="/root/.ssh/known_hosts" - self.run_in_host("touch %s"%temp_knownhosts ) - for hostname in hostnames: - utils.header("Scan Public %s key and store it in the known_host file(under the root image) "%hostname) - scan=self.run_in_host('ssh-keyscan -t rsa %s >> %s '%(hostname,temp_knownhosts)) - #Store the public keys in the right root image - self.copy_in_guest(temp_knownhosts,remote_knownhosts,True) - #clean the temp keys file used - self.run_in_host('rm -f %s '%temp_knownhosts ) - except Exception, err: - print err +# #to scan and store the nodes's public keys and avoid to ask for confirmation when ssh +# def scan_publicKeys(self,hostnames): +# try: +# temp_knownhosts="/root/known_hosts" +# remote_knownhosts="/root/.ssh/known_hosts" +# self.run_in_host("touch %s"%temp_knownhosts ) +# for hostname in hostnames: +# utils.header("Scan Public %s key and store it in the known_host file(under the root image) "%hostname) +# scan=self.run_in_host('ssh-keyscan -t rsa %s >> %s '%(hostname,temp_knownhosts)) +# #Store the public keys in the right root image +# self.copy_in_guest(temp_knownhosts,remote_knownhosts,True) +# #clean the temp keys file used +# self.run_in_host('rm -f %s '%temp_knownhosts ) +# except Exception, err: +# print err def do_check_nodesSsh(self,minutes): # compute timeout timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes) tocheck = self.all_hostnames() - self.scan_publicKeys(tocheck) +# self.scan_publicKeys(tocheck) utils.header("checking Connectivity on nodes %r"%tocheck) while tocheck: for hostname in tocheck: # try to ssh in nodes - access=self.run_in_guest('ssh -o StrictHostKeyChecking=no -o BatchMode=yes -i /etc/planetlab/root_ssh_key.rsa root@%s date'%hostname ) - if (not access): + node_test_ssh = TestSsh (hostname) + access=self.run_in_guest(node_test_ssh.actual_command("date")) + if not access: utils.header('The node %s is sshable -->'%hostname) # refresh tocheck tocheck.remove(hostname) else: + # we will have tried real nodes once, in case they're up - but if not, just skip (site_spec,node_spec)=self.locate_hostname(hostname) if TestNode.is_real_model(node_spec['node_fields']['model']): utils.header ("WARNING : check ssh access into real node %s - skipped"%hostname) @@ -506,10 +503,10 @@ class TestPlc: # only useful in empty plcs return True - def nodes_ssh(self, options): + def nodes_ssh(self): return self.do_check_nodesSsh(minutes=2) - def bootcd (self, options): + def bootcd (self): for site_spec in self.plc_spec['sites']: test_site = TestSite (self,site_spec) for node_spec in site_spec['nodes']: @@ -531,19 +528,19 @@ class TestPlc: return False return init_status - def check_initscripts(self, options): + def check_initscripts(self): return self.do_check_initscripts() - def initscripts (self, options): + def initscripts (self): for initscript in self.plc_spec['initscripts']: utils.pprint('Adding Initscript in plc %s'%self.plc_spec['name'],initscript) self.server.AddInitScript(self.auth_root(),initscript['initscript_fields']) return True - def slices (self, options): + def slices (self): return self.do_slices() - def clean_slices (self, options): + def clean_slices (self): return self.do_slices("delete") def do_slices (self, action="add"): @@ -560,40 +557,80 @@ class TestPlc: utils.header('Created Slice %s'%slice['slice_fields']['name']) return True - def check_slices(self, options): + def check_slices(self): for slice_spec in self.plc_spec['slices']: site_spec = self.locate_site (slice_spec['sitename']) test_site = TestSite(self,site_spec) test_slice=TestSlice(self,test_site,slice_spec) - status=test_slice.do_check_slice(options) + status=test_slice.do_check_slice(self.options) if (not status): return False return status - def start_nodes (self, options): + def start_nodes (self): utils.header("Starting nodes") for site_spec in self.plc_spec['sites']: - TestSite(self,site_spec).start_nodes (options) + TestSite(self,site_spec).start_nodes (self.options) return True - def stop_nodes (self, options): - self.kill_all_qemus(options) + def gather_all_logs (self): + # (1) get the plc's /var/log and store it locally in logs/-var-log/* + # (2) get all the nodes qemu log and store it as logs/-qemu.log + # (3) get the nodes /var/log and store is as logs/-var-log/* + # (4) as far as possible get the slice's /var/log as logs/--var-log/* + # (1) + self.gather_logs () + # (2) and (3) + for site_spec in self.plc_spec['sites']: + test_site = TestSite (self,site_spec) + for node_spec in site_spec['nodes']: + TestNode(self,test_site,node_spec).gather_qemu_logs() + TestNode(self,test_site,node_spec).gather_var_logs() return True - def check_tcp (self, options): - # we just need to create a sliver object nothing else - test_sliver=TestSliver(self, - TestNode(self, TestSite(self,self.plc_spec['sites'][0]), - self.plc_spec['sites'][0]['nodes'][0]), - TestSlice(self,TestSite(self,self.plc_spec['sites'][0]), - self.plc_spec['slices'])) - return test_sliver.do_check_tcp(self.plc_spec['tcp_param'],options) + def gather_logs (self): + utils.header("WARNING - Incomplete logs gathering TestPlc.gather_logs") + + + def check_tcp (self): + specs = self.plc_spec['tcp_test'] + overall=True + for spec in specs: + utils.header ("WARNING : xxx check_tcp is underway, spec=%r"%spec) + port = spec['port'] + # locate specs + (s_site,s_node) = self.locate_node(spec['server_node']) + s_slice = self.locate_slice (spec['server_slice']) + # build objects + s_test_site = TestSite (self, s_site) + s_test_node = TestNode (self, s_test_site,s_node) + # xxx the slice site is assumed to be the node site - mhh + s_test_slice = TestSlice (self, s_test_site, s_slice) + s_test_sliver = TestSliver (self, s_test_node, s_test_slice) + if not s_test_sliver.run_tcp_server(port): + overall=False + break + + # idem for the client side + (c_site,c_node) = self.locate_node(spec['server_node']) + c_slice = self.locate_slice (spec['server_slice']) + # build objects + c_test_site = TestSite (self, c_site) + c_test_node = TestNode (self, c_test_site,c_node) + # xxx the slice site is assumed to be the node site - mhh + c_test_slice = TestSlice (self, c_test_site, c_slice) + c_test_sliver = TestSliver (self, c_test_node, c_test_slice) + if not c_test_sliver.run_tcp_client(s_test_node.name(),port): + overall=False + + return overall + # returns the filename to use for sql dump/restore, using options.dbname if set - def dbfile (self, database, options): + def dbfile (self, database): # uses options.dbname if it is found try: - name=options.dbname + name=self.options.dbname if not isinstance(name,StringTypes): raise Exception except: @@ -602,15 +639,14 @@ class TestPlc: name=str(d) return "/root/%s-%s.sql"%(database,name) - def db_dump(self, options): - - dump=self.dbfile("planetab4",options) + def db_dump(self): + dump=self.dbfile("planetab4") self.run_in_guest('pg_dump -U pgsqluser planetlab4 -f '+ dump) utils.header('Dumped planetlab4 database in %s'%dump) return True - def db_restore(self, options): - dump=self.dbfile("planetab4",options) + def db_restore(self): + dump=self.dbfile("planetab4") ##stop httpd service self.run_in_guest('service httpd stop') # xxx - need another wrapper diff --git a/system/TestSlice.py b/system/TestSlice.py index 3448028..02d6caa 100644 --- a/system/TestSlice.py +++ b/system/TestSlice.py @@ -57,16 +57,16 @@ class TestSlice: utils.header("Adding initscript %s in %s"%(isname,slice_name)) self.test_plc.server.AddSliceAttribute(self.test_plc.auth_root(), slice_name,'initscript',isname) - def clear_known_hosts (self): - utils.header("Messing with known_hosts for slice %s"%self.name()) - hostnames=[] - # scan nodenames - for nodename in self.slice_spec['nodenames']: - (site_spec,node_spec) = self.test_plc.locate_node(nodename) - hostnames.append(node_spec['node_fields']['hostname']) - self.test_plc.run_in_guest("sed -i -e /^%s/d /root/.ssh/known_hosts"%node_spec['node_fields']['hostname']) - #scan public key and update the known_host file in the root image - self.test_plc.scan_publicKeys(hostnames) +# def clear_known_hosts (self): +# utils.header("Messing with known_hosts for slice %s"%self.name()) +# hostnames=[] +# # scan nodenames +# for nodename in self.slice_spec['nodenames']: +# (site_spec,node_spec) = self.test_plc.locate_node(nodename) +# hostnames.append(node_spec['node_fields']['hostname']) +# self.test_plc.run_in_guest("sed -i -e /^%s/d /root/.ssh/known_hosts"%node_spec['node_fields']['hostname']) +# #scan public key and update the known_host file in the root image +# self.test_plc.scan_publicKeys(hostnames) def locate_key(self,slice_spec): # locate the first avail. key @@ -89,50 +89,56 @@ class TestSlice: return (found,remote_privatekey) - def do_check_slice(self,options): - bool=True - self.clear_known_hosts() - start_time = datetime.datetime.now() - dead_time=start_time + datetime.timedelta(minutes=15) + def do_check_slice(self,minutes,options): +# self.clear_known_hosts() + timeout = datetime.datetime.now()+datetime.timedelta(minutes=minutes) + + # locate a key slice_spec = self.slice_spec + (found,remote_privatekey)=self.locate_key(slice_spec) + if not found : + utils.header("WARNING: Cannot find a valid key for slice %s"%self.name()) + return False + + # convert nodenames to real hostnames + restarted=[] + tocheck=[] for nodename in slice_spec['nodenames']: (site_spec,node_spec) = self.test_plc.locate_node(nodename) - hostname=node_spec['node_fields']['hostname'] - if TestNode.is_real_model(node_spec['node_fields']['model']): - utils.header("WARNING : Checking slice %s on real node %s skipped"%(self.name(),hostname)) - continue - (found,remote_privatekey)=self.locate_key(slice_spec) - if not found : - raise Exception,"Cannot find a valid key for slice %s"%self.name() - break - while (bool): - utils.header('trying to connect to %s@%s'%(self.name(),hostname)) - Date=self.test_plc.run_in_guest('ssh -i %s %s@%s date'%(remote_privatekey,self.name(),hostname)) - if (Date==0): - break - elif ( start_time <= dead_time ) : - start_time=datetime.datetime.now()+ datetime.timedelta(seconds=45) - time.sleep(45) - elif (options.forcenm): - utils.header('%s@%s : restarting nm in case is in option on %s'%(self.name(),hostname,hostname)) - access=self.test_plc.run_in_guest('ssh -i /etc/planetlab/root_ssh_key.rsa root@%s service nm restart'%hostname) - if (access==0): - utils.header('nm restarted on %s'%hostname) - else: - utils.header('%s@%s : Failed to restart the NM on %s'%(self.name(),hostname,hostname)) - utils.header('Try to reconnect to %s@%s after the tentative of restarting NM'%(self.name(),hostname)) - connect=self.test_plc.run_in_guest('ssh -i %s %s@%s date'%(remote_privatekey,self.name(),hostname)) - if (not connect): - utils.header('connected to %s@%s -->'%(self.name(),hostname)) - break - else: - utils.header('giving up with to %s@%s -->'%(self.name(),hostname)) - bool=False - break - else: - bool=False - break - return bool - - + tocheck.append(node_spec['node_fields']['hostname']) + while tocheck: + for hostname in tocheck: + (site_spec,node_spec) = self.test_plc.locate_host(hostname) + date_test_ssh = TestSsh (hostname,key=remote_privatekey,username=self.name()) + # this can be ran locally as we have the key + utils.header('Trying to enter into slice %s@%s'%(self.name(),hostname)) + date = date_test_ssh.run("date") + if not date: + utils.header("Successfuly entered slice %s on %s"%self.name(),hostname) + tocheck.remove(hostname) + else: + # real nodes will have been checked once in case they're up - skip if not + if TestNode.is_real_model(node_spec['node_fields']['model']): + utils.header("WARNING : Checking slice %s on real node %s skipped"%(self.name(),hostname)) + tocheck.remove(hostname) + # nm restart after first failure, if requested + if options.forcenm and hostname not in restarted: + utils.header ("forcenm option : restarting nm on %s"%hostname) + restart_test_ssh=TestSsh(hostname,key="/etc/planetlab/root_ssh_key.rsa") + access=self.test_plc.run_in_guest(restart_test_ssh.actual_command('service nm restart')) + if (access==0): + utils.header('nm restarted on %s'%hostname) + else: + utils.header('Failed to restart nm on %s'%(hostname)) + restarted.append(hostname) + if not tocheck: + # we're done + return True + if datetime.datetime.now() > timeout: + for hostname in tocheck: + utils.header("FAILURE to ssh into %s@%s"%(self.name(),hostname)) + return False + time.sleep (15) + # for an empty slice + return True diff --git a/system/TestSliver.py b/system/TestSliver.py index 654e401..5fd181a 100644 --- a/system/TestSliver.py +++ b/system/TestSliver.py @@ -39,19 +39,29 @@ class TestSliver: return True + def run_tcp_server (self,port): + print ("XXX run_tcp_server not implemented") + return True + def run_tcp_client (self,hostname,port): + print ("XXX run_tcp_client not implemented") + return True + def run_tcpcheck(self,peer_spec,remote_privatekey): if peer_spec['peer_name']=="server": - tcp_command="ssh -i %s %s@%s ./tcptest.py server -t 10"%(remote_privatekey, peer_spec['slice_name'], - peer_spec['server_name']) - return self.test_plc.run_in_guest(tcp_command) + slice_ssh = TestSsh (peer_spec['server_name'], + key=remote_privatekey, + username=peer_spec['slice_name']) + remote_server_command=slice_ssh.actual_command("./tcptest.py server -t 10") + return self.test_plc.run_in_guest(remote_server_command) else: - tcp_command="ssh -i %s %s@%s ./tcptest.py client -a %s -p %d"%(remote_privatekey, peer_spec['slice_name'], - peer_spec['client_name'],peer_spec['peer_server'], - peer_spec['server_port']) - return self.test_plc.run_in_guest(tcp_command) - - + slice_ssh = TestSsh (peer_spec['client_name'], + key=remote_privatekey, + username=peer_spec['slice_name']) + client_command="/tcptest.py client -a %s -p %d"%(peer_spec['peer_server'], + peer_spec['server_port']) + remote_client_command=slice_ssh.actual_command(client_command) + return self.test_plc.run_in_guest(remote_client_command) def do_check_tcp(self,tcp_param,options): for tcp_spec in tcp_param: diff --git a/system/TestSsh.py b/system/TestSsh.py index d504422..396d4b6 100644 --- a/system/TestSsh.py +++ b/system/TestSsh.py @@ -1,6 +1,19 @@ -#this class is used for any ssh command and -#also for any remote or a local command independently -#on which box this must be done. +# +# Thierry Parmentelat - INRIA +# +# class for issuing commands on a box, either local or remote +# +# the notion of 'buildname' is for providing each test run with a dir of its own +# buildname is generally the name of the build being tested, and can be considered unique +# +# thus 'run_in_buildname' mostly : +# (*) either runs locally in . - as on a local node we are already in a dedicated directory +# (*) or makes sure that there's a remote dir called 'buildname' and runs in it +# +# also, the copy operations +# (*) either do nothing if ran locally +# (*) or copy a local file into the remote 'buildname' +# import os.path import utils @@ -33,10 +46,11 @@ class TestSsh: utils.header("WARNING : something wrong in is_local_hostname with hostname=%s"%hostname) return False - def __init__(self,hostname,buildname=None,key=None): + def __init__(self,hostname,buildname=None,key=None, username=None): self.hostname=hostname self.buildname=buildname self.key=key + self.username=username def is_local(self): return TestSsh.is_local_hostname(self.hostname) @@ -46,7 +60,13 @@ class TestSsh: def key_part (self): if not self.key: return "" - return "-i %s.rsa "%self.key + return "-i %s "%self.key + + def hostname_part (self): + if not self.username: + return self.hostname + else: + return "%s@%s"%(self.username,self.hostname) # command gets run on the right box def actual_command (self, command): @@ -55,7 +75,7 @@ class TestSsh: ssh_command = "ssh " ssh_command += TestSsh.std_options ssh_command += self.key_part() - ssh_command += "%s %s" %(self.hostname,TestSsh.backslash_shell_specials(command)) + ssh_command += "%s %s" %(self.hostname_part(),TestSsh.backslash_shell_specials(command)) return ssh_command def run(self, command,background=False): @@ -78,7 +98,7 @@ class TestSsh: dirname="%s/%s"%(self.buildname,dirname) else: dirname=self.buildname - return self.run("mkdir %s"%dirname) + return self.run("mkdir -p %s"%dirname) def create_buildname_once (self): if self.is_local(): @@ -92,7 +112,7 @@ class TestSsh: def run_in_buildname (self,command, background=False): if self.is_local(): - return utils.system(command) + return utils.system(command,background) self.create_buildname_once() return self.run("cd %s ; %s"%(self.buildname,command),background) @@ -103,7 +123,7 @@ class TestSsh: scp_command="scp " if recursive: scp_command += "-r " scp_command += self.key_part() - scp_command += "%s %s:%s/%s"%(local_file,self.hostname,self.buildname, - os.path.basename(local_file) or ".") + scp_command += "%s %s:%s/%s"%(local_file,self.hostname_part(), + self.buildname,os.path.basename(local_file) or ".") return utils.system(scp_command) diff --git a/system/config_main.py b/system/config_main.py index de4a8a0..307447b 100644 --- a/system/config_main.py +++ b/system/config_main.py @@ -192,26 +192,28 @@ def slices (options): def all_slicenames (options): return [ slice['slice_fields']['name'] for slice in slices(options)] -def tcp_param (options): - try: - return [{ 'tcp_fields' : {'peer_name' : 'server', - 'slice_name' :all_slicenames(options)[0], - 'server_name': all_nodenames()[0] - }, - - }, - { 'tcp_fields':{'peer_name' : 'client', - 'slice_name' :all_slicenames()[1], - 'client_name': all_nodenames()[1], - 'peer_server' : all_nodenames()[0], - 'server_port' : 22 - }, - }, - - ] - except: - return None - +def tcp_tests (options): + all_tests = [ + # local test + { 'server_node': 'node1', + 'server_slice' : 'main_slicetest1', + 'client_node' : 'node1', + 'client_slice' : 'main_slicetest1', + 'port' : 2000, + }, + # remote test + { 'server_node': 'node1', + 'server_slice' : 'main_slicetest1', + 'client_node' : 'node2', + 'client_slice' : 'main_slicetest1', + 'port' : 4000, + }, + ] + if options.small_test: + return [all_tests[0]] + else: + return all_tests + def plc (options) : return { 'name' : 'onetest', @@ -236,7 +238,7 @@ def plc (options) : 'keys' : keys(options), 'initscripts': initscripts(options), 'slices' : slices(options), - 'tcp_param' : tcp_param(options), + 'tcp_test' : tcp_tests(options), } def config (plc_specs,options): diff --git a/system/ssh_config b/system/ssh_config deleted file mode 100644 index c18f3c6..0000000 --- a/system/ssh_config +++ /dev/null @@ -1,4 +0,0 @@ -# this config to be stored in /root/.ssh/config within the myplc image, either chroot or vserver -# this is to prevent ssh from complaining/prompting -BatchMode yes -StrictHostKeyChecking no diff --git a/system/utils.py b/system/utils.py index 6c26318..85ad7ab 100644 --- a/system/utils.py +++ b/system/utils.py @@ -1,6 +1,7 @@ # $Id$ import time import os +import re from pprint import PrettyPrinter # how could this accept a list again ? @@ -90,10 +91,16 @@ def show_test_spec_pass (plc_spec,passno): if key not in ['sites','initscripts','slices','keys']: print '\t',key,':',val -def system(command): +def system(command,background=False): now=time.strftime("%H:%M:%S", time.localtime()) + if background: command += " &" print "+",now,':',command return os.system("set -x; " + command) - +def match (string, pattern): + # tmp - there's probably much simpler + # rewrite * into .*, ? into . + pattern=pattern.replace("*",".*") + pattern=pattern.replace("?",".") + return re.compile(pattern).match(string) -- 2.43.0