X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=system%2FTestSlice.py;h=3b37d629997c2c5104e7a498ce32c1085f06236c;hb=649f388f61085663d060f100fa011fe81305053d;hp=762758f2800a2e14a36d59e603c8d44bc4990406;hpb=31970aedc37344e730171c07ed73a10769f0253e;p=tests.git diff --git a/system/TestSlice.py b/system/TestSlice.py index 762758f..3b37d62 100644 --- a/system/TestSlice.py +++ b/system/TestSlice.py @@ -1,3 +1,6 @@ +# Thierry Parmentelat +# Copyright (C) 2010 INRIA +# import utils import os, os.path import datetime @@ -6,6 +9,7 @@ import time from TestKey import TestKey from TestUser import TestUser from TestNode import TestNode +from TestSsh import TestSsh class TestSlice: @@ -13,115 +17,163 @@ class TestSlice: self.test_plc=test_plc self.test_site=test_site self.slice_spec=slice_spec - + self.test_ssh=TestSsh(self.test_plc.test_ssh) + def name(self): return self.slice_spec['slice_fields']['name'] + + def get_slice(self,slice_name): + for slice_spec in self.test_plc.plc_spec['slices']: + if(slice_spec['slice_fields']['name']== slice_name): + return slice_spec - def delete_slice(self): + def owner_auth(self): owner_spec = self.test_site.locate_user(self.slice_spec['owner']) - auth = TestUser(self,self.test_site,owner_spec).auth() - slice_fields = self.slice_spec['slice_fields'] - slice_name = slice_fields['name'] - self.test_plc.server.DeleteSlice(auth,slice_fields['name']) - utils.header("Deleted slice %s"%slice_fields['name']) + return TestUser(self,self.test_site,owner_spec).auth() - + def slice_name (self): + return self.slice_spec['slice_fields']['name'] + + # init slice with people, and then add nodes def create_slice(self): - owner_spec = self.test_site.locate_user(self.slice_spec['owner']) - auth = TestUser(self,self.test_site,owner_spec).auth() + auth = self.owner_auth() slice_fields = self.slice_spec['slice_fields'] slice_name = slice_fields['name'] - - self.test_plc.server.AddSlice(auth,slice_fields) + utils.header("Creating slice %s"%slice_name) + self.test_plc.apiserver.AddSlice(auth,slice_fields) for username in self.slice_spec['usernames']: user_spec=self.test_site.locate_user(username) test_user=TestUser(self,self.test_site,user_spec) - self.test_plc.server.AddPersonToSlice(auth, test_user.name(), slice_name) + self.test_plc.apiserver.AddPersonToSlice(auth, test_user.name(), slice_name) + # add initscript code or name as appropriate + if self.slice_spec.has_key('initscriptcode'): + iscode=self.slice_spec['initscriptcode'] + utils.header("Adding initscript code %s in %s"%(iscode,slice_name)) + self.test_plc.apiserver.AddSliceTag(self.test_plc.auth_root(), slice_name,'initscript_code',iscode) + elif self.slice_spec.has_key('initscriptname'): + isname=self.slice_spec['initscriptname'] + utils.header("Adding initscript name %s in %s"%(isname,slice_name)) + self.test_plc.apiserver.AddSliceTag(self.test_plc.auth_root(), slice_name,'initscript',isname) + if self.slice_spec.has_key ('vref'): + vref_value=self.slice_spec['vref'] + self.test_plc.apiserver.AddSliceTag(self.test_plc.auth_root(), slice_name,'vref',vref_value) + + self.add_nodes() + # just add the nodes and handle tags + def add_nodes (self): + auth = self.owner_auth() + slice_name = self.slice_name() hostnames=[] for nodename in self.slice_spec['nodenames']: node_spec=self.test_site.locate_node(nodename) test_node=TestNode(self,self.test_site,node_spec) hostnames += [test_node.name()] utils.header("Adding %r in %s"%(hostnames,slice_name)) - self.test_plc.server.AddSliceToNodes(auth, slice_name, hostnames) - if self.slice_spec.has_key('initscriptname'): - isname=self.slice_spec['initscriptname'] - utils.header("Adding initscript %s in %s"%(isname,slice_name)) - self.test_plc.server.AddSliceAttribute(self.test_plc.auth_root(), slice_name,'initscript',isname) + self.test_plc.apiserver.AddSliceToNodes(auth, slice_name, hostnames) - def clear_known_hosts (self): - utils.header("Messing with known_hosts for slice %s"%self.name()) - # scan nodenames - for nodename in self.slice_spec['nodenames']: - self.test_plc.run_in_guest("sed -i -e '/^%s/d' /root/.ssh/known_hosts"%nodename) - - ###the logic is quit wrong, must be rewritten - def do_check_slices(self): - # Do not wait here, as this step can be run directly in which case you don't want to wait - # just add the 5 minutes to the overall timeout - #utils.header("Waiting for the nodes to fully boot") - #time.sleep(300) - bool=bool1=True - secondes=15 - self.test_plc.clear_ssh_config() - self.clear_known_hosts() - start_time = datetime.datetime.now() - dead_time=start_time + datetime.timedelta(minutes=11) - for slice_spec in self.test_plc.plc_spec['slices']: - for hostname in slice_spec['nodenames']: - slicename=slice_spec['slice_fields']['name'] - # locate the first avail. key - found=False - for username in slice_spec['usernames']: - user_spec=self.test_site.locate_user(username) - for keyname in user_spec['keynames']: - key_spec=self.test_plc.locate_key(keyname) - publickey=TestKey(self.test_plc,key_spec).publicpath() - privatekey=TestKey(self.test_plc,key_spec).privatepath() - if os.path.isfile(publickey) and os.path.isfile(privatekey): - found=True - break - if not found: - raise Exception,"Cannot find a valid key for slice %s"%slicename - - while(bool): - utils.header('restarting nm on %s'%hostname) - access=self.test_plc.run_in_guest('ssh -i /etc/planetlab/root_ssh_key.rsa root@%s service nm restart'%hostname ) - if (access==0): - utils.header('nm restarted on %s'%hostname) - while(bool1): - utils.header('trying to connect to %s@%s'%(slicename,hostname)) - Date=utils.system('ssh -i %s %s@%s date'%(privatekey,slicename,hostname)) - if (Date==0): - break - elif ( start_time <= dead_time ) : - start_time=datetime.datetime.now()+ datetime.timedelta(seconds=30) - time.sleep(secondes) - else: - bool1=False - if(bool1): - utils.header('connected to %s@%s -->'%(slicename,hostname)) + # trash the slice altogether + def delete_slice(self): + auth = self.owner_auth() + slice_name = self.slice_name() + utils.header("Deleting slice %s"%slice_name) + self.test_plc.apiserver.DeleteSlice(auth,slice_name) + + # keep the slice alive and just delete nodes + def delete_nodes (self): + auth = self.owner_auth() + slice_name = self.slice_name() + print 'retrieving slice %s'%slice_name + slice=self.test_plc.apiserver.GetSlices(auth,slice_name)[0] + node_ids=slice['node_ids'] + utils.header ("Deleting %d nodes from slice %s"%\ + (len(node_ids),slice_name)) + self.test_plc.apiserver.DeleteSliceFromNodes (auth,slice_name, node_ids) + + def locate_key(self): + # locate the first avail. key + found=False + for username in self.slice_spec['usernames']: + user_spec=self.test_site.locate_user(username) + for key_name in user_spec['key_names']: + key_spec=self.test_plc.locate_key(key_name) + test_key=TestKey(self.test_plc,key_spec) + publickey=test_key.publicpath() + privatekey=test_key.privatepath() + if os.path.isfile(publickey) and os.path.isfile(privatekey): + found=True + return (found,privatekey) + + + # trying to reach the slice through ssh - expected to answer + def ssh_slice (self, options, *args, **kwds): + "tries to ssh-enter the slice with the user key, to ensure slice creation" + return self.do_ssh_slice(options, expected=True, *args, **kwds) + + # when we expect the slice is not reachable + def ssh_slice_off (self, options, *args, **kwds): + "tries to ssh-enter the slice with the user key, expecting it to be unreachable" + return self.do_ssh_slice(options, expected=False, *args, **kwds) + + def do_ssh_slice(self,options,expected=True,timeout_minutes=20,silent_minutes=10,period=15): + timeout = datetime.datetime.now()+datetime.timedelta(minutes=timeout_minutes) + graceout = datetime.datetime.now()+datetime.timedelta(minutes=silent_minutes) + # locate a key + (found,remote_privatekey)=self.locate_key() + if not found : + utils.header("WARNING: Cannot find a valid key for slice %s"%self.name()) + return False + + # convert nodenames to real hostnames + slice_spec = self.slice_spec + restarted=[] + tocheck=[] + for nodename in slice_spec['nodenames']: + (site_spec,node_spec) = self.test_plc.locate_node(nodename) + tocheck.append(node_spec['node_fields']['hostname']) + + if expected: msg="ssh slice access enabled" + else: msg="ssh slice access disabled" + + utils.header("checking for %s -- slice %s on nodes %r"%(msg,self.name(),tocheck)) + utils.header("max timeout is %d minutes, silent for %d minutes (period is %s)"%\ + (timeout_minutes,silent_minutes,period)) + while tocheck: + for hostname in tocheck: + (site_spec,node_spec) = self.test_plc.locate_hostname(hostname) + date_test_ssh = TestSsh (hostname,key=remote_privatekey,username=self.name()) + command = date_test_ssh.actual_command("echo hostname ; hostname; echo id; id; echo uname -a ; uname -a") + date = utils.system (command, silent=datetime.datetime.now() < graceout) + if getattr(options,'dry_run',None): return True + if expected: success = date==0 + else: success = date!=0 + + if success: + utils.header("OK %s - slice=%s@%s"%(msg,self.name(),hostname)) + tocheck.remove(hostname) + else: + # real nodes will have been checked once in case they're up - skip if not + if TestNode.is_real_model(node_spec['node_fields']['model']): + utils.header("WARNING : Checking slice %s on real node %s skipped"%(self.name(),hostname)) + tocheck.remove(hostname) + # nm restart after first failure, if requested + if options.forcenm and hostname not in restarted: + utils.header ("forcenm option : restarting nm on %s"%hostname) + restart_test_ssh=TestSsh(hostname,key="keys/key_admin.rsa") + access=restart_test_ssh.actual_command('service nm restart') + if (access==0): + utils.header('nm restarted on %s'%hostname) else: - utils.header('%s@%s : last chance - restarting nm on %s'%(slicename,hostname,hostname)) - access=utils.system('ssh -i /etc/planetlab/root_ssh_key.rsa root@%s service nm restart'%hostname) - time.sleep(240)##temoprally adding some delay due to the network slowness - if (access==0): - utils.header('trying to connect (2) to %s@%s'%(slicename,hostname)) - Date=utils.system('ssh -i %s %s@%s date'%(privatekey,slicename,hostname)) - if (Date==0): - utils.header('connected to %s@%s -->'%(slicename,hostname)) - else: - utils.header('giving up with to %s@%s -->'%(slicename,hostname)) - return False - else : - utils.header('Last chance failed on %s@%s -->'%(slicename,hostname)) - break - elif ( start_time <= dead_time ) : - start_time=datetime.datetime.now()+ datetime.timedelta(minutes=1) - time.sleep(secondes) - else: - bool=False - - return bool - + utils.header('Failed to restart nm on %s'%(hostname)) + restarted.append(hostname) + if not tocheck: + # we're done + return True + if datetime.datetime.now() > timeout: + for hostname in tocheck: + utils.header("FAILED %s slice=%s@%s"%(msg,self.name(),hostname)) + return False + # wait for the period + time.sleep (period) + # for an empty slice + return True