+ utils.header("checking ssh access into slice %s on nodes %r"%(self.name(),tocheck))
+ utils.header("max timeout is %d minutes, silent for %d minutes (period is %s)"%\
+ (timeout_minutes,silent_minutes,period))
+ while tocheck:
+ for hostname in tocheck:
+ (site_spec,node_spec) = self.test_plc.locate_hostname(hostname)
+ date_test_ssh = TestSsh (hostname,key=remote_privatekey,username=self.name())
+ command = date_test_ssh.actual_command("echo hostname ; hostname; echo id; id; echo uname -a ; uname -a")
+ date = utils.system (command, silent=datetime.datetime.now() < graceout)
+ if date==0:
+ utils.header("Successfuly entered slice %s on %s"%(self.name(),hostname))
+ tocheck.remove(hostname)
+ else:
+ # real nodes will have been checked once in case they're up - skip if not
+ if TestNode.is_real_model(node_spec['node_fields']['model']):
+ utils.header("WARNING : Checking slice %s on real node %s skipped"%(self.name(),hostname))
+ tocheck.remove(hostname)
+ # nm restart after first failure, if requested
+ if options.forcenm and hostname not in restarted:
+ utils.header ("forcenm option : restarting nm on %s"%hostname)
+ restart_test_ssh=TestSsh(hostname,key="keys/key1.rsa")
+ access=restart_test_ssh.actual_command('service nm restart')
+ if (access==0):
+ utils.header('nm restarted on %s'%hostname)
+ else:
+ utils.header('Failed to restart nm on %s'%(hostname))
+ restarted.append(hostname)
+ if not tocheck:
+ # we're done
+ return True
+ if datetime.datetime.now() > timeout:
+ for hostname in tocheck:
+ utils.header("FAILURE to ssh into %s@%s"%(self.name(),hostname))
+ return False
+ # wait for the period
+ time.sleep (period)
+ # for an empty slice
+ return True