X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=src%2Fnepi%2Futil%2Fserver.py;h=ed9bccd30f9e591bd757dd4ac076159ef6c01f9a;hb=70c33a3b0fe396cce12d3d1c0eb6cf40ce537a95;hp=5a37fab73ec4870e7c680009dbbcadcdb1395183;hpb=80ee37b784355888ac0d8149741ef96988d3660b;p=nepi.git diff --git a/src/nepi/util/server.py b/src/nepi/util/server.py index 5a37fab7..ed9bccd3 100644 --- a/src/nepi/util/server.py +++ b/src/nepi/util/server.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- from nepi.util.constants import DeploymentConfiguration as DC @@ -22,6 +21,7 @@ import tempfile import defer import functools import collections +import hashlib CTRL_SOCK = "ctrl.sock" CTRL_PID = "ctrl.pid" @@ -32,6 +32,8 @@ STOP_MSG = "STOP" TRACE = os.environ.get("NEPI_TRACE", "false").lower() in ("true", "1", "on") +OPENSSH_HAS_PERSIST = None + if hasattr(os, "devnull"): DEV_NULL = os.devnull else: @@ -39,6 +41,29 @@ else: SHELL_SAFE = re.compile('^[-a-zA-Z0-9_=+:.,/]*$') +hostbyname_cache = dict() + +def gethostbyname(host): + hostbyname = hostbyname_cache.get(host) + if not hostbyname: + hostbyname = socket.gethostbyname(host) + hostbyname_cache[host] = hostbyname + return hostbyname + +def openssh_has_persist(): + global OPENSSH_HAS_PERSIST + if OPENSSH_HAS_PERSIST is None: + proc = subprocess.Popen(["ssh","-v"], + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + stdin = open("/dev/null","r") ) + out,err = proc.communicate() + proc.wait() + + vre = re.compile(r'OpenSSH_(?:[6-9]|5[.][8-9]|5[.][1-9][0-9]|[1-9][0-9]).*', re.I) + OPENSSH_HAS_PERSIST = bool(vre.match(out)) + return OPENSSH_HAS_PERSIST + def shell_escape(s): """ Escapes strings so that they are safe to use as command-line arguments """ if SHELL_SAFE.match(s): @@ -553,9 +578,11 @@ def _make_server_key_args(server_key, host, port, args): host = '%s:%s' % (host,port) # Create a temporary server key file tmp_known_hosts = tempfile.NamedTemporaryFile() - + + hostbyname = gethostbyname(host) + # Add the intended host key - tmp_known_hosts.write('%s,%s %s\n' % (host, socket.gethostbyname(host), server_key)) + tmp_known_hosts.write('%s,%s %s\n' % (host, hostbyname, server_key)) # If we're not in strict mode, add user-configured keys if os.environ.get('NEPI_STRICT_AUTH_MODE',"").lower() not in ('1','true','on'): @@ -579,15 +606,14 @@ def popen_ssh_command(command, host, port, user, agent, timeout = None, retry = 0, err_on_timeout = True, - connect_timeout = 30): + connect_timeout = 60, + persistent = True, + hostip = None): """ Executes a remote commands, returns ((stdout,stderr),process) """ - if TRACE: - print "ssh", host, command - + tmp_known_hosts = None - connkey = repr((user,host,port)).encode("base64").strip().replace('/','.') args = ['ssh', '-C', # Don't bother with localhost. Makes test easier '-o', 'NoHostAuthenticationForLocalhost=yes', @@ -595,10 +621,12 @@ def popen_ssh_command(command, host, port, user, agent, '-o', 'ConnectionAttempts=3', '-o', 'ServerAliveInterval=30', '-o', 'TCPKeepAlive=yes', + '-l', user, hostip or host] + if persistent and openssh_has_persist(): + args.extend([ '-o', 'ControlMaster=auto', - '-o', 'ControlPath=/tmp/nepi_ssh_pl_%s' % ( connkey, ), - '-o', 'ControlPersist=60', - '-l', user, host] + '-o', 'ControlPath=/tmp/nepi_ssh-%r@%h:%p', + '-o', 'ControlPersist=60' ]) if agent: args.append('-A') if port: @@ -607,6 +635,7 @@ def popen_ssh_command(command, host, port, user, agent, args.extend(('-i', ident_key)) if tty: args.append('-t') + args.append('-t') if server_key: # Create a temporary server key file tmp_known_hosts = _make_server_key_args( @@ -623,23 +652,34 @@ def popen_ssh_command(command, host, port, user, agent, # attach tempfile object to the process, to make sure the file stays # alive until the process is finished with it proc._known_hosts = tmp_known_hosts - + try: out, err = _communicate(proc, stdin, timeout, err_on_timeout) - if proc.poll() and err.strip().startswith('ssh: '): - # SSH error, can safely retry - continue + if TRACE: + print "COMMAND host %s, command %s, out %s, error %s" % (host, " ".join(args), out, err) + + if proc.poll(): + if err.strip().startswith('ssh: ') or err.strip().startswith('mux_client_hello_exchange: '): + # SSH error, can safely retry + continue + elif : + ControlSocket /tmp/nepi_ssh-inria_alina@planetlab04.cnds.unibe.ch:22 already exists, disabling multiplexing + # SSH error, can safely retry (but need to delete controlpath file) + # TODO: delete file + continue + elif retry: + # Probably timed out or plain failed but can retry + continue break except RuntimeError,e: + if TRACE: + print "EXCEPTION host %s, command %s, out %s, error %s, exception TIMEOUT -> %s" % ( + host, " ".join(args), out, err, e.args) + if retry <= 0: raise - if TRACE: - print " timedout -> ", e.args retry -= 1 - if TRACE: - print " -> ", out, err - return ((out, err), proc) def popen_scp(source, dest, @@ -696,18 +736,19 @@ def popen_scp(source, dest, user,host = remspec.rsplit('@',1) tmp_known_hosts = None - connkey = repr((user,host,port)).encode("base64").strip().replace('/','.') args = ['ssh', '-l', user, '-C', # Don't bother with localhost. Makes test easier '-o', 'NoHostAuthenticationForLocalhost=yes', - '-o', 'ConnectTimeout=30', + '-o', 'ConnectTimeout=60', '-o', 'ConnectionAttempts=3', '-o', 'ServerAliveInterval=30', '-o', 'TCPKeepAlive=yes', - '-o', 'ControlMaster=auto', - '-o', 'ControlPath=/tmp/nepi_ssh_pl_%s' % ( connkey, ), - '-o', 'ControlPersist=60', host ] + if openssh_has_persist(): + args.extend([ + '-o', 'ControlMaster=auto', + '-o', 'ControlPath=/tmp/nepi_ssh-%r@%h:%p', + '-o', 'ControlPersist=60' ]) if port: args.append('-P%d' % port) if ident_key: @@ -831,7 +872,12 @@ def popen_scp(source, dest, tmp_known_hosts = None args = ['scp', '-q', '-p', '-C', # Don't bother with localhost. Makes test easier - '-o', 'NoHostAuthenticationForLocalhost=yes' ] + '-o', 'NoHostAuthenticationForLocalhost=yes', + '-o', 'ConnectTimeout=60', + '-o', 'ConnectionAttempts=3', + '-o', 'ServerAliveInterval=30', + '-o', 'TCPKeepAlive=yes' ] + if port: args.append('-P%d' % port) if recursive: @@ -845,6 +891,10 @@ def popen_scp(source, dest, if isinstance(source,list): args.extend(source) else: + if openssh_has_persist(): + args.extend([ + '-o', 'ControlMaster=auto', + '-o', 'ControlPath=/tmp/nepi_ssh-%r@%h:%p']) args.append(source) args.append(dest)