From 2cf4ac128c85ecdfd49d2142a6042bf11f9fbdb0 Mon Sep 17 00:00:00 2001 From: Thierry Parmentelat Date: Fri, 9 Oct 2015 10:19:16 +0200 Subject: [PATCH] first set of semantic changes for python3 we can run ping.py - that's the only test so far --- src/nepi/execution/ec.py | 2 +- src/nepi/resources/linux/node.py | 4 ++- src/nepi/util/guid.py | 2 +- src/nepi/util/sfarspec_proc.py | 5 +--- src/nepi/util/sshfuncs.py | 43 +++++++++++++++++++++----------- 5 files changed, 35 insertions(+), 21 deletions(-) diff --git a/src/nepi/execution/ec.py b/src/nepi/execution/ec.py index 15bef5cc..0ccd6aab 100644 --- a/src/nepi/execution/ec.py +++ b/src/nepi/execution/ec.py @@ -587,7 +587,7 @@ class ExperimentController(object): """ # Get next available guid - guid = self._guid_generator.next(guid) + guid = self._guid_generator.__next__(guid) # Instantiate RM rm = ResourceFactory.create(rtype, self, guid) diff --git a/src/nepi/resources/linux/node.py b/src/nepi/resources/linux/node.py index b440038d..7558475c 100644 --- a/src/nepi/resources/linux/node.py +++ b/src/nepi/resources/linux/node.py @@ -765,7 +765,9 @@ class LinuxNode(ResourceManager): if text and not os.path.isfile(src): # src is text input that should be uploaded as file # create a temporal file with the content to upload - f = tempfile.NamedTemporaryFile(delete=False) + # in python3 we need to open in binary mode if str is bytes + mode = 'w' if isinstance(src, str) else 'wb' + f = tempfile.NamedTemporaryFile(mode=mode, delete=False) f.write(src) f.close() src = f.name diff --git a/src/nepi/util/guid.py b/src/nepi/util/guid.py index 24ca3cca..2886d49a 100644 --- a/src/nepi/util/guid.py +++ b/src/nepi/util/guid.py @@ -22,7 +22,7 @@ class GuidGenerator(object): def __init__(self): self._last_guid = 0 - def next(self, guid = None): + def __next__(self, guid = None): if guid == None: guid = self._last_guid + 1 diff --git a/src/nepi/util/sfarspec_proc.py b/src/nepi/util/sfarspec_proc.py index c9da060f..d5e07359 100644 --- a/src/nepi/util/sfarspec_proc.py +++ b/src/nepi/util/sfarspec_proc.py @@ -24,9 +24,6 @@ except ImportError: log.debug("Package sfa-common not installed.\ Could not import sfa.rspecs.rspec and sfa.util.xrn") -from types import StringTypes, ListType - - class SfaRSpecProcessing(object): """ Class to process SFA RSpecs, parse the RSpec replies such as Advertisement RSpecs, @@ -37,7 +34,7 @@ class SfaRSpecProcessing(object): self.config = config def make_dict_rec(self, obj): - if not obj or isinstance(obj, (StringTypes, bool)): + if not obj or isinstance(obj, (str, bool)): return obj if isinstance(obj, list): objcopy = [] diff --git a/src/nepi/util/sshfuncs.py b/src/nepi/util/sshfuncs.py index 048b818c..34d19a24 100644 --- a/src/nepi/util/sshfuncs.py +++ b/src/nepi/util/sshfuncs.py @@ -84,6 +84,7 @@ def resolve_hostname(host): shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines = True, ) stdout, stderr = p.communicate() m = _re_inet.findall(stdout) @@ -124,7 +125,8 @@ def openssh_has_persist(): ["ssh", "-v"], stdout = subprocess.PIPE, stderr = subprocess.STDOUT, - stdin = open("/dev/null"), + stdin = subprocess.DEVNULL, + universal_newlines = True, ) out,err = proc.communicate() proc.wait() @@ -708,6 +710,7 @@ def _retry_rexec(args, stdout = stdout, stdin = stdin, stderr = stderr, + universal_newlines = True, ) # attach tempfile object to the process, to make sure the file stays # alive until the process is finished with it @@ -722,7 +725,9 @@ def _retry_rexec(args, # The method communicate was re implemented for performance issues # when using python subprocess communicate method the ssh commands # last one minute each + #log("BEFORE communicate", level=logging.INFO); import time; beg=time.time() out, err = _communicate(proc, input=None) + #log("AFTER communicate - {}s".format(time.time()-beg), level=logging.INFO) elif stdout: out = proc.stdout.read() @@ -839,15 +844,25 @@ def _communicate(proc, input, timeout=None, err_on_timeout=True): write_set.remove(proc.stdin) if proc.stdout in rlist: - data = os.read(proc.stdout.fileno(), 1024) - if data == "": + # python2 version used to do this + # data = os.read(proc.stdout.fileno(), 1024) + # however this always returned bytes... + data = proc.stdout.read() + log('we have read {}'.format(data)) + # data should be str and not bytes because we use + # universal_lines = True, but to be clean + # instead of saying data != "" + if not data: + log('closing stdout') proc.stdout.close() read_set.remove(proc.stdout) stdout.append(data) if proc.stderr in rlist: - data = os.read(proc.stderr.fileno(), 1024) - if data == "": + # likewise (see above) + # data = os.read(proc.stderr.fileno(), 1024) + data = proc.stderr.read() + if not data: proc.stderr.close() read_set.remove(proc.stderr) stderr.append(data) @@ -858,15 +873,15 @@ def _communicate(proc, input, timeout=None, err_on_timeout=True): if stderr is not None: stderr = ''.join(stderr) - # Translate newlines, if requested. We cannot let the file - # object do the translation: It is based on stdio, which is - # impossible to combine with select (unless forcing no - # buffering). - if proc.universal_newlines and hasattr(file, 'newlines'): - if stdout: - stdout = proc._translate_newlines(stdout) - if stderr: - stderr = proc._translate_newlines(stderr) +# # Translate newlines, if requested. We cannot let the file +# # object do the translation: It is based on stdio, which is +# # impossible to combine with select (unless forcing no +# # buffering). +# if proc.universal_newlines and hasattr(file, 'newlines'): +# if stdout: +# stdout = proc._translate_newlines(stdout) +# if stderr: +# stderr = proc._translate_newlines(stderr) if killed and err_on_timeout: errcode = proc.poll() -- 2.43.0