From 3fe2e6f7812888dc1366915545dd2243ff6fb1bb Mon Sep 17 00:00:00 2001 From: Thierry Parmentelat Date: Tue, 3 Nov 2015 16:15:49 +0100 Subject: [PATCH] the big merge --- examples/ns3/multi_host/topology.py | 16 ++++++-- examples/planetlab/update_fedora_repo.py | 4 +- setup.py | 28 ++++++++----- src/nepi/execution/ec.py | 6 ++- src/nepi/execution/scheduler.py | 8 +++- src/nepi/resources/linux/node.py | 6 ++- src/nepi/resources/netns/netnswrapper.py | 8 ++-- src/nepi/resources/ns3/ns3netdevice.py | 13 +++++- src/nepi/resources/ns3/ns3wifimac.py | 7 +++- src/nepi/resources/ns3/ns3wifiphy.py | 7 +++- src/nepi/resources/ns3/ns3wrapper.py | 11 ++++-- src/nepi/resources/planetlab/plcapi.py | 22 ++++++----- src/nepi/util/guid.py | 1 + src/nepi/util/manifoldapi.py | 4 +- src/nepi/util/netgraph.py | 26 ++++++++---- src/nepi/util/parallel.py | 13 +++--- src/nepi/util/parsers/xml_parser.py | 50 +++++++++++++++++------- src/nepi/util/sfarspec_proc.py | 4 +- src/nepi/util/sshfuncs.py | 33 +++++++++++----- test/execution/scheduler.py | 8 ++-- 20 files changed, 194 insertions(+), 81 deletions(-) diff --git a/examples/ns3/multi_host/topology.py b/examples/ns3/multi_host/topology.py index 70dbbbf5..f4d6504c 100644 --- a/examples/ns3/multi_host/topology.py +++ b/examples/ns3/multi_host/topology.py @@ -19,7 +19,11 @@ # Alina Quereilhac # -import ipaddr +from six import PY2, next +if PY2: + import ipaddr +else: + import ipaddress from optparse import OptionParser import os from random import randint @@ -229,10 +233,14 @@ def add_ns3_route(ec, ns3_node, network, prefixlen, nexthop): def build_ns3_topology(ec, simu, node_count, network, prefixlen, agent_ip): channel = add_ns3_wifi_channel(ec) - net = ipaddr.IPv4Network("%s/%s" % (network, prefixlen)) - itr = net.iterhosts() + if PY2: + net = ipaddr.IPv4Network("%s/%s" % (network, prefixlen)) + itr = net.iterhosts() + else: + net = ipaddress.IPv4Network("%s/%s" % (network, prefixlen)) + itr = net.hosts() - ap_ip = itr.next().exploded + ap_ip = next(itr).exploded ap = add_ns3_node(ec, simu, ap_ip, prefixlen, channel, ap_mode=True) agent = None diff --git a/examples/planetlab/update_fedora_repo.py b/examples/planetlab/update_fedora_repo.py index 71ec4d8f..ed26f8df 100644 --- a/examples/planetlab/update_fedora_repo.py +++ b/examples/planetlab/update_fedora_repo.py @@ -29,6 +29,8 @@ from __future__ import print_function +from six.moves import input + from nepi.execution.ec import ExperimentController from optparse import OptionParser, SUPPRESS_HELP @@ -57,7 +59,7 @@ parser.add_option("-k", "--pl-ssh-key", dest="pl_ssh_key", (options, args) = parser.parse_args() -proceed = raw_input ("Executing this script will modify the fedora yum repositories in the selected PlanetLab hosts. Are you sure to continue? [y/N] ") +proceed = input ("Executing this script will modify the fedora yum repositories in the selected PlanetLab hosts. Are you sure to continue? [y/N] ") if proceed.lower() not in ['yes', 'y']: os._exit(1) diff --git a/setup.py b/setup.py index ef584b0d..235743d5 100755 --- a/setup.py +++ b/setup.py @@ -3,6 +3,8 @@ from distutils.core import setup import sys +PY2 = sys.version_info[0] == 2 + with open('VERSION') as f: version_tag = f.read().strip() with open("COPYING") as f: @@ -12,6 +14,22 @@ with open("README") as f: data_files = [ ('/etc/nepi', [ 'VERSION', 'COPYING', 'README' ] ) ] +### requirements - used by pip install +required_modules = [ ] + # we are now using six for a portable code +required_modules.append('six') + # ipaddr in py2 used to be a separate lib + # within recent py3, it is now in standard library but named ipaddress +if PY2: + required_modules.append('ipaddr') + # this is required regardless of the python version +required_modules.append('networkx') + # refrain from mentioning these ones that are not exactly crucial + # and that have additional, non-python, dependencies + # that can easily break the whole install +#required_modules.append('matplotlib') +#required_modules.append('pygraphviz') + setup( name = "nepi", version = version_tag, @@ -53,13 +71,5 @@ setup( "nepi.resources.linux" : [ "scripts/*.py" ], "nepi.resources.linux.ns3" : [ "dependencies/*.tar.gz" ] }, - install_requires = [ - "ipaddr", - "networkx", - # refrain from mentioning these ones that are not exactly crucial - # and that have additional, non-python, dependencies - # that can easily break the whole install - # "matplotlib", - # "pygraphviz", - ] + install_requires = required_modules, ) diff --git a/src/nepi/execution/ec.py b/src/nepi/execution/ec.py index d93ef7a7..c403d797 100644 --- a/src/nepi/execution/ec.py +++ b/src/nepi/execution/ec.py @@ -16,6 +16,8 @@ # # Author: Alina Quereilhac +from six import next + from nepi.util import guid from nepi.util.parallel import ParallelRun from nepi.util.timefuncs import tnow, tdiffsec, stabsformat, tsformat @@ -587,6 +589,7 @@ class ExperimentController(object): """ # Get next available guid + # xxx_next_hiccup guid = self._guid_generator.next(guid) # Instantiate RM @@ -985,6 +988,7 @@ class ExperimentController(object): new_group = False if not group: new_group = True + # xxx_next_hiccup group = self._group_id_generator.next() if group not in self._groups: @@ -1186,7 +1190,7 @@ class ExperimentController(object): try: self._cond.acquire() - task = self._scheduler.next() + task = next(self._scheduler) if not task: # No task to execute. Wait for a new task to be scheduled. diff --git a/src/nepi/execution/scheduler.py b/src/nepi/execution/scheduler.py index 6ebcfea3..e07ba752 100644 --- a/src/nepi/execution/scheduler.py +++ b/src/nepi/execution/scheduler.py @@ -16,6 +16,8 @@ # # Author: Alina Quereilhac +from six import next + import itertools import heapq @@ -75,7 +77,7 @@ class HeapScheduler(object): :type task: task """ if task.id == None: - task.id = self._idgen.next() + task.id = next(self._idgen) entry = (task.timestamp, task.id, task) self._valid.add(task.id) @@ -94,6 +96,10 @@ class HeapScheduler(object): except: pass + # py3 compat + def __next__(self): + return self.next() + def next(self): """ Get the next task in the queue by timestamp and arrival order """ diff --git a/src/nepi/resources/linux/node.py b/src/nepi/resources/linux/node.py index a411c644..375ff3a6 100644 --- a/src/nepi/resources/linux/node.py +++ b/src/nepi/resources/linux/node.py @@ -32,6 +32,8 @@ import time import threading import traceback +from six import PY3 + # TODO: Unify delays!! # TODO: Validate outcome of uploads!! @@ -1194,5 +1196,7 @@ class LinuxNode(ResourceManager): if not dests: return [] - return dests.values() + retcod = dests.values() + if PY3: retcod = list(retcod) + return retcod diff --git a/src/nepi/resources/netns/netnswrapper.py b/src/nepi/resources/netns/netnswrapper.py index 9dff830e..5a621947 100644 --- a/src/nepi/resources/netns/netnswrapper.py +++ b/src/nepi/resources/netns/netnswrapper.py @@ -22,6 +22,8 @@ import os import sys import uuid +from six import integer_types, string_types + class NetNSWrapper(object): def __init__(self, loglevel = logging.INFO, enable_dump = False): super(NetNSWrapper, self).__init__() @@ -111,10 +113,10 @@ class NetNSWrapper(object): result = method(*realargs, **realkwargs) # If the result is an object (not a base value), - # then keep track of the object a return the object + # then keep track of the object and return the object # reference (newuuid) - if not (result is None or type(result) in [ - bool, float, long, str, int]): + if result is not None \ + and not isinstance(result, (bool, float) + integer_types + string_types): self._objects[newuuid] = result result = newuuid diff --git a/src/nepi/resources/ns3/ns3netdevice.py b/src/nepi/resources/ns3/ns3netdevice.py index 946f01a1..5c99cd10 100644 --- a/src/nepi/resources/ns3/ns3netdevice.py +++ b/src/nepi/resources/ns3/ns3netdevice.py @@ -21,7 +21,13 @@ from nepi.execution.resource import clsinit_copy from nepi.execution.trace import Trace from nepi.resources.ns3.ns3base import NS3Base -import ipaddr +import sys +PY2 = sys.version_info[0] == 2 + +if PY2: + import ipaddr +else: + import ipaddress @clsinit_copy class NS3BaseNetDevice(NS3Base): @@ -150,7 +156,10 @@ class NS3BaseNetDevice(NS3Base): ip = self.get("ip") prefix = self.get("prefix") - i = ipaddr.IPAddress(ip) + if PY2: + i = ipaddr.IPAddress(ip) + else: + i = ipaddress.ip_address(ip) if i.version == 4: # IPv4 ipv4 = self.node.ipv4 diff --git a/src/nepi/resources/ns3/ns3wifimac.py b/src/nepi/resources/ns3/ns3wifimac.py index 7b488307..801a2ee2 100644 --- a/src/nepi/resources/ns3/ns3wifimac.py +++ b/src/nepi/resources/ns3/ns3wifimac.py @@ -16,6 +16,8 @@ # # Author: Alina Quereilhac +from six import PY3 + from nepi.execution.attribute import Attribute, Flags, Types from nepi.execution.resource import clsinit_copy from nepi.resources.ns3.ns3base import NS3Base @@ -27,9 +29,12 @@ class NS3BaseWifiMac(NS3Base): @classmethod def _register_attributes(cls): + # stay safe and keep extra list() added by 2to3 + allowed = WIFI_STANDARDS.keys() + if PY3: allowed = list(allowed) standard = Attribute("Standard", "Wireless standard", default = "WIFI_PHY_STANDARD_80211a", - allowed = WIFI_STANDARDS.keys(), + allowed = allowed, type = Types.Enumerate, flags = Flags.Design) diff --git a/src/nepi/resources/ns3/ns3wifiphy.py b/src/nepi/resources/ns3/ns3wifiphy.py index 331c9444..b3359744 100644 --- a/src/nepi/resources/ns3/ns3wifiphy.py +++ b/src/nepi/resources/ns3/ns3wifiphy.py @@ -16,6 +16,8 @@ # # Author: Alina Quereilhac +from six import PY3 + from nepi.execution.attribute import Attribute, Flags, Types from nepi.execution.resource import clsinit_copy from nepi.resources.ns3.ns3base import NS3Base @@ -27,9 +29,12 @@ class NS3BaseWifiPhy(NS3Base): @classmethod def _register_attributes(cls): + # stay safe and keep extra list() added by 2to3 + allowed = WIFI_STANDARDS.keys() + if PY3: allowed = list(allowed) standard = Attribute("Standard", "Wireless standard", default = "WIFI_PHY_STANDARD_80211a", - allowed = WIFI_STANDARDS.keys(), + allowed = allowed, type = Types.Enumerate, flags = Flags.Design) diff --git a/src/nepi/resources/ns3/ns3wrapper.py b/src/nepi/resources/ns3/ns3wrapper.py index a9a5e126..7172ad1c 100644 --- a/src/nepi/resources/ns3/ns3wrapper.py +++ b/src/nepi/resources/ns3/ns3wrapper.py @@ -23,6 +23,8 @@ import threading import time import uuid +from six import integer_types, string_types + SINGLETON = "singleton::" SIMULATOR_UUID = "singleton::Simulator" CONFIG_UUID = "singleton::Config" @@ -326,8 +328,8 @@ class NS3Wrapper(object): # If the result is an object (not a base value), # then keep track of the object a return the object # reference (newuuid) - if not (result is None or type(result) in [ - bool, float, long, str, int]): + if result is not None \ + and not isinstance(result, (bool, float) + integer_types + string_types): self._objects[newuuid] = result result = newuuid @@ -482,7 +484,10 @@ class NS3Wrapper(object): condition.release() # contextId is defined as general context - contextId = long(0xffffffff) + # xxx possible distortion when upgrading to python3 + # really not sure what's the point in this long() business.. + #contextId = long(0xffffffff) + contextId = 0xffffffff # delay 0 means that the event is expected to execute inmediately delay = self.ns3.Seconds(0) diff --git a/src/nepi/resources/planetlab/plcapi.py b/src/nepi/resources/planetlab/plcapi.py index 01e3c102..42d652ca 100644 --- a/src/nepi/resources/planetlab/plcapi.py +++ b/src/nepi/resources/planetlab/plcapi.py @@ -22,7 +22,9 @@ import socket import os import time import threading -import xmlrpclib + +from six import integer_types, string_types +from six.moves import xmlrpc_client def _retry(fn): def rv(*p, **kw): @@ -156,15 +158,15 @@ class PLCAPI(object): self._url = urlpattern % {'hostname':hostname} if (proxy is not None): - import urllib2 - class HTTPSProxyTransport(xmlrpclib.Transport): + from six.moves.urllib import request as urllib_request + class HTTPSProxyTransport(xmlrpc_client.Transport): def __init__(self, proxy, use_datetime=0): - opener = urllib2.build_opener(urllib2.ProxyHandler({"https" : proxy})) - xmlrpclib.Transport.__init__(self, use_datetime) + opener = urllib_request.build_opener(urllib2.ProxyHandler({"https" : proxy})) + xmlrpc_client.Transport.__init__(self, use_datetime) self.opener = opener def request(self, host, handler, request_body, verbose=0): - req = urllib2.Request('https://%s%s' % (host, handler), request_body) + req = urllib_request.Request('https://%s%s' % (host, handler), request_body) req.add_header('User-agent', self.user_agent) self.verbose = verbose return self.parse_response(self.opener.open(req)) @@ -182,7 +184,7 @@ class PLCAPI(object): @property def api(self): # Cannot reuse same proxy in all threads, py2.7 is not threadsafe - return xmlrpclib.ServerProxy( + return xmlrcp_client.ServerProxy( self._url , transport = self._proxy_transport(), allow_none = True) @@ -212,7 +214,7 @@ class PLCAPI(object): try: # test authorization network_types = _retry(self.mcapi.GetNetworkTypes)(self.auth) - except (xmlrpclib.ProtocolError, xmlrpclib.Fault) as e: + except (xmlrpc_client.ProtocolError, xmlrpc_client.Fault) as e: warnings.warn(str(e)) return True @@ -283,7 +285,7 @@ class PLCAPI(object): * nodefamily : string, the nodefamily this node should be based upon * plain : boolean, use plain bootstrapfs image if set (for tests) """ - if not isinstance(node, (str, int, long)): + if not isinstance(node, integer_types + string_types): raise ValueError("Node must be either a non-unicode string or an int") return _retry(self.mcapi.GetNodeFlavour)(self.auth, node) @@ -432,7 +434,7 @@ class PLCAPI(object): return _retry(self.mcapi.DeleteSliceFromNodes)(self.auth, slice_id_or_name, node_id_or_hostname) def start_multicall(self): - self.threadlocal.mc = xmlrpclib.MultiCall(self.mcapi) + self.threadlocal.mc = xmlrpc_client.MultiCall(self.mcapi) def finish_multicall(self): mc = self.threadlocal.mc diff --git a/src/nepi/util/guid.py b/src/nepi/util/guid.py index 24ca3cca..45852118 100644 --- a/src/nepi/util/guid.py +++ b/src/nepi/util/guid.py @@ -22,6 +22,7 @@ class GuidGenerator(object): def __init__(self): self._last_guid = 0 + # xxx_next_hiccup - this is used as a plain function, and only in ec.py def next(self, guid = None): if guid == None: guid = self._last_guid + 1 diff --git a/src/nepi/util/manifoldapi.py b/src/nepi/util/manifoldapi.py index 51724102..b879b716 100644 --- a/src/nepi/util/manifoldapi.py +++ b/src/nepi/util/manifoldapi.py @@ -18,7 +18,7 @@ from __future__ import print_function -import xmlrpclib +from six.moves import xmlrpc_client import hashlib import threading @@ -38,7 +38,7 @@ class MANIFOLDAPI(object): @property def api(self): - return xmlrpclib.Server(self._url, allow_none = True) + return xmlrpc_client.Server(self._url, allow_none = True) def get_session_key(self): """ diff --git a/src/nepi/util/netgraph.py b/src/nepi/util/netgraph.py index 7a681ec3..96fb384d 100644 --- a/src/nepi/util/netgraph.py +++ b/src/nepi/util/netgraph.py @@ -16,11 +16,17 @@ # # Author: Alina Quereilhac -import ipaddr import networkx import math import random +from six import next, PY2, PY3 +if PY2: + import ipaddr +else: + import ipaddress + + class TopologyType: LINEAR = "linear" LADDER = "ladder" @@ -196,7 +202,9 @@ class NetGraph(object): return self.topology.node[nid].get(name) def node_annotations(self, nid): - return self.topology.node[nid].keys() + retcod = self.topology.node[nid].keys() + if PY3: retcod = list(retcod) + return retcod def del_node_annotation(self, nid, name): del self.topology.node[nid][name] @@ -224,7 +232,9 @@ class NetGraph(object): return self.topology.edge[nid1][nid2].get(name) def edge_annotations(self, nid1, nid2): - return self.topology.edge[nid1][nid2].keys() + retcod = self.topology.edge[nid1][nid2].keys() + if PY3: retcod = list(retcod) + return retcod def del_edge_annotation(self, nid1, nid2, name): del self.topology.edge[nid1][nid2][name] @@ -250,10 +260,10 @@ class NetGraph(object): # Assign IP addresses to host netblock = "%s/%d" % (network, prefix) if version == 4: - net = ipaddr.IPv4Network(netblock) + net = ipaddr.IPv4Network(netblock) if PY2 else ipaddress.ip_network(netblock) new_prefix = 30 elif version == 6: - net = ipaddr.IPv6Network(netblock) + net = ipaddr.IPv6Network(netblock) if PY2 else ipaddress.ip_network(netblock) new_prefix = 30 else: raise RuntimeError("Invalid IP version %d" % version) @@ -269,15 +279,15 @@ class NetGraph(object): #### Compute subnets for each link # get a subnet of base_add with prefix /30 - subnet = sub_itr.next() + subnet = next(sub_itr) mask = subnet.netmask.exploded network = subnet.network.exploded prefixlen = subnet.prefixlen # get host addresses in that subnet i = subnet.iterhosts() - addr1 = i.next() - addr2 = i.next() + addr1 = next(i) + addr2 = next(i) ip1 = addr1.exploded ip2 = addr2.exploded diff --git a/src/nepi/util/parallel.py b/src/nepi/util/parallel.py index 04c9d17f..4b1acc81 100644 --- a/src/nepi/util/parallel.py +++ b/src/nepi/util/parallel.py @@ -19,11 +19,12 @@ # import threading -import Queue import traceback import sys import os +from six.moves import queue + N_PROCS = None class WorkerThread(threading.Thread): @@ -64,12 +65,12 @@ class ParallelRun(object): self.maxqueue = maxqueue self.maxthreads = maxthreads - self.queue = Queue.Queue(self.maxqueue or 0) + self.queue = queue.Queue(self.maxqueue or 0) self.delayed_exceptions = [] if results: - self.rvqueue = Queue.Queue() + self.rvqueue = queue.Queue() else: self.rvqueue = None @@ -111,7 +112,7 @@ class ParallelRun(object): try: self.queue.get(block = False) self.queue.task_done() - except Queue.Empty: + except queue.Empty: break def destroy(self): @@ -151,10 +152,10 @@ class ParallelRun(object): while True: try: yield self.rvqueue.get_nowait() - except Queue.Empty: + except queue.Empty: self.queue.join() try: yield self.rvqueue.get_nowait() - except Queue.Empty: + except queue.Empty: raise StopIteration diff --git a/src/nepi/util/parsers/xml_parser.py b/src/nepi/util/parsers/xml_parser.py index c11782d7..d82ea691 100644 --- a/src/nepi/util/parsers/xml_parser.py +++ b/src/nepi/util/parsers/xml_parser.py @@ -32,22 +32,44 @@ BOOL = "bool" INTEGER = "integer" DOUBLE = "float" -def xmlencode(s): - if isinstance(s, str): - rv = s.decode("latin1") - if isinstance(s, datetime.datetime): - rv = tsformat(s) - elif not isinstance(s, unicode): - rv = unicode(s) - else: - rv = s - return rv.replace(u'\x00',u'�') - +from six import PY2 + +if PY2: + # xxx old py2 code had a hack, that had 'latin1' hardcoded + # as the encoding for 8-byte strings + # this is very wrong; I keep it for now + # but will probably remove it altogether some day + def xmlencode(s): + """xml encoder for python2""" + if isinstance(s, str): + rv = s.decode("latin1") + if isinstance(s, datetime.datetime): + rv = tsformat(s) + elif not isinstance(s, unicode): + rv = unicode(s) + else: + rv = s + return rv.replace(u'\x00',u'�') +else: + # use sys.getdefaultencoding() to decode bytes into string + def xmlencode(s): + """xml encoder for python3""" + if isinstance(s, datetime.datetime): + rv = tsformat(s) + elif isinstance(s, bytes): + rv = s.decode(sys.getdefaultencoding()) + else: + rv = s + return rv.replace('\x00', '�') + def xmldecode(s, cast = str): - ret = s.replace(u'�',u'\x00').encode("ascii") - ret = cast(ret) - if s == "None": + if s is None: return None + if PY2: + ret = s.replace(u'�', u'\x00').encode("ascii") + else: + ret = s.replace('�', '\x00') + ret = cast(ret) return ret def from_type(value): diff --git a/src/nepi/util/sfarspec_proc.py b/src/nepi/util/sfarspec_proc.py index f8c8bb2f..b131ddc3 100644 --- a/src/nepi/util/sfarspec_proc.py +++ b/src/nepi/util/sfarspec_proc.py @@ -24,7 +24,7 @@ except ImportError: log.debug("Package sfa-common not installed.\ Could not import sfa.rspecs.rspec and sfa.util.xrn") -from types import StringTypes, ListType +from six import string_types class SfaRSpecProcessing(object): @@ -37,7 +37,7 @@ class SfaRSpecProcessing(object): self.config = config def make_dict_rec(self, obj): - if not obj or isinstance(obj, (StringTypes, bool)): + if not obj or isinstance(obj, (bool,) + string_types): return obj if isinstance(obj, list): objcopy = [] diff --git a/src/nepi/util/sshfuncs.py b/src/nepi/util/sshfuncs.py index 7587e97e..1236ed56 100644 --- a/src/nepi/util/sshfuncs.py +++ b/src/nepi/util/sshfuncs.py @@ -34,6 +34,8 @@ import threading import time import tempfile +from six import PY2 + _re_inet = re.compile("\d+:\s+(?P[a-z0-9_-]+)\s+inet6?\s+(?P[a-f0-9.:/]+)\s+(brd\s+[0-9.]+)?.*scope\s+global.*") logger = logging.getLogger("sshfuncs") @@ -79,11 +81,13 @@ def resolve_hostname(host): ip = None if host in ["localhost", "127.0.0.1", "::1"]: + extras = {} if PY2 else {'universal_newlines' : True} p = subprocess.Popen( "ip -o addr list", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + **extras ) stdout, stderr = p.communicate() m = _re_inet.findall(stdout) @@ -120,12 +124,14 @@ def openssh_has_persist(): """ global OPENSSH_HAS_PERSIST if OPENSSH_HAS_PERSIST is None: + extras = {} if PY2 else {'universal_newlines' : True} with open("/dev/null") as null: proc = subprocess.Popen( ["ssh", "-v"], stdout = subprocess.PIPE, stderr = subprocess.STDOUT, stdin = null, + **extras ) out,err = proc.communicate() proc.wait() @@ -701,6 +707,7 @@ def _retry_rexec(args, # display command actually invoked when debug is turned on message = " ".join( [ "'{}'".format(arg) for arg in args ] ) log("sshfuncs: invoking {}".format(message), logging.DEBUG) + extras = {} if PY2 else {'universal_newlines' : True} # connects to the remote host and starts a remote connection proc = subprocess.Popen( args, @@ -708,6 +715,7 @@ def _retry_rexec(args, stdout = stdout, stdin = stdin, stderr = stderr, + **extras ) # attach tempfile object to the process, to make sure the file stays # alive until the process is finished with it @@ -840,16 +848,21 @@ def _communicate(proc, input, timeout=None, err_on_timeout=True): proc.stdin.close() write_set.remove(proc.stdin) + # xxx possible distortion when upgrading to python3 + # original py2 version used to do + # data = os.read(proc.stdout.fileno(), 1024) + # but this would return bytes, so.. if proc.stdout in rlist: - data = os.read(proc.stdout.fileno(), 1024) - if data == "": + data = proc.stdout.read() + if not data: proc.stdout.close() read_set.remove(proc.stdout) stdout.append(data) + # likewise if proc.stderr in rlist: - data = os.read(proc.stderr.fileno(), 1024) - if data == "": + data = proc.stderr.read() + if not data: proc.stderr.close() read_set.remove(proc.stderr) stderr.append(data) @@ -864,11 +877,13 @@ def _communicate(proc, input, timeout=None, err_on_timeout=True): # object do the translation: It is based on stdio, which is # impossible to combine with select (unless forcing no # buffering). - if proc.universal_newlines and hasattr(file, 'newlines'): - if stdout: - stdout = proc._translate_newlines(stdout) - if stderr: - stderr = proc._translate_newlines(stderr) + # this however seems to make no sense in the context of python3 + if PY2: + if proc.universal_newlines and hasattr(file, 'newlines'): + if stdout: + stdout = proc._translate_newlines(stdout) + if stderr: + stderr = proc._translate_newlines(stderr) if killed and err_on_timeout: errcode = proc.poll() diff --git a/test/execution/scheduler.py b/test/execution/scheduler.py index b36a86bd..a7e3e9ee 100755 --- a/test/execution/scheduler.py +++ b/test/execution/scheduler.py @@ -22,6 +22,8 @@ from nepi.util.timefuncs import tnow, stabsformat import unittest +from six import next + class SchedulerTestCase(unittest.TestCase): def test_task_order(self): def first(): @@ -49,13 +51,13 @@ class SchedulerTestCase(unittest.TestCase): scheduler.schedule(tsk1) # Make sure tasks are retrieved in teh correct order - tsk = scheduler.next() + tsk = next(scheduler) self.assertEqual(tsk.callback(), 1) - tsk = scheduler.next() + tsk = next(scheduler) self.assertEqual(tsk.callback(), 2) - tsk = scheduler.next() + tsk = next(scheduler) self.assertEqual(tsk.callback(), 3) -- 2.43.0