# Alina Quereilhac <alina.quereilhac@inria.fr>\r
#\r
\r
-import ipaddr\r
+from six import PY2, next\r
+if PY2:\r
+ import ipaddr\r
+else:\r
+ import ipaddress\r
from optparse import OptionParser\r
import os\r
from random import randint\r
def build_ns3_topology(ec, simu, node_count, network, prefixlen, agent_ip):\r
channel = add_ns3_wifi_channel(ec)\r
\r
- net = ipaddr.IPv4Network("%s/%s" % (network, prefixlen)) \r
- itr = net.iterhosts()\r
+ if PY2:\r
+ net = ipaddr.IPv4Network("%s/%s" % (network, prefixlen)) \r
+ itr = net.iterhosts()\r
+ else:\r
+ net = ipaddress.IPv4Network("%s/%s" % (network, prefixlen)) \r
+ itr = net.hosts()\r
\r
- ap_ip = itr.next().exploded\r
+ ap_ip = next(itr).exploded\r
ap = add_ns3_node(ec, simu, ap_ip, prefixlen, channel, ap_mode=True)\r
\r
agent = None\r
from __future__ import print_function
+from six.moves import input
+
from nepi.execution.ec import ExperimentController
from optparse import OptionParser, SUPPRESS_HELP
(options, args) = parser.parse_args()
-proceed = raw_input ("Executing this script will modify the fedora yum repositories in the selected PlanetLab hosts. Are you sure to continue? [y/N] ")
+proceed = input ("Executing this script will modify the fedora yum repositories in the selected PlanetLab hosts. Are you sure to continue? [y/N] ")
if proceed.lower() not in ['yes', 'y']:
os._exit(1)
from distutils.core import setup
import sys
+PY2 = sys.version_info[0] == 2
+
with open('VERSION') as f:
version_tag = f.read().strip()
with open("COPYING") as f:
data_files = [ ('/etc/nepi', [ 'VERSION', 'COPYING', 'README' ] ) ]
+### requirements - used by pip install
+required_modules = [ ]
+ # we are now using six for a portable code
+required_modules.append('six')
+ # ipaddr in py2 used to be a separate lib
+ # within recent py3, it is now in standard library but named ipaddress
+if PY2:
+ required_modules.append('ipaddr')
+ # this is required regardless of the python version
+required_modules.append('networkx')
+ # refrain from mentioning these ones that are not exactly crucial
+ # and that have additional, non-python, dependencies
+ # that can easily break the whole install
+#required_modules.append('matplotlib')
+#required_modules.append('pygraphviz')
+
setup(
name = "nepi",
version = version_tag,
"nepi.resources.linux" : [ "scripts/*.py" ],
"nepi.resources.linux.ns3" : [ "dependencies/*.tar.gz" ]
},
- install_requires = [
- "ipaddr",
- "networkx",
- # refrain from mentioning these ones that are not exactly crucial
- # and that have additional, non-python, dependencies
- # that can easily break the whole install
- # "matplotlib",
- # "pygraphviz",
- ]
+ install_requires = required_modules,
)
#
# Author: Alina Quereilhac <alina.quereilhac@inria.fr>
+from six import next
+
from nepi.util import guid
from nepi.util.parallel import ParallelRun
from nepi.util.timefuncs import tnow, tdiffsec, stabsformat, tsformat
"""
# Get next available guid
+ # xxx_next_hiccup
guid = self._guid_generator.next(guid)
# Instantiate RM
new_group = False
if not group:
new_group = True
+ # xxx_next_hiccup
group = self._group_id_generator.next()
if group not in self._groups:
try:
self._cond.acquire()
- task = self._scheduler.next()
+ task = next(self._scheduler)
if not task:
# No task to execute. Wait for a new task to be scheduled.
#
# Author: Alina Quereilhac <alina.quereilhac@inria.fr>
+from six import next
+
import itertools
import heapq
:type task: task
"""
if task.id == None:
- task.id = self._idgen.next()
+ task.id = next(self._idgen)
entry = (task.timestamp, task.id, task)
self._valid.add(task.id)
except:
pass
+ # py3 compat
+ def __next__(self):
+ return self.next()
+
def next(self):
""" Get the next task in the queue by timestamp and arrival order
"""
import threading
import traceback
+from six import PY3
+
# TODO: Unify delays!!
# TODO: Validate outcome of uploads!!
if not dests:
return []
- return dests.values()
+ retcod = dests.values()
+ if PY3: retcod = list(retcod)
+ return retcod
import sys
import uuid
+from six import integer_types, string_types
+
class NetNSWrapper(object):
def __init__(self, loglevel = logging.INFO, enable_dump = False):
super(NetNSWrapper, self).__init__()
result = method(*realargs, **realkwargs)
# If the result is an object (not a base value),
- # then keep track of the object a return the object
+ # then keep track of the object and return the object
# reference (newuuid)
- if not (result is None or type(result) in [
- bool, float, long, str, int]):
+ if result is not None \
+ and not isinstance(result, (bool, float) + integer_types + string_types):
self._objects[newuuid] = result
result = newuuid
from nepi.execution.trace import Trace
from nepi.resources.ns3.ns3base import NS3Base
-import ipaddr
+import sys
+PY2 = sys.version_info[0] == 2
+
+if PY2:
+ import ipaddr
+else:
+ import ipaddress
@clsinit_copy
class NS3BaseNetDevice(NS3Base):
ip = self.get("ip")
prefix = self.get("prefix")
- i = ipaddr.IPAddress(ip)
+ if PY2:
+ i = ipaddr.IPAddress(ip)
+ else:
+ i = ipaddress.ip_address(ip)
if i.version == 4:
# IPv4
ipv4 = self.node.ipv4
#
# Author: Alina Quereilhac <alina.quereilhac@inria.fr>
+from six import PY3
+
from nepi.execution.attribute import Attribute, Flags, Types
from nepi.execution.resource import clsinit_copy
from nepi.resources.ns3.ns3base import NS3Base
@classmethod
def _register_attributes(cls):
+ # stay safe and keep extra list() added by 2to3
+ allowed = WIFI_STANDARDS.keys()
+ if PY3: allowed = list(allowed)
standard = Attribute("Standard", "Wireless standard",
default = "WIFI_PHY_STANDARD_80211a",
- allowed = WIFI_STANDARDS.keys(),
+ allowed = allowed,
type = Types.Enumerate,
flags = Flags.Design)
#
# Author: Alina Quereilhac <alina.quereilhac@inria.fr>
+from six import PY3
+
from nepi.execution.attribute import Attribute, Flags, Types
from nepi.execution.resource import clsinit_copy
from nepi.resources.ns3.ns3base import NS3Base
@classmethod
def _register_attributes(cls):
+ # stay safe and keep extra list() added by 2to3
+ allowed = WIFI_STANDARDS.keys()
+ if PY3: allowed = list(allowed)
standard = Attribute("Standard", "Wireless standard",
default = "WIFI_PHY_STANDARD_80211a",
- allowed = WIFI_STANDARDS.keys(),
+ allowed = allowed,
type = Types.Enumerate,
flags = Flags.Design)
import time
import uuid
+from six import integer_types, string_types
+
SINGLETON = "singleton::"
SIMULATOR_UUID = "singleton::Simulator"
CONFIG_UUID = "singleton::Config"
# If the result is an object (not a base value),
# then keep track of the object a return the object
# reference (newuuid)
- if not (result is None or type(result) in [
- bool, float, long, str, int]):
+ if result is not None \
+ and not isinstance(result, (bool, float) + integer_types + string_types):
self._objects[newuuid] = result
result = newuuid
condition.release()
# contextId is defined as general context
- contextId = long(0xffffffff)
+ # xxx possible distortion when upgrading to python3
+ # really not sure what's the point in this long() business..
+ #contextId = long(0xffffffff)
+ contextId = 0xffffffff
# delay 0 means that the event is expected to execute inmediately
delay = self.ns3.Seconds(0)
import os
import time
import threading
-import xmlrpclib
+
+from six import integer_types, string_types
+from six.moves import xmlrpc_client
def _retry(fn):
def rv(*p, **kw):
self._url = urlpattern % {'hostname':hostname}
if (proxy is not None):
- import urllib2
- class HTTPSProxyTransport(xmlrpclib.Transport):
+ from six.moves.urllib import request as urllib_request
+ class HTTPSProxyTransport(xmlrpc_client.Transport):
def __init__(self, proxy, use_datetime=0):
- opener = urllib2.build_opener(urllib2.ProxyHandler({"https" : proxy}))
- xmlrpclib.Transport.__init__(self, use_datetime)
+ opener = urllib_request.build_opener(urllib2.ProxyHandler({"https" : proxy}))
+ xmlrpc_client.Transport.__init__(self, use_datetime)
self.opener = opener
def request(self, host, handler, request_body, verbose=0):
- req = urllib2.Request('https://%s%s' % (host, handler), request_body)
+ req = urllib_request.Request('https://%s%s' % (host, handler), request_body)
req.add_header('User-agent', self.user_agent)
self.verbose = verbose
return self.parse_response(self.opener.open(req))
@property
def api(self):
# Cannot reuse same proxy in all threads, py2.7 is not threadsafe
- return xmlrpclib.ServerProxy(
+ return xmlrcp_client.ServerProxy(
self._url ,
transport = self._proxy_transport(),
allow_none = True)
try:
# test authorization
network_types = _retry(self.mcapi.GetNetworkTypes)(self.auth)
- except (xmlrpclib.ProtocolError, xmlrpclib.Fault) as e:
+ except (xmlrpc_client.ProtocolError, xmlrpc_client.Fault) as e:
warnings.warn(str(e))
return True
* nodefamily : string, the nodefamily this node should be based upon
* plain : boolean, use plain bootstrapfs image if set (for tests)
"""
- if not isinstance(node, (str, int, long)):
+ if not isinstance(node, integer_types + string_types):
raise ValueError("Node must be either a non-unicode string or an int")
return _retry(self.mcapi.GetNodeFlavour)(self.auth, node)
return _retry(self.mcapi.DeleteSliceFromNodes)(self.auth, slice_id_or_name, node_id_or_hostname)
def start_multicall(self):
- self.threadlocal.mc = xmlrpclib.MultiCall(self.mcapi)
+ self.threadlocal.mc = xmlrpc_client.MultiCall(self.mcapi)
def finish_multicall(self):
mc = self.threadlocal.mc
def __init__(self):
self._last_guid = 0
+ # xxx_next_hiccup - this is used as a plain function, and only in ec.py
def next(self, guid = None):
if guid == None:
guid = self._last_guid + 1
from __future__ import print_function
-import xmlrpclib
+from six.moves import xmlrpc_client
import hashlib
import threading
@property
def api(self):
- return xmlrpclib.Server(self._url, allow_none = True)
+ return xmlrpc_client.Server(self._url, allow_none = True)
def get_session_key(self):
"""
#
# Author: Alina Quereilhac <alina.quereilhac@inria.fr>
-import ipaddr
import networkx
import math
import random
+from six import next, PY2, PY3
+if PY2:
+ import ipaddr
+else:
+ import ipaddress
+
+
class TopologyType:
LINEAR = "linear"
LADDER = "ladder"
return self.topology.node[nid].get(name)
def node_annotations(self, nid):
- return self.topology.node[nid].keys()
+ retcod = self.topology.node[nid].keys()
+ if PY3: retcod = list(retcod)
+ return retcod
def del_node_annotation(self, nid, name):
del self.topology.node[nid][name]
return self.topology.edge[nid1][nid2].get(name)
def edge_annotations(self, nid1, nid2):
- return self.topology.edge[nid1][nid2].keys()
+ retcod = self.topology.edge[nid1][nid2].keys()
+ if PY3: retcod = list(retcod)
+ return retcod
def del_edge_annotation(self, nid1, nid2, name):
del self.topology.edge[nid1][nid2][name]
# Assign IP addresses to host
netblock = "%s/%d" % (network, prefix)
if version == 4:
- net = ipaddr.IPv4Network(netblock)
+ net = ipaddr.IPv4Network(netblock) if PY2 else ipaddress.ip_network(netblock)
new_prefix = 30
elif version == 6:
- net = ipaddr.IPv6Network(netblock)
+ net = ipaddr.IPv6Network(netblock) if PY2 else ipaddress.ip_network(netblock)
new_prefix = 30
else:
raise RuntimeError("Invalid IP version %d" % version)
#### Compute subnets for each link
# get a subnet of base_add with prefix /30
- subnet = sub_itr.next()
+ subnet = next(sub_itr)
mask = subnet.netmask.exploded
network = subnet.network.exploded
prefixlen = subnet.prefixlen
# get host addresses in that subnet
i = subnet.iterhosts()
- addr1 = i.next()
- addr2 = i.next()
+ addr1 = next(i)
+ addr2 = next(i)
ip1 = addr1.exploded
ip2 = addr2.exploded
#
import threading
-import Queue
import traceback
import sys
import os
+from six.moves import queue
+
N_PROCS = None
class WorkerThread(threading.Thread):
self.maxqueue = maxqueue
self.maxthreads = maxthreads
- self.queue = Queue.Queue(self.maxqueue or 0)
+ self.queue = queue.Queue(self.maxqueue or 0)
self.delayed_exceptions = []
if results:
- self.rvqueue = Queue.Queue()
+ self.rvqueue = queue.Queue()
else:
self.rvqueue = None
try:
self.queue.get(block = False)
self.queue.task_done()
- except Queue.Empty:
+ except queue.Empty:
break
def destroy(self):
while True:
try:
yield self.rvqueue.get_nowait()
- except Queue.Empty:
+ except queue.Empty:
self.queue.join()
try:
yield self.rvqueue.get_nowait()
- except Queue.Empty:
+ except queue.Empty:
raise StopIteration
INTEGER = "integer"
DOUBLE = "float"
-def xmlencode(s):
- if isinstance(s, str):
- rv = s.decode("latin1")
- if isinstance(s, datetime.datetime):
- rv = tsformat(s)
- elif not isinstance(s, unicode):
- rv = unicode(s)
- else:
- rv = s
- return rv.replace(u'\x00',u'�')
-
+from six import PY2
+
+if PY2:
+ # xxx old py2 code had a hack, that had 'latin1' hardcoded
+ # as the encoding for 8-byte strings
+ # this is very wrong; I keep it for now
+ # but will probably remove it altogether some day
+ def xmlencode(s):
+ """xml encoder for python2"""
+ if isinstance(s, str):
+ rv = s.decode("latin1")
+ if isinstance(s, datetime.datetime):
+ rv = tsformat(s)
+ elif not isinstance(s, unicode):
+ rv = unicode(s)
+ else:
+ rv = s
+ return rv.replace(u'\x00',u'�')
+else:
+ # use sys.getdefaultencoding() to decode bytes into string
+ def xmlencode(s):
+ """xml encoder for python3"""
+ if isinstance(s, datetime.datetime):
+ rv = tsformat(s)
+ elif isinstance(s, bytes):
+ rv = s.decode(sys.getdefaultencoding())
+ else:
+ rv = s
+ return rv.replace('\x00', '�')
+
def xmldecode(s, cast = str):
- ret = s.replace(u'�',u'\x00').encode("ascii")
- ret = cast(ret)
- if s == "None":
+ if s is None:
return None
+ if PY2:
+ ret = s.replace(u'�', u'\x00').encode("ascii")
+ else:
+ ret = s.replace('�', '\x00')
+ ret = cast(ret)
return ret
def from_type(value):
log.debug("Package sfa-common not installed.\
Could not import sfa.rspecs.rspec and sfa.util.xrn")
-from types import StringTypes, ListType
+from six import string_types
class SfaRSpecProcessing(object):
self.config = config
def make_dict_rec(self, obj):
- if not obj or isinstance(obj, (StringTypes, bool)):
+ if not obj or isinstance(obj, (bool,) + string_types):
return obj
if isinstance(obj, list):
objcopy = []
import time
import tempfile
+from six import PY2
+
_re_inet = re.compile("\d+:\s+(?P<name>[a-z0-9_-]+)\s+inet6?\s+(?P<inet>[a-f0-9.:/]+)\s+(brd\s+[0-9.]+)?.*scope\s+global.*")
logger = logging.getLogger("sshfuncs")
ip = None
if host in ["localhost", "127.0.0.1", "::1"]:
+ extras = {} if PY2 else {'universal_newlines' : True}
p = subprocess.Popen(
"ip -o addr list",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
+ **extras
)
stdout, stderr = p.communicate()
m = _re_inet.findall(stdout)
"""
global OPENSSH_HAS_PERSIST
if OPENSSH_HAS_PERSIST is None:
+ extras = {} if PY2 else {'universal_newlines' : True}
with open("/dev/null") as null:
proc = subprocess.Popen(
["ssh", "-v"],
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
stdin = null,
+ **extras
)
out,err = proc.communicate()
proc.wait()
# display command actually invoked when debug is turned on
message = " ".join( [ "'{}'".format(arg) for arg in args ] )
log("sshfuncs: invoking {}".format(message), logging.DEBUG)
+ extras = {} if PY2 else {'universal_newlines' : True}
# connects to the remote host and starts a remote connection
proc = subprocess.Popen(
args,
stdout = stdout,
stdin = stdin,
stderr = stderr,
+ **extras
)
# attach tempfile object to the process, to make sure the file stays
# alive until the process is finished with it
proc.stdin.close()
write_set.remove(proc.stdin)
+ # xxx possible distortion when upgrading to python3
+ # original py2 version used to do
+ # data = os.read(proc.stdout.fileno(), 1024)
+ # but this would return bytes, so..
if proc.stdout in rlist:
- data = os.read(proc.stdout.fileno(), 1024)
- if data == "":
+ data = proc.stdout.read()
+ if not data:
proc.stdout.close()
read_set.remove(proc.stdout)
stdout.append(data)
+ # likewise
if proc.stderr in rlist:
- data = os.read(proc.stderr.fileno(), 1024)
- if data == "":
+ data = proc.stderr.read()
+ if not data:
proc.stderr.close()
read_set.remove(proc.stderr)
stderr.append(data)
# object do the translation: It is based on stdio, which is
# impossible to combine with select (unless forcing no
# buffering).
- if proc.universal_newlines and hasattr(file, 'newlines'):
- if stdout:
- stdout = proc._translate_newlines(stdout)
- if stderr:
- stderr = proc._translate_newlines(stderr)
+ # this however seems to make no sense in the context of python3
+ if PY2:
+ if proc.universal_newlines and hasattr(file, 'newlines'):
+ if stdout:
+ stdout = proc._translate_newlines(stdout)
+ if stderr:
+ stderr = proc._translate_newlines(stderr)
if killed and err_on_timeout:
errcode = proc.poll()
import unittest
+from six import next
+
class SchedulerTestCase(unittest.TestCase):
def test_task_order(self):
def first():
scheduler.schedule(tsk1)
# Make sure tasks are retrieved in teh correct order
- tsk = scheduler.next()
+ tsk = next(scheduler)
self.assertEqual(tsk.callback(), 1)
- tsk = scheduler.next()
+ tsk = next(scheduler)
self.assertEqual(tsk.callback(), 2)
- tsk = scheduler.next()
+ tsk = next(scheduler)
self.assertEqual(tsk.callback(), 3)
git://git.onelab.eu / nepi.git / commitdiff