From adcd0b17587b5adf027f8fd414eb46adde2c50f2 Mon Sep 17 00:00:00 2001
From: Claudio-Daniel Freire <claudio-daniel.freire@inria.fr>
Date: Wed, 13 Apr 2011 10:37:14 +0200
Subject: [PATCH] Ticket #10: netrefs, initial implementation

---
 src/nepi/core/execute.py               | 112 ++++++++++++++++++++++---
 src/nepi/core/testbed_impl.py          | 103 +++++++++++++++++++++--
 src/nepi/testbeds/netns/execute.py     |  37 ++++++--
 src/nepi/testbeds/ns3/execute.py       |  30 +++++--
 src/nepi/testbeds/planetlab/execute.py |  50 +++++------
 test/lib/mock/execute.py               |  26 +++++-
 6 files changed, 293 insertions(+), 65 deletions(-)

diff --git a/src/nepi/core/execute.py b/src/nepi/core/execute.py
index f31f6c13..dea3866c 100644
--- a/src/nepi/core/execute.py
+++ b/src/nepi/core/execute.py
@@ -3,13 +3,14 @@
 
 from nepi.core.attributes import Attribute, AttributesMap
 from nepi.util import proxy, validation
-from nepi.util.constants import STATUS_FINISHED
+from nepi.util.constants import STATUS_FINISHED, TIME_NOW
 from nepi.util.parser._xml import XmlExperimentParser
 import sys
 import re
 
 ATTRIBUTE_PATTERN_BASE = re.compile(r"\{#\[(?P<label>[-a-zA-Z0-9._]*)\](?P<expr>(?P<component>\.addr\[[0-9]+\]|\.route\[[0-9]+\]|\.trace\[[0-9]+\]|).\[(?P<attribute>[-a-zA-Z0-9._]*)\])#}")
 ATTRIBUTE_PATTERN_GUID_SUB = r"{#[%(guid)s]%(expr)s#}"
+COMPONENT_PATTERN = re.compile(r"(?P<kind>[a-z]*)\[(?P<index>.*)\]")
 
 class ConnectorType(object):
     def __init__(self, testbed_id, factory_id, name, max = -1, min = 0):
@@ -252,6 +253,26 @@ class TestbedInstance(object):
 
     def get(self, time, guid, name):
         raise NotImplementedError
+    
+    def get_route(self, guid, index, attribute):
+        """
+        Params:
+            
+            guid: guid of box to query
+            index: number of routing entry to fetch
+            attribute: one of Destination, NextHop, NetPrefix
+        """
+        raise NotImplementedError
+
+    def get_address(self, guid, index, attribute='Address'):
+        """
+        Params:
+            
+            guid: guid of box to query
+            index: number of inteface to select
+            attribute: one of Address, NetPrefix, Broadcast
+        """
+        raise NotImplementedError
 
     def action(self, time, guid, action):
         raise NotImplementedError
@@ -259,7 +280,7 @@ class TestbedInstance(object):
     def status(self, guid):
         raise NotImplementedError
 
-    def trace(self, guid, trace_id):
+    def trace(self, guid, trace_id, attribute='value'):
         raise NotImplementedError
 
     def shutdown(self):
@@ -270,6 +291,7 @@ class ExperimentController(object):
         self._experiment_xml = experiment_xml
         self._testbeds = dict()
         self._access_config = dict()
+        self._netrefs = dict()
 
     @property
     def experiment_xml(self):
@@ -288,6 +310,8 @@ class ExperimentController(object):
         for testbed in self._testbeds.values():
             testbed.do_create()
             testbed.do_connect()
+        self.do_netrefs(fail_if_undefined=True)
+        for testbed in self._testbeds.values():
             testbed.do_configure()
         for testbed in self._testbeds.values():
             testbed.do_cross_connect()
@@ -309,12 +333,70 @@ class ExperimentController(object):
        for testbed in self._testbeds.values():
            testbed.shutdown()
 
+    @staticmethod
+    def _netref_component_split(component):
+        match = COMPONENT_PATTERN.match(component)
+        if match:
+            return match.group("kind"), match.group("index")
+        else:
+            return component, None
+
+    def do_netrefs(self, fail_if_undefined = False):
+        COMPONENT_GETTERS = {
+            'addr' :
+                lambda testbed, guid, index, name : 
+                    testbed.get_address(guid, index, name),
+            'route' :
+                lambda testbed, guid, index, name : 
+                    testbed.get_route(guid, index, name),
+            'trace' :
+                lambda testbed, guid, index, name : 
+                    testbed.trace(guid, index, name),
+            '' : 
+                lambda testbed, guid, index, name : 
+                    testbed.get(TIME_NOW, guid, name),
+        }
+        
+        for (testbed_guid, guid), attrs in self._netrefs.iteritems():
+            testbed = self._testbeds[testbed_guid]
+            for name in attrs:
+                value = testbed.get(TIME_NOW, guid, name)
+                if isinstance(value, basestring):
+                    match = ATTRIBUTE_PATTERN_BASE.search(value)
+                    if match:
+                        label = match.group("label")
+                        if label.startswith('GUID-'):
+                            ref_guid = int(label[5:])
+                            if ref_guid:
+                                expr = match.group("expr")
+                                component = match.group("component")[1:] # skip the dot
+                                attribute = match.group("attribute")
+                                
+                                # split compound components into component kind and index
+                                # eg: 'addr[0]' -> ('addr', '0')
+                                component, component_index = self._netref_component_split(component)
+                                
+                                # find object and resolve expression
+                                for ref_testbed in self._testbeds.itervalues():
+                                    if component not in COMPONENT_GETTERS:
+                                        raise ValueError, "Malformed netref: %r - unknown component" % (expr,)
+                                    else:
+                                        value = COMPONENT_GETTERS[component](
+                                            ref_testbed, ref_guid, component_index, attribute)
+                                        if value: 
+                                            break
+                                else:
+                                    # couldn't find value
+                                    if fail_if_undefined:
+                                        raise ValueError, "Unresolvable GUID: %r, in netref: %r" % (ref_guid, expr)
+
     def _create_testbed_instances(self):
         parser = XmlExperimentParser()
         data = parser.from_xml_to_data(self._experiment_xml)
         element_guids = list()
         label_guids = dict()
         data_guids = data.guids
+        netrefs = self._netrefs
         for guid in data_guids:
             if data.is_testbed_data(guid):
                 (testbed_id, testbed_version) = data.get_testbed_data(guid)
@@ -339,15 +421,23 @@ class ExperimentController(object):
                         match = ATTRIBUTE_PATTERN_BASE.search(value)
                         if match:
                             label = match.group("label")
-                            ref_guid = label_guids.get(label)
-                            if ref_guid is not None:
-                                value = ATTRIBUTE_PATTERN_BASE.sub(
-                                    ATTRIBUTE_PATTERN_GUID_SUB % dict(
-                                        guid=ref_guid,
-                                        expr=match.group("expr"),
-                                        label=label), 
-                                    value)
-                                data.set_attribute_data(guid, name, value)
+                            if not label.startswith('GUID-'):
+                                ref_guid = label_guids.get(label)
+                                if ref_guid is not None:
+                                    value = ATTRIBUTE_PATTERN_BASE.sub(
+                                        ATTRIBUTE_PATTERN_GUID_SUB % dict(
+                                            guid='GUID-%d' % (ref_guid,),
+                                            expr=match.group("expr"),
+                                            label=label), 
+                                        value)
+                                    data.set_attribute_data(guid, name, value)
+                                    
+                                    # memorize which guid-attribute pairs require
+                                    # postprocessing, to avoid excessive controller-testbed
+                                    # communication at configuration time
+                                    # (which could require high-latency network I/O)
+                                    (testbed_guid, factory_id) = data.get_box_data(guid)
+                                    netrefs.setdefault((testbed_guid,guid),set()).add(name)
         self._program_testbed_instances(element_guids, data)
 
     def _program_testbed_instances(self, element_guids, data):
diff --git a/src/nepi/core/testbed_impl.py b/src/nepi/core/testbed_impl.py
index 54df3195..f99466f2 100644
--- a/src/nepi/core/testbed_impl.py
+++ b/src/nepi/core/testbed_impl.py
@@ -167,8 +167,7 @@ class TestbedInstance(execute.TestbedInstance):
             self._add_route[guid] = list()
         self._add_route[guid].append((destination, netprefix, nexthop)) 
 
-    def do_setup(self):
-        raise NotImplementedError
+    #do_setup(self): NotImplementedError
 
     def do_create(self):
         guids = dict()
@@ -259,8 +258,81 @@ class TestbedInstance(execute.TestbedInstance):
             self._set[guid][time] = dict()
         self._set[guid][time][name] = value
 
-    def get(self, time, guid, name):
-        raise NotImplementedError
+    def box_get(self, time, guid, name):
+        """
+        Helper for subclasses, gets an attribute from box definitions
+        if available. Throws KeyError if the GUID wasn't created
+        through the defer_create interface, and AttributeError if the
+        attribute isn't available (doesn't exist or is design-only)
+        """
+        if not guid in self._create:
+            raise KeyError, "Element guid %d doesn't exist" % guid
+        factory_id = self._create[guid]
+        factory = self._factories[factory_id]
+        if not factory.box_attributes.has_attribute(name):
+            raise AttributeError, "Invalid attribute %s for element type %s" % (name, factory_id)
+        if self._started and factory.is_attribute_design_only(name):
+            raise AttributeError, "Attribute %s can only be queried during experiment design" % name
+        return factory.box_attributes.get_attribute_value(name)
+
+    #get: NotImplementedError
+
+    def box_get_route(self, guid, index, attribute):
+        """
+        Helper implementation for get_route, returns information
+        given to defer_add_route.
+        
+        Raises AttributeError if an invalid attribute is requested
+            or if the indexed routing rule does not exist.
+        
+        Raises KeyError if the GUID has not been seen by
+            defer_add_route
+        """
+        ATTRIBUTES = ['Destination', 'NetPrefix', 'NextHop']
+        
+        if attribute not in ATTRIBUTES:
+            raise AttributeError, "Attribute %r invalid for addresses of %r" % (attribute, guid)
+        
+        attribute_index = ATTRIBUTES.index(attribute)
+        
+        routes = self._add_route.get(guid)
+        if not routes:
+            raise KeyError, "GUID %r not found in %s" % (guid, self._testbed_id)
+        
+        if not (0 <= index < len(addresses)):
+            raise AttributeError, "GUID %r at %s does not have a routing entry #%s" % (
+                guid, self._testbed_id, index)
+        
+        return routes[index][attribute_index]
+
+    def box_get_address(self, guid, index, attribute='Address'):
+        """
+        Helper implementation for get_address, returns information
+        given to defer_add_address
+        
+        Raises AttributeError if an invalid attribute is requested
+            or if the indexed routing rule does not exist.
+        
+        Raises KeyError if the GUID has not been seen by
+            defer_add_address
+        """
+        ATTRIBUTES = ['Address', 'NetPrefix', 'Broadcast']
+        
+        if attribute not in ATTRIBUTES:
+            raise AttributeError, "Attribute %r invalid for addresses of %r" % (attribute, guid)
+        
+        attribute_index = ATTRIBUTES.index(attribute)
+        
+        addresses = self._add_address.get(guid)
+        if not addresses:
+            raise KeyError, "GUID %r not found in %s" % (guid, self._testbed_id)
+        
+        if not (0 <= index < len(addresses)):
+            raise AttributeError, "GUID %r at %s does not have an address #%s" % (
+                guid, self._testbed_id, index)
+        
+        return addresses[index][attribute_index]
+
 
     def start(self, time = TIME_NOW):
         for guid, factory_id in self._create.iteritems():
@@ -270,8 +342,7 @@ class TestbedInstance(execute.TestbedInstance):
                 start_function(self, guid)
         self._started = True
 
-    def action(self, time, guid, action):
-        raise NotImplementedError
+    #action: NotImplementedError
 
     def stop(self, time = TIME_NOW):
         for guid, factory_id in self._create.iteritems():
@@ -290,11 +361,25 @@ class TestbedInstance(execute.TestbedInstance):
             return status_function(self, guid)
         return STATUS_UNDETERMINED
 
-    def trace(self, guid, trace_id):
+    def trace(self, guid, trace_id, attribute='value'):
+        if attribute == 'value':
+            fd = open("%s" % self.trace_filename(guid, trace_id), "r")
+            content = fd.read()
+            fd.close()
+        elif attribute == 'path':
+            content = self.trace_filename(guid, trace_id)
+        else:
+            content = None
+        return content
+
+    def trace_filename(self, guid, trace_id):
+        """
+        Return a trace's file path, for TestbedInstance's default 
+        implementation of trace()
+        """
         raise NotImplementedError
 
-    def shutdown(self):
-        raise NotImplementedError
+    #shutdown: NotImplementedError
 
     def get_connected(self, guid, connector_type_name, 
             other_connector_type_name):
diff --git a/src/nepi/testbeds/netns/execute.py b/src/nepi/testbeds/netns/execute.py
index c6fa941a..d1904b00 100644
--- a/src/nepi/testbeds/netns/execute.py
+++ b/src/nepi/testbeds/netns/execute.py
@@ -27,25 +27,44 @@ class TestbedInstance(testbed_impl.TestbedInstance):
 
     def set(self, time, guid, name, value):
         super(TestbedInstance, self).set(time, guid, name, value)
+        
         # TODO: take on account schedule time for the task 
-        element = self._elements[guid]
+        element = self._elements.get(guid)
         if element:
             setattr(element, name, value)
 
     def get(self, time, guid, name):
         # TODO: take on account schedule time for the task
-        element = self._elements[guid]
-        return getattr(element, name)
+        element = self._elements.get(guid)
+        if element:
+            try:
+                if hasattr(element, name):
+                    # Runtime attribute
+                    return getattr(element, name)
+                else:
+                    # Try design-time attributes
+                    return self.box_get(time, guid, name)
+            except KeyError, AttributeError:
+                return None
+
+    def get_route(self, guid, index, attribute):
+        # TODO: fetch real data from netns
+        try:
+            return self.box_get_route(guid, int(index), attribute)
+        except KeyError, AttributeError:
+            return None
+
+    def get_address(self, guid, index, attribute='Address'):
+        # TODO: fetch real data from netns
+        try:
+            return self.box_get_address(guid, int(index), attribute)
+        except KeyError, AttributeError:
+            return None
+
 
     def action(self, time, guid, action):
         raise NotImplementedError
 
-    def trace(self, guid, trace_id):
-        fd = open("%s" % self.trace_filename(guid, trace_id), "r")
-        content = fd.read()
-        fd.close()
-        return content
-
     def shutdown(self):
         for trace in self._traces.values():
             trace.close()
diff --git a/src/nepi/testbeds/ns3/execute.py b/src/nepi/testbeds/ns3/execute.py
index 470f638e..2e7b8a04 100644
--- a/src/nepi/testbeds/ns3/execute.py
+++ b/src/nepi/testbeds/ns3/execute.py
@@ -49,9 +49,12 @@ class TestbedInstance(testbed_impl.TestbedInstance):
         TypeId = self.ns3.TypeId()
         typeid = TypeId.LookupByName(factory_id)
         info = TypeId.AttributeInfo()
-        if not typeid.LookupAttributeByName(name, info):
-            raise RuntimeError("Attribute %s doesn't belong to element %s" \
-                   % (name, factory_id))
+        if not typeid or not typeid.LookupAttributeByName(name, info):
+            try:
+                # Try design-time attributes
+                return self.box_get(time, guid, name)
+            except KeyError, AttributeError:
+                return None
         checker = info.checker
         ns3_value = checker.Create() 
         element = self._elements[guid]
@@ -68,15 +71,24 @@ class TestbedInstance(testbed_impl.TestbedInstance):
             return value == "true"
         return value
 
+    def get_route(self, guid, index, attribute):
+        # TODO: fetch real data from ns3
+        try:
+            return self.box_get_route(guid, int(index), attribute)
+        except KeyError, AttributeError:
+            return None
+
+    def get_address(self, guid, index, attribute='Address'):
+        # TODO: fetch real data from ns3
+        try:
+            return self.box_get_address(guid, int(index), attribute)
+        except KeyError, AttributeError:
+            return None
+
+
     def action(self, time, guid, action):
         raise NotImplementedError
 
-    def trace(self, guid, trace_id):
-        fd = open("%s" % self.trace_filename(guid, trace_id), "r")
-        content = fd.read()
-        fd.close()
-        return content
-
     def trace_filename(self, guid, trace_id):
         # TODO: Need to be defined inside a home!!!! with and experiment id_code
         filename = self._traces[guid][trace_id]
diff --git a/src/nepi/testbeds/planetlab/execute.py b/src/nepi/testbeds/planetlab/execute.py
index c6fa941a..313e51f9 100644
--- a/src/nepi/testbeds/planetlab/execute.py
+++ b/src/nepi/testbeds/planetlab/execute.py
@@ -8,7 +8,6 @@ import os
 class TestbedInstance(testbed_impl.TestbedInstance):
     def __init__(self, testbed_version):
         super(TestbedInstance, self).__init__(TESTBED_ID, testbed_version)
-        self._netns = None
         self._home_directory = None
         self._traces = dict()
 
@@ -16,14 +15,9 @@ class TestbedInstance(testbed_impl.TestbedInstance):
     def home_directory(self):
         return self._home_directory
 
-    @property
-    def netns(self):
-        return self._netns
-
     def do_setup(self):
         self._home_directory = self._attributes.\
             get_attribute_value("homeDirectory")
-        self._netns = self._load_netns_module()
 
     def set(self, time, guid, name, value):
         super(TestbedInstance, self).set(time, guid, name, value)
@@ -34,18 +28,36 @@ class TestbedInstance(testbed_impl.TestbedInstance):
 
     def get(self, time, guid, name):
         # TODO: take on account schedule time for the task
-        element = self._elements[guid]
-        return getattr(element, name)
+        element = self._elements.get(guid)
+        if element:
+            try:
+                if hasattr(element, name):
+                    # Runtime attribute
+                    return getattr(element, name)
+                else:
+                    # Try design-time attributes
+                    return self.box_get(time, guid, name)
+            except KeyError, AttributeError:
+                return None
+
+    def get_route(self, guid, index, attribute):
+        # TODO: fetch real data from planetlab
+        try:
+            return self.box_get_route(guid, int(index), attribute)
+        except KeyError, AttributeError:
+            return None
+
+    def get_address(self, guid, index, attribute='Address'):
+        # TODO: fetch real data from planetlab
+        try:
+            return self.box_get_address(guid, int(index), attribute)
+        except KeyError, AttributeError:
+            return None
+
 
     def action(self, time, guid, action):
         raise NotImplementedError
 
-    def trace(self, guid, trace_id):
-        fd = open("%s" % self.trace_filename(guid, trace_id), "r")
-        content = fd.read()
-        fd.close()
-        return content
-
     def shutdown(self):
         for trace in self._traces.values():
             trace.close()
@@ -59,14 +71,4 @@ class TestbedInstance(testbed_impl.TestbedInstance):
     def follow_trace(self, trace_id, trace):
         self._traces[trace_id] = trace
 
-    def _load_netns_module(self):
-        # TODO: Do something with the configuration!!!
-        import sys
-        __import__("netns")
-        netns_mod = sys.modules["netns"]
-        # enable debug
-        enable_debug = self._attributes.get_attribute_value("enableDebug")
-        if enable_debug:
-            netns_mod.environ.set_log_level(netns_mod.environ.LOG_DEBUG)
-        return netns_mod
 
diff --git a/test/lib/mock/execute.py b/test/lib/mock/execute.py
index 1244dbd5..3abdeb9e 100644
--- a/test/lib/mock/execute.py
+++ b/test/lib/mock/execute.py
@@ -18,17 +18,37 @@ class TestbedInstance(testbed_impl.TestbedInstance):
         super(TestbedInstance, self).set(time, guid, name, value)
 
     def get(self, time, guid, name):
-        return True 
+        try:
+            return self.box_get(time, guid, name)
+        except KeyError, AttributeError:
+            return None
+
+    def get_route(self, guid, index, attribute):
+        try:
+            return self.box_get_route(guid, int(index), attribute)
+        except KeyError, AttributeError:
+            return None
+
+    def get_address(self, guid, index, attribute='Address'):
+        try:
+            return self.box_get_address(guid, int(index), attribute)
+        except KeyError, AttributeError:
+            return None
 
     def action(self, time, guid, action):
         raise NotImplementedError
 
-    def trace(self, guid, trace_id):
-        return """PING 10.0.0.2 (10.0.0.2) 56(84) bytes of data.
+    def trace(self, guid, trace_id, attribute='value'):
+        if attribute == 'value':
+            return """PING 10.0.0.2 (10.0.0.2) 56(84) bytes of data.
 
 --- 10.0.0.2 ping statistics ---
 1 packets transmitted, 1 received, 0% packet loss, time 0ms
 """
+        elif attribute == 'path':
+            return '<test>'
+        else:
+            return None
 
     def shutdown(self):
 	    pass
-- 
2.43.0