From ac9d7574625f3ddfb75ca794eb70eaaf525c08e0 Mon Sep 17 00:00:00 2001
From: Claudio-Daniel Freire <claudio-daniel.freire@inria.fr>
Date: Tue, 3 May 2011 14:20:14 +0200
Subject: [PATCH] Testbed netrefs implemented. Some basic tests added.

---
 src/nepi/core/execute.py       | 181 +++++++++++++++++++--------------
 test/core/integration.py       |  41 +++++++-
 test/lib/mock/metadata_v01.py  |   6 ++
 test/lib/mock2/metadata_v01.py |   6 ++
 4 files changed, 157 insertions(+), 77 deletions(-)

diff --git a/src/nepi/core/execute.py b/src/nepi/core/execute.py
index d7282998..de3e7e16 100644
--- a/src/nepi/core/execute.py
+++ b/src/nepi/core/execute.py
@@ -11,6 +11,7 @@ import re
 import threading
 import ConfigParser
 import os
+import collections
 
 ATTRIBUTE_PATTERN_BASE = re.compile(r"\{#\[(?P<label>[-a-zA-Z0-9._]*)\](?P<expr>(?P<component>\.addr\[[0-9]+\]|\.route\[[0-9]+\]|\.trace\[[0-9]+\]|).\[(?P<attribute>[-a-zA-Z0-9._]*)\])#}")
 ATTRIBUTE_PATTERN_GUID_SUB = r"{#[%(guid)s]%(expr)s#}"
@@ -312,7 +313,8 @@ class ExperimentController(object):
         self._experiment_xml = experiment_xml
         self._testbeds = dict()
         self._deployment_config = dict()
-        self._netrefs = dict()
+        self._netrefs = collections.defaultdict(set)
+        self._testbed_netrefs = collections.defaultdict(set)
         self._cross_data = dict()
         self._root_dir = root_dir
         self._netreffed_testbeds = set()
@@ -341,7 +343,10 @@ class ExperimentController(object):
             thread.join()
 
     def start(self):
-        self._init_testbed_controllers()
+        parser = XmlExperimentParser()
+        data = parser.from_xml_to_data(self._experiment_xml)
+        
+        self._init_testbed_controllers(data)
         
         # persist testbed connection data, for potential recovery
         self._persist_testbed_proxies()
@@ -375,12 +380,12 @@ class ExperimentController(object):
         
         if self._netreffed_testbeds:
             # initally resolve netrefs
-            self.do_netrefs(fail_if_undefined=False)
+            self.do_netrefs(data, fail_if_undefined=False)
             
             # rinse and repeat, for netreffed testbeds
             netreffed_testbeds = set(self._netreffed_testbeds)
 
-            self._init_testbed_controllers()
+            self._init_testbed_controllers(data)
             
             # persist testbed connection data, for potential recovery
             self._persist_testbed_proxies()
@@ -389,7 +394,7 @@ class ExperimentController(object):
             steps_to_configure(self, netreffed_testbeds)
             
         # final netref step, fail if anything's left unresolved
-        self.do_netrefs(fail_if_undefined=True)
+        self.do_netrefs(data, fail_if_undefined=True)
         
         # perform do_configure in parallel for al testbeds
         # (it's internal configuration for each)
@@ -502,60 +507,78 @@ class ExperimentController(object):
         else:
             return component, None
 
-    def do_netrefs(self, fail_if_undefined = False):
-        COMPONENT_GETTERS = {
-            'addr':
-                lambda testbed, guid, index, name: 
-                    testbed.get_address(guid, index, name),
-            'route' :
-                lambda testbed, guid, index, name: 
-                    testbed.get_route(guid, index, name),
-            'trace' :
-                lambda testbed, guid, index, name: 
-                    testbed.trace(guid, index, name),
-            '' : 
-                lambda testbed, guid, index, name: 
-                    testbed.get(guid, name),
-        }
-        
+    _NETREF_COMPONENT_GETTERS = {
+        'addr':
+            lambda testbed, guid, index, name: 
+                testbed.get_address(guid, index, name),
+        'route' :
+            lambda testbed, guid, index, name: 
+                testbed.get_route(guid, index, name),
+        'trace' :
+            lambda testbed, guid, index, name: 
+                testbed.trace(guid, index, name),
+        '' : 
+            lambda testbed, guid, index, name: 
+                testbed.get(guid, name),
+    }
+    
+    def resolve_netref_value(self, value):
+        match = ATTRIBUTE_PATTERN_BASE.search(value)
+        if match:
+            label = match.group("label")
+            if label.startswith('GUID-'):
+                ref_guid = int(label[5:])
+                if ref_guid:
+                    expr = match.group("expr")
+                    component = match.group("component")[1:] # skip the dot
+                    attribute = match.group("attribute")
+                    
+                    # split compound components into component kind and index
+                    # eg: 'addr[0]' -> ('addr', '0')
+                    component, component_index = self._netref_component_split(component)
+                    
+                    # find object and resolve expression
+                    for ref_testbed in self._testbeds.itervalues():
+                        if component not in self._NETREF_COMPONENT_GETTERS:
+                            raise ValueError, "Malformed netref: %r - unknown component" % (expr,)
+                        else:
+                            ref_value = self._NETREF_COMPONENT_GETTERS[component](
+                                ref_testbed, ref_guid, component_index, attribute)
+                            if ref_value:
+                                return value.replace(match.group(), ref_value)
+        # couldn't find value
+        return None
+    
+    def do_netrefs(self, data, fail_if_undefined = False):
+        # element netrefs
         for (testbed_guid, guid), attrs in self._netrefs.iteritems():
             testbed = self._testbeds[testbed_guid]
             for name in attrs:
                 value = testbed.get(guid, name)
                 if isinstance(value, basestring):
-                    match = ATTRIBUTE_PATTERN_BASE.search(value)
-                    if match:
-                        label = match.group("label")
-                        if label.startswith('GUID-'):
-                            ref_guid = int(label[5:])
-                            if ref_guid:
-                                expr = match.group("expr")
-                                component = match.group("component")[1:] # skip the dot
-                                attribute = match.group("attribute")
-                                
-                                # split compound components into component kind and index
-                                # eg: 'addr[0]' -> ('addr', '0')
-                                component, component_index = self._netref_component_split(component)
-                                
-                                # find object and resolve expression
-                                for ref_testbed in self._testbeds.itervalues():
-                                    if component not in COMPONENT_GETTERS:
-                                        raise ValueError, "Malformed netref: %r - unknown component" % (expr,)
-                                    else:
-                                        ref_value = COMPONENT_GETTERS[component](
-                                            ref_testbed, ref_guid, component_index, attribute)
-                                        if ref_value:
-                                            testbed.set(guid, name, 
-                                                    value.replace(match.group(), ref_value))
-                                            break
-                                else:
-                                    # couldn't find value
-                                    if fail_if_undefined:
-                                        raise ValueError, "Unresolvable GUID: %r, in netref: %r" % (ref_guid, expr)
+                    ref_value = self.resolve_netref_value(value)
+                    if ref_value is not None:
+                        testbed.set(guid, name, ref_value)
+                    elif fail_if_undefined:
+                        raise ValueError, "Unresolvable netref in: %r" % (value,)
+        
+        # testbed netrefs
+        for testbed_guid, attrs in self._testbed_netrefs.iteritems():
+            tb_data = dict(data.get_attribute_data(testbed_guid))
+            if data:
+                for name in attrs:
+                    value = tb_data.get(name)
+                    if isinstance(value, basestring):
+                        ref_value = self.resolve_netref_value(value)
+                        if ref_value is not None:
+                            data.set_attribute_data(testbed_guid, name, ref_value)
+                        elif fail_if_undefined:
+                            raise ValueError, "Unresolvable netref in: %r" % (value,)
+        
+        self._netrefs.clear()
+        self._testbed_netrefs.clear()
 
-    def _init_testbed_controllers(self, recover = False):
-        parser = XmlExperimentParser()
-        data = parser.from_xml_to_data(self._experiment_xml)
+    def _init_testbed_controllers(self, data, recover = False):
         blacklist_testbeds = set(self._testbeds)
         element_guids = list()
         label_guids = dict()
@@ -586,30 +609,33 @@ class ExperimentController(object):
 
     def _resolve_labels(self, data, data_guids, label_guids):
         netrefs = self._netrefs
+        testbed_netrefs = self._testbed_netrefs
         for guid in data_guids:
-            if not data.is_testbed_data(guid):
-                for name, value in data.get_attribute_data(guid):
-                    if isinstance(value, basestring):
-                        match = ATTRIBUTE_PATTERN_BASE.search(value)
-                        if match:
-                            label = match.group("label")
-                            if not label.startswith('GUID-'):
-                                ref_guid = label_guids.get(label)
-                                if ref_guid is not None:
-                                    value = ATTRIBUTE_PATTERN_BASE.sub(
-                                        ATTRIBUTE_PATTERN_GUID_SUB % dict(
-                                            guid = 'GUID-%d' % (ref_guid,),
-                                            expr = match.group("expr"),
-                                            label = label), 
-                                        value)
-                                    data.set_attribute_data(guid, name, value)
-                                    
-                                    # memorize which guid-attribute pairs require
-                                    # postprocessing, to avoid excessive controller-testbed
-                                    # communication at configuration time
-                                    # (which could require high-latency network I/O)
+            for name, value in data.get_attribute_data(guid):
+                if isinstance(value, basestring):
+                    match = ATTRIBUTE_PATTERN_BASE.search(value)
+                    if match:
+                        label = match.group("label")
+                        if not label.startswith('GUID-'):
+                            ref_guid = label_guids.get(label)
+                            if ref_guid is not None:
+                                value = ATTRIBUTE_PATTERN_BASE.sub(
+                                    ATTRIBUTE_PATTERN_GUID_SUB % dict(
+                                        guid = 'GUID-%d' % (ref_guid,),
+                                        expr = match.group("expr"),
+                                        label = label), 
+                                    value)
+                                data.set_attribute_data(guid, name, value)
+                                
+                                # memorize which guid-attribute pairs require
+                                # postprocessing, to avoid excessive controller-testbed
+                                # communication at configuration time
+                                # (which could require high-latency network I/O)
+                                if not data.is_testbed_data(guid):
                                     (testbed_guid, factory_id) = data.get_box_data(guid)
-                                    netrefs.setdefault((testbed_guid, guid), set()).add(name)
+                                    netrefs[(testbed_guid, guid)].add(name)
+                                else:
+                                    testbed_netrefs[guid].add(name)
 
     def _create_testbed_controller(self, guid, data, element_guids, recover):
         (testbed_id, testbed_version) = data.get_testbed_data(guid)
@@ -617,7 +643,7 @@ class ExperimentController(object):
         
         if deployment_config is None:
             # need to create one
-            deployment_config = self._deployment_config[guid] = proxy.AccessConfiguration()
+            deployment_config = proxy.AccessConfiguration()
             
             for (name, value) in data.get_attribute_data(guid):
                 if value is not None and deployment_config.has_attribute(name):
@@ -630,6 +656,9 @@ class ExperimentController(object):
                     
                     # copy deployment config attribute
                     deployment_config.set_attribute_value(name, value)
+            
+            # commit config
+            self._deployment_config[guid] = deployment_config
         
         if deployment_config is not None:
             # force recovery mode 
diff --git a/test/core/integration.py b/test/core/integration.py
index 3618c870..f099c08f 100755
--- a/test/core/integration.py
+++ b/test/core/integration.py
@@ -224,7 +224,40 @@ class ExecuteTestCase(unittest.TestCase):
         addr.set_attribute_value("Address", "10.0.0.2")
         iface2.set_attribute_value("test", "{#[some].addr[0].[Address]#}")
         
-        # TODO: Test actual substitution
+        xml = exp_desc.to_xml()
+        access_config = None
+        controller = proxy.create_controller(xml, access_config)
+        controller.start()
+        while not controller.is_finished(app.guid):
+            time.sleep(0.5)
+        fake_result = controller.trace(desc.guid, app.guid, "fake")
+        comp_result = """PING 10.0.0.2 (10.0.0.2) 56(84) bytes of data.
+
+--- 10.0.0.2 ping statistics ---
+1 packets transmitted, 1 received, 0% packet loss, time 0ms
+"""
+        
+        self.assertTrue(fake_result.startswith(comp_result))
+        
+        self.assertEqual(
+            controller._testbeds[desc.guid].get(iface2.guid, "test"),
+            addr.get_attribute_value("Address") )
+        
+        controller.stop()
+        controller.shutdown()
+
+    def test_testbed_reference_expressions(self):
+        exp_desc, desc, app, node1, node2, iface1, iface2 = self.make_test_experiment()
+        
+        iface1.set_attribute_value("label", "some")
+        addr = iface1.add_address()
+        addr.set_attribute_value("Address", "10.0.0.2")
+
+        desc2 = exp_desc.add_testbed_description(
+            FactoriesProvider("mock2", "01") )
+        desc2.set_attribute_value(DC.DEPLOYMENT_HOST, "{#[some].addr[0].[Address]#}")
+        # DC.DEPLOYMENT_HOST should be ignored if DC.DEPLOYMENT_CONNECTION is not set
+        # But it should be resolved anyway
         
         xml = exp_desc.to_xml()
         access_config = None
@@ -239,6 +272,12 @@ class ExecuteTestCase(unittest.TestCase):
 1 packets transmitted, 1 received, 0% packet loss, time 0ms
 """
         self.assertTrue(fake_result.startswith(comp_result))
+
+        self.assertEqual(
+            controller._deployment_config[desc2.guid]
+                .get_attribute_value(DC.DEPLOYMENT_HOST),
+            addr.get_attribute_value("Address") )
+        
         controller.stop()
         controller.shutdown()
 
diff --git a/test/lib/mock/metadata_v01.py b/test/lib/mock/metadata_v01.py
index d12e9ae9..5478c7e6 100644
--- a/test/lib/mock/metadata_v01.py
+++ b/test/lib/mock/metadata_v01.py
@@ -171,6 +171,12 @@ testbed_attributes = dict({
                 "allowed": None,
                 "validation_function": validation.is_bool
             }),
+        "test": dict({
+                "name": "test",
+                "help": "test attribute",
+                "type": Attribute.STRING,
+                "validation_function": validation.is_string
+            }),
     })
 
 class VersionedMetadataInfo(metadata.VersionedMetadataInfo):
diff --git a/test/lib/mock2/metadata_v01.py b/test/lib/mock2/metadata_v01.py
index 672f9ec1..39757bd2 100644
--- a/test/lib/mock2/metadata_v01.py
+++ b/test/lib/mock2/metadata_v01.py
@@ -171,6 +171,12 @@ testbed_attributes = dict({
                 "allowed": None,
                 "validation_function": validation.is_bool
             }),
+        "test": dict({
+                "name": "test",
+                "help": "test attribute",
+                "type": Attribute.STRING,
+                "validation_function": validation.is_string
+            }),
     })
 
 class VersionedMetadataInfo(metadata.VersionedMetadataInfo):
-- 
2.45.2