src/nepi/core/execute.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from nepi.core.attributes import Attribute, AttributesMap
   5 from nepi.core.connector import ConnectorTypeBase
   6 from nepi.util import validation
   7 from nepi.util.constants import STATUS_FINISHED, TIME_NOW
   8 from nepi.util.parser._xml import XmlExperimentParser
   9 import sys
  10 import re
  11 import threading
  12 import ConfigParser
  13 import os
  14 import collections
  15 import functools
  16
  17 ATTRIBUTE_PATTERN_BASE = re.compile(r"\{#\[(?P<label>[-a-zA-Z0-9._]*)\](?P<expr>(?P<component>\.addr\[[0-9]+\]|\.route\[[0-9]+\]|\.trace\[[0-9]+\])?.\[(?P<attribute>[-a-zA-Z0-9._]*)\])#}")
  18 ATTRIBUTE_PATTERN_GUID_SUB = r"{#[%(guid)s]%(expr)s#}"
  19 COMPONENT_PATTERN = re.compile(r"(?P<kind>[a-z]*)\[(?P<index>.*)\]")
  20
  21 class ConnectorType(ConnectorTypeBase):
  22     def __init__(self, testbed_id, factory_id, name, max = -1, min = 0):
  23         super(ConnectorType, self).__init__(testbed_id, factory_id, name, max, min)
  24         # from_connections -- connections where the other connector is the "From"
  25         # to_connections -- connections where the other connector is the "To"
  26         # keys in the dictionary correspond to the
  27         # connector_type_id for possible connections. The value is a tuple:
  28         # (can_cross, connect)
  29         # can_cross: indicates if the connection is allowed accros different
  30         #    testbed instances
  31         # code: is the connection function to be invoked when the elements
  32         #    are connected
  33         self._from_connections = dict()
  34         self._to_connections = dict()
  35
  36     def add_from_connection(self, testbed_id, factory_id, name, can_cross,
  37             init_code, compl_code):
  38         type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  39         self._from_connections[type_id] = (can_cross, init_code, compl_code)
  40
  41     def add_to_connection(self, testbed_id, factory_id, name, can_cross,
  42             init_code, compl_code):
  43         type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  44         self._to_connections[type_id] = (can_cross, init_code, compl_code)
  45
  46     def can_connect(self, testbed_id, factory_id, name, count,
  47             must_cross):
  48         connector_type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  49         for lookup_type_id in self._type_resolution_order(connector_type_id):
  50             if lookup_type_id in self._from_connections:
  51                 (can_cross, init_code, compl_code) = self._from_connections[lookup_type_id]
  52             elif lookup_type_id in self._to_connections:
  53                 (can_cross, init_code, compl_code) = self._to_connections[lookup_type_id]
  54             else:
  55                 # keep trying
  56                 continue
  57             return not must_cross or can_cross
  58         else:
  59             return False
  60
  61     def _connect_to_code(self, testbed_id, factory_id, name,
  62             must_cross):
  63         connector_type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  64         for lookup_type_id in self._type_resolution_order(connector_type_id):
  65             if lookup_type_id in self._to_connections:
  66                 (can_cross, init_code, compl_code) = self._to_connections[lookup_type_id]
  67                 if not must_cross or can_cross:
  68                     return (init_code, compl_code)
  69         else:
  70             return (False, False)
  71
  72     def connect_to_init_code(self, testbed_id, factory_id, name, must_cross):
  73         return self._connect_to_code(testbed_id, factory_id, name, must_cross)[0]
  74
  75     def connect_to_compl_code(self, testbed_id, factory_id, name, must_cross):
  76         return self._connect_to_code(testbed_id, factory_id, name, must_cross)[1]
  77
  78 class Factory(AttributesMap):
  79     def __init__(self, factory_id, create_function, start_function,
  80             stop_function, status_function,
  81             configure_function, preconfigure_function,
  82             prestart_function,
  83             allow_addresses = False, has_addresses = False,
  84             allow_routes = False, has_routes = False):
  85         super(Factory, self).__init__()
  86         self._factory_id = factory_id
  87         self._allow_addresses = bool(allow_addresses)
  88         self._allow_routes = bool(allow_routes)
  89         self._has_addresses = bool(has_addresses) or self._allow_addresses
  90         self._has_routes = bool(has_routes) or self._allow_routes
  91         self._create_function = create_function
  92         self._start_function = start_function
  93         self._stop_function = stop_function
  94         self._status_function = status_function
  95         self._configure_function = configure_function
  96         self._preconfigure_function = preconfigure_function
  97         self._prestart_function = prestart_function
  98         self._connector_types = dict()
  99         self._traces = list()
 100         self._box_attributes = AttributesMap()
 101
 102     @property
 103     def factory_id(self):
 104         return self._factory_id
 105
 106     @property
 107     def allow_addresses(self):
 108         return self._allow_addresses
 109
 110     @property
 111     def allow_routes(self):
 112         return self._allow_routes
 113
 114     @property
 115     def has_addresses(self):
 116         return self._has_addresses
 117
 118     @property
 119     def has_routes(self):
 120         return self._has_routes
 121
 122     @property
 123     def box_attributes(self):
 124         return self._box_attributes
 125
 126     @property
 127     def create_function(self):
 128         return self._create_function
 129
 130     @property
 131     def prestart_function(self):
 132         return self._prestart_function
 133
 134     @property
 135     def start_function(self):
 136         return self._start_function
 137
 138     @property
 139     def stop_function(self):
 140         return self._stop_function
 141
 142     @property
 143     def status_function(self):
 144         return self._status_function
 145
 146     @property
 147     def configure_function(self):
 148         return self._configure_function
 149
 150     @property
 151     def preconfigure_function(self):
 152         return self._preconfigure_function
 153
 154     @property
 155     def traces(self):
 156         return self._traces
 157
 158     def connector_type(self, name):
 159         return self._connector_types[name]
 160
 161     def add_connector_type(self, connector_type):
 162         self._connector_types[connector_type.name] = connector_type
 163
 164     def add_trace(self, trace_id):
 165         self._traces.append(trace_id)
 166
 167     def add_box_attribute(self, name, help, type, value = None, range = None,
 168         allowed = None, flags = Attribute.NoFlags, validation_function = None):
 169         self._box_attributes.add_attribute(name, help, type, value, range,
 170                 allowed, flags, validation_function)
 171
 172 class TestbedController(object):
 173     def __init__(self, testbed_id, testbed_version):
 174         self._testbed_id = testbed_id
 175         self._testbed_version = testbed_version
 176
 177     @property
 178     def testbed_id(self):
 179         return self._testbed_id
 180
 181     @property
 182     def testbed_version(self):
 183         return self._testbed_version
 184
 185     @property
 186     def guids(self):
 187         raise NotImplementedError
 188
 189     def defer_configure(self, name, value):
 190         """Instructs setting a configuartion attribute for the testbed instance"""
 191         raise NotImplementedError
 192
 193     def defer_create(self, guid, factory_id):
 194         """Instructs creation of element """
 195         raise NotImplementedError
 196
 197     def defer_create_set(self, guid, name, value):
 198         """Instructs setting an initial attribute on an element"""
 199         raise NotImplementedError
 200
 201     def defer_factory_set(self, guid, name, value):
 202         """Instructs setting an attribute on a factory"""
 203         raise NotImplementedError
 204
 205     def defer_connect(self, guid1, connector_type_name1, guid2,
 206             connector_type_name2):
 207         """Instructs creation of a connection between the given connectors"""
 208         raise NotImplementedError
 209
 210     def defer_cross_connect(self,
 211             guid, connector_type_name,
 212             cross_guid, cross_testbed_guid,
 213             cross_testbed_id, cross_factory_id,
 214             cross_connector_type_name):
 215         """
 216         Instructs creation of a connection between the given connectors
 217         of different testbed instances
 218         """
 219         raise NotImplementedError
 220
 221     def defer_add_trace(self, guid, trace_id):
 222         """Instructs the addition of a trace"""
 223         raise NotImplementedError
 224
 225     def defer_add_address(self, guid, address, netprefix, broadcast):
 226         """Instructs the addition of an address"""
 227         raise NotImplementedError
 228
 229     def defer_add_route(self, guid, destination, netprefix, nexthop):
 230         """Instructs the addition of a route"""
 231         raise NotImplementedError
 232
 233     def do_setup(self):
 234         """After do_setup the testbed initial configuration is done"""
 235         raise NotImplementedError
 236
 237     def do_create(self):
 238         """
 239         After do_create all instructed elements are created and
 240         attributes setted
 241         """
 242         raise NotImplementedError
 243
 244     def do_connect_init(self):
 245         """
 246         After do_connect_init all internal connections between testbed elements
 247         are initiated
 248         """
 249         raise NotImplementedError
 250
 251     def do_connect_compl(self):
 252         """
 253         After do_connect all internal connections between testbed elements
 254         are completed
 255         """
 256         raise NotImplementedError
 257
 258     def do_preconfigure(self):
 259         """
 260         Done just before resolving netrefs, after connection, before cross connections,
 261         useful for early stages of configuration, for setting up stuff that might be
 262         required for netref resolution.
 263         """
 264         raise NotImplementedError
 265
 266     def do_configure(self):
 267         """After do_configure elements are configured"""
 268         raise NotImplementedError
 269
 270     def do_prestart(self):
 271         """Before do_start elements are prestart-configured"""
 272         raise NotImplementedError
 273
 274     def do_cross_connect_init(self, cross_data):
 275         """
 276         After do_cross_connect_init initiation of all external connections
 277         between different testbed elements is performed
 278         """
 279         raise NotImplementedError
 280
 281     def do_cross_connect_compl(self, cross_data):
 282         """
 283         After do_cross_connect_compl completion of all external connections
 284         between different testbed elements is performed
 285         """
 286         raise NotImplementedError
 287
 288     def start(self):
 289         raise NotImplementedError
 290
 291     def stop(self):
 292         raise NotImplementedError
 293
 294     def set(self, guid, name, value, time = TIME_NOW):
 295         raise NotImplementedError
 296
 297     def get(self, guid, name, time = TIME_NOW):
 298         raise NotImplementedError
 299
 300     def get_route(self, guid, index, attribute):
 301         """
 302         Params:
 303
 304             guid: guid of box to query
 305             index: number of routing entry to fetch
 306             attribute: one of Destination, NextHop, NetPrefix
 307         """
 308         raise NotImplementedError
 309
 310     def get_address(self, guid, index, attribute='Address'):
 311         """
 312         Params:
 313
 314             guid: guid of box to query
 315             index: number of inteface to select
 316             attribute: one of Address, NetPrefix, Broadcast
 317         """
 318         raise NotImplementedError
 319
 320     def get_attribute_list(self, guid):
 321         raise NotImplementedError
 322
 323     def action(self, time, guid, action):
 324         raise NotImplementedError
 325
 326     def status(self, guid):
 327         raise NotImplementedError
 328
 329     def trace(self, guid, trace_id, attribute='value'):
 330         raise NotImplementedError
 331
 332     def shutdown(self):
 333         raise NotImplementedError
 334
 335 class ExperimentController(object):
 336     def __init__(self, experiment_xml, root_dir):
 337         self._experiment_xml = experiment_xml
 338         self._testbeds = dict()
 339         self._deployment_config = dict()
 340         self._netrefs = collections.defaultdict(set)
 341         self._testbed_netrefs = collections.defaultdict(set)
 342         self._cross_data = dict()
 343         self._root_dir = root_dir
 344         self._netreffed_testbeds = set()
 345         self._guids_in_testbed_cache = dict()
 346
 347         self.persist_experiment_xml()
 348
 349     @property
 350     def experiment_xml(self):
 351         return self._experiment_xml
 352
 353     def persist_experiment_xml(self):
 354         xml_path = os.path.join(self._root_dir, "experiment.xml")
 355         f = open(xml_path, "w")
 356         f.write(self._experiment_xml)
 357         f.close()
 358
 359     def trace(self, testbed_guid, guid, trace_id, attribute='value'):
 360         return self._testbeds[testbed_guid].trace(guid, trace_id, attribute)
 361
 362     @staticmethod
 363     def _parallel(callables):
 364         excs = []
 365         def wrap(callable):
 366             @functools.wraps(callable)
 367             def wrapped(*p, **kw):
 368                 try:
 369                     callable(*p, **kw)
 370                 except:
 371                     import traceback
 372                     traceback.print_exc(file=sys.stderr)
 373                     excs.append(sys.exc_info())
 374             return wrapped
 375         threads = [ threading.Thread(target=wrap(callable)) for callable in callables ]
 376         for thread in threads:
 377             thread.start()
 378         for thread in threads:
 379             thread.join()
 380         for exc in excs:
 381             eTyp, eVal, eLoc = exc
 382             raise eTyp, eVal, eLoc
 383
 384     def start(self):
 385         parser = XmlExperimentParser()
 386         data = parser.from_xml_to_data(self._experiment_xml)
 387
 388         self._init_testbed_controllers(data)
 389
 390         # persist testbed connection data, for potential recovery
 391         self._persist_testbed_proxies()
 392
 393         def steps_to_configure(self, allowed_guids):
 394             # perform setup in parallel for all test beds,
 395             # wait for all threads to finish
 396             self._parallel([testbed.do_setup
 397                             for guid,testbed in self._testbeds.iteritems()
 398                             if guid in allowed_guids])
 399
 400             # perform create-connect in parallel, wait
 401             # (internal connections only)
 402             self._parallel([testbed.do_create
 403                             for guid,testbed in self._testbeds.iteritems()
 404                             if guid in allowed_guids])
 405
 406             self._parallel([testbed.do_connect_init
 407                             for guid,testbed in self._testbeds.iteritems()
 408                             if guid in allowed_guids])
 409
 410             self._parallel([testbed.do_connect_compl
 411                             for guid,testbed in self._testbeds.iteritems()
 412                             if guid in allowed_guids])
 413
 414             self._parallel([testbed.do_preconfigure
 415                             for guid,testbed in self._testbeds.iteritems()
 416                             if guid in allowed_guids])
 417
 418         steps_to_configure(self, self._testbeds)
 419
 420         if self._netreffed_testbeds:
 421             # initally resolve netrefs
 422             self.do_netrefs(data, fail_if_undefined=False)
 423
 424             # rinse and repeat, for netreffed testbeds
 425             netreffed_testbeds = set(self._netreffed_testbeds)
 426
 427             self._init_testbed_controllers(data)
 428
 429             # persist testbed connection data, for potential recovery
 430             self._persist_testbed_proxies()
 431
 432             # configure dependant testbeds
 433             steps_to_configure(self, netreffed_testbeds)
 434
 435         # final netref step, fail if anything's left unresolved
 436         self.do_netrefs(data, fail_if_undefined=True)
 437
 438         self._program_testbed_cross_connections(data)
 439
 440         # perform do_configure in parallel for al testbeds
 441         # (it's internal configuration for each)
 442         self._parallel([testbed.do_configure
 443                         for testbed in self._testbeds.itervalues()])
 444
 445
 446         #print >>sys.stderr, "DO IT"
 447         #import time
 448         #time.sleep(60)
 449
 450         # cross-connect (cannot be done in parallel)
 451         for guid, testbed in self._testbeds.iteritems():
 452             cross_data = self._get_cross_data(guid)
 453             testbed.do_cross_connect_init(cross_data)
 454         for guid, testbed in self._testbeds.iteritems():
 455             cross_data = self._get_cross_data(guid)
 456             testbed.do_cross_connect_compl(cross_data)
 457
 458         # Last chance to configure (parallel on all testbeds)
 459         self._parallel([testbed.do_prestart
 460                         for testbed in self._testbeds.itervalues()])
 461
 462         # start experiment (parallel start on all testbeds)
 463         self._parallel([testbed.start
 464                         for testbed in self._testbeds.itervalues()])
 465
 466     def _persist_testbed_proxies(self):
 467         TRANSIENT = ('Recover',)
 468
 469         # persist access configuration for all testbeds, so that
 470         # recovery mode can reconnect to them if it becomes necessary
 471         conf = ConfigParser.RawConfigParser()
 472         for testbed_guid, testbed_config in self._deployment_config.iteritems():
 473             testbed_guid = str(testbed_guid)
 474             conf.add_section(testbed_guid)
 475             for attr in testbed_config.attributes_list:
 476                 if attr not in TRANSIENT:
 477                     conf.set(testbed_guid, attr,
 478                         testbed_config.get_attribute_value(attr))
 479
 480         f = open(os.path.join(self._root_dir, 'deployment_config.ini'), 'w')
 481         conf.write(f)
 482         f.close()
 483
 484     def _load_testbed_proxies(self):
 485         TYPEMAP = {
 486             STRING : 'get',
 487             INTEGER : 'getint',
 488             FLOAT : 'getfloat',
 489             BOOLEAN : 'getboolean',
 490         }
 491
 492         # deferred import because proxy needs
 493         # our class definitions to define proxies
 494         import nepi.util.proxy as proxy
 495
 496         conf = ConfigParser.RawConfigParser()
 497         conf.read(os.path.join(self._root_dir, 'deployment_config.ini'))
 498         for testbed_guid in conf.sections():
 499             testbed_config = proxy.AccessConfiguration()
 500             for attr in conf.options(testbed_guid):
 501                 testbed_config.set_attribute_value(attr,
 502                     conf.get(testbed_guid, attr) )
 503
 504             testbed_guid = str(testbed_guid)
 505             conf.add_section(testbed_guid)
 506             for attr in testbed_config.attributes_list:
 507                 if attr not in TRANSIENT:
 508                     getter = getattr(conf, TYPEMAP.get(
 509                         testbed_config.get_attribute_type(attr),
 510                         'get') )
 511                     testbed_config.set_attribute_value(
 512                         testbed_guid, attr, getter(attr))
 513
 514     def _unpersist_testbed_proxies(self):
 515         try:
 516             os.remove(os.path.join(self._root_dir, 'deployment_config.ini'))
 517         except:
 518             # Just print exceptions, this is just cleanup
 519             import traceback
 520             traceback.print_exc(file=sys.stderr)
 521
 522     def stop(self):
 523        for testbed in self._testbeds.values():
 524            testbed.stop()
 525        self._unpersist_testbed_proxies()
 526
 527     def recover(self):
 528         # reload perviously persisted testbed access configurations
 529         self._load_testbed_proxies()
 530
 531         # recreate testbed proxies by reconnecting only
 532         self._init_testbed_controllers(recover = True)
 533
 534         # another time, for netrefs
 535         self._init_testbed_controllers(recover = True)
 536
 537     def is_finished(self, guid):
 538         for testbed in self._testbeds.values():
 539             if guid in testbed.guids:
 540                 return testbed.status(guid) == STATUS_FINISHED
 541         raise RuntimeError("No element exists with guid %d" % guid)
 542
 543     def set(self, testbed_guid, guid, name, value, time = TIME_NOW):
 544         testbed = self._testbeds[testbed_guid]
 545         testbed.set(guid, name, value, time)
 546
 547     def get(self, testbed_guid, guid, name, time = TIME_NOW):
 548         testbed = self._testbeds[testbed_guid]
 549         return testbed.get(guid, name, time)
 550
 551     def shutdown(self):
 552         for testbed in self._testbeds.values():
 553             testbed.shutdown()
 554
 555     def _guids_in_testbed(self, testbed_guid):
 556         if testbed_guid not in self._testbeds:
 557             return set()
 558         if testbed_guid not in self._guids_in_testbed_cache:
 559             self._guids_in_testbed_cache[testbed_guid] = \
 560                 set(self._testbeds[testbed_guid].guids)
 561         return self._guids_in_testbed_cache[testbed_guid]
 562
 563     @staticmethod
 564     def _netref_component_split(component):
 565         match = COMPONENT_PATTERN.match(component)
 566         if match:
 567             return match.group("kind"), match.group("index")
 568         else:
 569             return component, None
 570
 571     _NETREF_COMPONENT_GETTERS = {
 572         'addr':
 573             lambda testbed, guid, index, name:
 574                 testbed.get_address(guid, int(index), name),
 575         'route' :
 576             lambda testbed, guid, index, name:
 577                 testbed.get_route(guid, int(index), name),
 578         'trace' :
 579             lambda testbed, guid, index, name:
 580                 testbed.trace(guid, index, name),
 581         '' :
 582             lambda testbed, guid, index, name:
 583                 testbed.get(guid, name),
 584     }
 585
 586     def resolve_netref_value(self, value, failval = None):
 587         match = ATTRIBUTE_PATTERN_BASE.search(value)
 588         if match:
 589             label = match.group("label")
 590             if label.startswith('GUID-'):
 591                 ref_guid = int(label[5:])
 592                 if ref_guid:
 593                     expr = match.group("expr")
 594                     component = (match.group("component") or "")[1:] # skip the dot
 595                     attribute = match.group("attribute")
 596
 597                     # split compound components into component kind and index
 598                     # eg: 'addr[0]' -> ('addr', '0')
 599                     component, component_index = self._netref_component_split(component)
 600
 601                     # find object and resolve expression
 602                     for ref_testbed_guid, ref_testbed in self._testbeds.iteritems():
 603                         if component not in self._NETREF_COMPONENT_GETTERS:
 604                             raise ValueError, "Malformed netref: %r - unknown component" % (expr,)
 605                         elif ref_guid not in self._guids_in_testbed(ref_testbed_guid):
 606                             pass
 607                         else:
 608                             ref_value = self._NETREF_COMPONENT_GETTERS[component](
 609                                 ref_testbed, ref_guid, component_index, attribute)
 610                             if ref_value:
 611                                 return value.replace(match.group(), ref_value)
 612         # couldn't find value
 613         return failval
 614
 615     def do_netrefs(self, data, fail_if_undefined = False):
 616         # element netrefs
 617         for (testbed_guid, guid), attrs in self._netrefs.items():
 618             testbed = self._testbeds.get(testbed_guid)
 619             if testbed is not None:
 620                 for name in set(attrs):
 621                     value = testbed.get(guid, name)
 622                     if isinstance(value, basestring):
 623                         ref_value = self.resolve_netref_value(value)
 624                         if ref_value is not None:
 625                             testbed.set(guid, name, ref_value)
 626                             attrs.remove(name)
 627                         elif fail_if_undefined:
 628                             raise ValueError, "Unresolvable netref in: %r=%r" % (name,value,)
 629                 if not attrs:
 630                     del self._netrefs[(testbed_guid, guid)]
 631
 632         # testbed netrefs
 633         for testbed_guid, attrs in self._testbed_netrefs.items():
 634             tb_data = dict(data.get_attribute_data(testbed_guid))
 635             if data:
 636                 for name in set(attrs):
 637                     value = tb_data.get(name)
 638                     if isinstance(value, basestring):
 639                         ref_value = self.resolve_netref_value(value)
 640                         if ref_value is not None:
 641                             data.set_attribute_data(testbed_guid, name, ref_value)
 642                             attrs.remove(name)
 643                         elif fail_if_undefined:
 644                             raise ValueError, "Unresolvable netref in: %r" % (value,)
 645                 if not attrs:
 646                     del self._testbed_netrefs[testbed_guid]
 647
 648
 649     def _init_testbed_controllers(self, data, recover = False):
 650         blacklist_testbeds = set(self._testbeds)
 651         element_guids = list()
 652         label_guids = dict()
 653         data_guids = data.guids
 654
 655         # create testbed controllers
 656         for guid in data_guids:
 657             if data.is_testbed_data(guid):
 658                 if guid not in self._testbeds:
 659                     self._create_testbed_controller(guid, data, element_guids,
 660                             recover)
 661             else:
 662                 (testbed_guid, factory_id) = data.get_box_data(guid)
 663                 if testbed_guid not in blacklist_testbeds:
 664                     element_guids.append(guid)
 665                     label = data.get_attribute_data(guid, "label")
 666                     if label is not None:
 667                         if label in label_guids:
 668                             raise RuntimeError, "Label %r is not unique" % (label,)
 669                         label_guids[label] = guid
 670
 671         # replace references to elements labels for its guid
 672         self._resolve_labels(data, data_guids, label_guids)
 673
 674         # program testbed controllers
 675         if not recover:
 676             self._program_testbed_controllers(element_guids, data)
 677
 678     def _resolve_labels(self, data, data_guids, label_guids):
 679         netrefs = self._netrefs
 680         testbed_netrefs = self._testbed_netrefs
 681         for guid in data_guids:
 682             for name, value in data.get_attribute_data(guid):
 683                 if isinstance(value, basestring):
 684                     match = ATTRIBUTE_PATTERN_BASE.search(value)
 685                     if match:
 686                         label = match.group("label")
 687                         if not label.startswith('GUID-'):
 688                             ref_guid = label_guids.get(label)
 689                             if ref_guid is not None:
 690                                 value = ATTRIBUTE_PATTERN_BASE.sub(
 691                                     ATTRIBUTE_PATTERN_GUID_SUB % dict(
 692                                         guid = 'GUID-%d' % (ref_guid,),
 693                                         expr = match.group("expr"),
 694                                         label = label),
 695                                     value)
 696                                 data.set_attribute_data(guid, name, value)
 697
 698                                 # memorize which guid-attribute pairs require
 699                                 # postprocessing, to avoid excessive controller-testbed
 700                                 # communication at configuration time
 701                                 # (which could require high-latency network I/O)
 702                                 if not data.is_testbed_data(guid):
 703                                     (testbed_guid, factory_id) = data.get_box_data(guid)
 704                                     netrefs[(testbed_guid, guid)].add(name)
 705                                 else:
 706                                     testbed_netrefs[guid].add(name)
 707
 708     def _create_testbed_controller(self, guid, data, element_guids, recover):
 709         (testbed_id, testbed_version) = data.get_testbed_data(guid)
 710         deployment_config = self._deployment_config.get(guid)
 711
 712         # deferred import because proxy needs
 713         # our class definitions to define proxies
 714         import nepi.util.proxy as proxy
 715
 716         if deployment_config is None:
 717             # need to create one
 718             deployment_config = proxy.AccessConfiguration()
 719
 720             for (name, value) in data.get_attribute_data(guid):
 721                 if value is not None and deployment_config.has_attribute(name):
 722                     # if any deployment config attribute has a netref, we can't
 723                     # create this controller yet
 724                     if isinstance(value, basestring) and ATTRIBUTE_PATTERN_BASE.search(value):
 725                         # remember to re-issue this one
 726                         self._netreffed_testbeds.add(guid)
 727                         return
 728
 729                     # copy deployment config attribute
 730                     deployment_config.set_attribute_value(name, value)
 731
 732             # commit config
 733             self._deployment_config[guid] = deployment_config
 734
 735         if deployment_config is not None:
 736             # force recovery mode
 737             deployment_config.set_attribute_value("recover",recover)
 738
 739         testbed = proxy.create_testbed_controller(testbed_id,
 740                 testbed_version, deployment_config)
 741         for (name, value) in data.get_attribute_data(guid):
 742             testbed.defer_configure(name, value)
 743         self._testbeds[guid] = testbed
 744         if guid in self._netreffed_testbeds:
 745             self._netreffed_testbeds.remove(guid)
 746
 747     def _program_testbed_controllers(self, element_guids, data):
 748         for guid in element_guids:
 749             (testbed_guid, factory_id) = data.get_box_data(guid)
 750             testbed = self._testbeds.get(testbed_guid)
 751             if testbed:
 752                 testbed.defer_create(guid, factory_id)
 753                 for (name, value) in data.get_attribute_data(guid):
 754                     # Try to resolve create-time netrefs, if possible
 755                     if isinstance(value, basestring) and ATTRIBUTE_PATTERN_BASE.search(value):
 756                         try:
 757                             nuvalue = self.resolve_netref_value(value)
 758                         except:
 759                             # Any trouble means we're not in shape to resolve the netref yet
 760                             nuvalue = None
 761                         if nuvalue is not None:
 762                             # Only if we succeed we remove the netref deferral entry
 763                             value = nuvalue
 764                             data.set_attribute_data(guid, name, value)
 765                             if (testbed_guid, guid) in self._netrefs:
 766                                 self._netrefs[(testbed_guid, guid)].discard(name)
 767                     testbed.defer_create_set(guid, name, value)
 768
 769         for guid in element_guids:
 770             (testbed_guid, factory_id) = data.get_box_data(guid)
 771             testbed = self._testbeds.get(testbed_guid)
 772             if testbed:
 773                 for (connector_type_name, cross_guid, cross_connector_type_name) \
 774                         in data.get_connection_data(guid):
 775                     (testbed_guid, factory_id) = data.get_box_data(guid)
 776                     (cross_testbed_guid, cross_factory_id) = data.get_box_data(
 777                             cross_guid)
 778                     if testbed_guid == cross_testbed_guid:
 779                         testbed.defer_connect(guid, connector_type_name,
 780                                 cross_guid, cross_connector_type_name)
 781                 for trace_id in data.get_trace_data(guid):
 782                     testbed.defer_add_trace(guid, trace_id)
 783                 for (autoconf, address, netprefix, broadcast) in \
 784                         data.get_address_data(guid):
 785                     if address != None:
 786                         testbed.defer_add_address(guid, address, netprefix,
 787                                 broadcast)
 788                 for (destination, netprefix, nexthop) in data.get_route_data(guid):
 789                     testbed.defer_add_route(guid, destination, netprefix, nexthop)
 790
 791     def _program_testbed_cross_connections(self, data):
 792         data_guids = data.guids
 793
 794         for guid in data_guids:
 795             if not data.is_testbed_data(guid):
 796                 (testbed_guid, factory_id) = data.get_box_data(guid)
 797                 testbed = self._testbeds.get(testbed_guid)
 798                 if testbed:
 799                     for (connector_type_name, cross_guid, cross_connector_type_name) \
 800                             in data.get_connection_data(guid):
 801                         (testbed_guid, factory_id) = data.get_box_data(guid)
 802                         (cross_testbed_guid, cross_factory_id) = data.get_box_data(
 803                                 cross_guid)
 804                         if testbed_guid != cross_testbed_guid:
 805                             cross_testbed = self._testbeds[cross_testbed_guid]
 806                             cross_testbed_id = cross_testbed.testbed_id
 807                             testbed.defer_cross_connect(guid, connector_type_name, cross_guid,
 808                                     cross_testbed_guid, cross_testbed_id, cross_factory_id,
 809                                     cross_connector_type_name)
 810                             # save cross data for later
 811                             self._add_crossdata(testbed_guid, guid, cross_testbed_guid,
 812                                     cross_guid)
 813
 814     def _add_crossdata(self, testbed_guid, guid, cross_testbed_guid, cross_guid):
 815         if testbed_guid not in self._cross_data:
 816             self._cross_data[testbed_guid] = dict()
 817         if cross_testbed_guid not in self._cross_data[testbed_guid]:
 818             self._cross_data[testbed_guid][cross_testbed_guid] = set()
 819         self._cross_data[testbed_guid][cross_testbed_guid].add(cross_guid)
 820
 821     def _get_cross_data(self, testbed_guid):
 822         cross_data = dict()
 823         if not testbed_guid in self._cross_data:
 824             return cross_data
 825         for cross_testbed_guid, guid_list in \
 826                 self._cross_data[testbed_guid].iteritems():
 827             cross_data[cross_testbed_guid] = dict()
 828             cross_testbed = self._testbeds[cross_testbed_guid]
 829             for cross_guid in guid_list:
 830                 elem_cross_data = dict(
 831                     _guid = cross_guid,
 832                     _testbed_guid = cross_testbed_guid,
 833                     _testbed_id = cross_testbed.testbed_id,
 834                     _testbed_version = cross_testbed.testbed_version)
 835                 cross_data[cross_testbed_guid][cross_guid] = elem_cross_data
 836                 attributes_list = cross_testbed.get_attribute_list(cross_guid)
 837                 for attr_name in attributes_list:
 838                     attr_value = cross_testbed.get(cross_guid, attr_name)
 839                     elem_cross_data[attr_name] = attr_value
 840         return cross_data
 841