src/nepi/core/execute.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from nepi.core.attributes import Attribute, AttributesMap
   5 from nepi.core.connector import ConnectorTypeBase
   6 from nepi.util import validation
   7 from nepi.util.constants import STATUS_FINISHED, TIME_NOW
   8 from nepi.util.parser._xml import XmlExperimentParser
   9 import sys
  10 import re
  11 import threading
  12 import ConfigParser
  13 import os
  14 import collections
  15 import functools
  16
  17 ATTRIBUTE_PATTERN_BASE = re.compile(r"\{#\[(?P<label>[-a-zA-Z0-9._]*)\](?P<expr>(?P<component>\.addr\[[0-9]+\]|\.route\[[0-9]+\]|\.trace\[[0-9]+\])?.\[(?P<attribute>[-a-zA-Z0-9._]*)\])#}")
  18 ATTRIBUTE_PATTERN_GUID_SUB = r"{#[%(guid)s]%(expr)s#}"
  19 COMPONENT_PATTERN = re.compile(r"(?P<kind>[a-z]*)\[(?P<index>.*)\]")
  20
  21 class ConnectorType(ConnectorTypeBase):
  22     def __init__(self, testbed_id, factory_id, name, max = -1, min = 0):
  23         super(ConnectorType, self).__init__(testbed_id, factory_id, name, max, min)
  24         # from_connections -- connections where the other connector is the "From"
  25         # to_connections -- connections where the other connector is the "To"
  26         # keys in the dictionary correspond to the
  27         # connector_type_id for possible connections. The value is a tuple:
  28         # (can_cross, connect)
  29         # can_cross: indicates if the connection is allowed accros different
  30         #    testbed instances
  31         # code: is the connection function to be invoked when the elements
  32         #    are connected
  33         self._from_connections = dict()
  34         self._to_connections = dict()
  35
  36     def add_from_connection(self, testbed_id, factory_id, name, can_cross,
  37             init_code, compl_code):
  38         type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  39         self._from_connections[type_id] = (can_cross, init_code, compl_code)
  40
  41     def add_to_connection(self, testbed_id, factory_id, name, can_cross,
  42             init_code, compl_code):
  43         type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  44         self._to_connections[type_id] = (can_cross, init_code, compl_code)
  45
  46     def can_connect(self, testbed_id, factory_id, name, count,
  47             must_cross):
  48         connector_type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  49         for lookup_type_id in self._type_resolution_order(connector_type_id):
  50             if lookup_type_id in self._from_connections:
  51                 (can_cross, init_code, compl_code) = self._from_connections[lookup_type_id]
  52             elif lookup_type_id in self._to_connections:
  53                 (can_cross, init_code, compl_code) = self._to_connections[lookup_type_id]
  54             else:
  55                 # keep trying
  56                 continue
  57             return not must_cross or can_cross
  58         else:
  59             return False
  60
  61     def _connect_to_code(self, testbed_id, factory_id, name,
  62             must_cross):
  63         connector_type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  64         for lookup_type_id in self._type_resolution_order(connector_type_id):
  65             if lookup_type_id in self._to_connections:
  66                 (can_cross, init_code, compl_code) = self._to_connections[lookup_type_id]
  67                 if not must_cross or can_cross:
  68                     return (init_code, compl_code)
  69         else:
  70             return (False, False)
  71
  72     def connect_to_init_code(self, testbed_id, factory_id, name, must_cross):
  73         return self._connect_to_code(testbed_id, factory_id, name, must_cross)[0]
  74
  75     def connect_to_compl_code(self, testbed_id, factory_id, name, must_cross):
  76         return self._connect_to_code(testbed_id, factory_id, name, must_cross)[1]
  77
  78 class Factory(AttributesMap):
  79     def __init__(self, factory_id, create_function, start_function,
  80             stop_function, status_function,
  81             configure_function, preconfigure_function,
  82             prestart_function,
  83             allow_addresses = False, has_addresses = False,
  84             allow_routes = False, has_routes = False):
  85         super(Factory, self).__init__()
  86         self._factory_id = factory_id
  87         self._allow_addresses = bool(allow_addresses)
  88         self._allow_routes = bool(allow_routes)
  89         self._has_addresses = bool(has_addresses) or self._allow_addresses
  90         self._has_routes = bool(has_routes) or self._allow_routes
  91         self._create_function = create_function
  92         self._start_function = start_function
  93         self._stop_function = stop_function
  94         self._status_function = status_function
  95         self._configure_function = configure_function
  96         self._preconfigure_function = preconfigure_function
  97         self._prestart_function = prestart_function
  98         self._connector_types = dict()
  99         self._traces = list()
 100         self._box_attributes = AttributesMap()
 101
 102     @property
 103     def factory_id(self):
 104         return self._factory_id
 105
 106     @property
 107     def allow_addresses(self):
 108         return self._allow_addresses
 109
 110     @property
 111     def allow_routes(self):
 112         return self._allow_routes
 113
 114     @property
 115     def has_addresses(self):
 116         return self._has_addresses
 117
 118     @property
 119     def has_routes(self):
 120         return self._has_routes
 121
 122     @property
 123     def box_attributes(self):
 124         return self._box_attributes
 125
 126     @property
 127     def create_function(self):
 128         return self._create_function
 129
 130     @property
 131     def prestart_function(self):
 132         return self._prestart_function
 133
 134     @property
 135     def start_function(self):
 136         return self._start_function
 137
 138     @property
 139     def stop_function(self):
 140         return self._stop_function
 141
 142     @property
 143     def status_function(self):
 144         return self._status_function
 145
 146     @property
 147     def configure_function(self):
 148         return self._configure_function
 149
 150     @property
 151     def preconfigure_function(self):
 152         return self._preconfigure_function
 153
 154     @property
 155     def traces(self):
 156         return self._traces
 157
 158     def connector_type(self, name):
 159         return self._connector_types[name]
 160
 161     def add_connector_type(self, connector_type):
 162         self._connector_types[connector_type.name] = connector_type
 163
 164     def add_trace(self, trace_id):
 165         self._traces.append(trace_id)
 166
 167     def add_box_attribute(self, name, help, type, value = None, range = None,
 168         allowed = None, flags = Attribute.NoFlags, validation_function = None,
 169         category = None):
 170         self._box_attributes.add_attribute(name, help, type, value, range,
 171                 allowed, flags, validation_function, category)
 172
 173 class TestbedController(object):
 174     def __init__(self, testbed_id, testbed_version):
 175         self._testbed_id = testbed_id
 176         self._testbed_version = testbed_version
 177
 178     @property
 179     def testbed_id(self):
 180         return self._testbed_id
 181
 182     @property
 183     def testbed_version(self):
 184         return self._testbed_version
 185
 186     @property
 187     def guids(self):
 188         raise NotImplementedError
 189
 190     def defer_configure(self, name, value):
 191         """Instructs setting a configuartion attribute for the testbed instance"""
 192         raise NotImplementedError
 193
 194     def defer_create(self, guid, factory_id):
 195         """Instructs creation of element """
 196         raise NotImplementedError
 197
 198     def defer_create_set(self, guid, name, value):
 199         """Instructs setting an initial attribute on an element"""
 200         raise NotImplementedError
 201
 202     def defer_factory_set(self, guid, name, value):
 203         """Instructs setting an attribute on a factory"""
 204         raise NotImplementedError
 205
 206     def defer_connect(self, guid1, connector_type_name1, guid2,
 207             connector_type_name2):
 208         """Instructs creation of a connection between the given connectors"""
 209         raise NotImplementedError
 210
 211     def defer_cross_connect(self,
 212             guid, connector_type_name,
 213             cross_guid, cross_testbed_guid,
 214             cross_testbed_id, cross_factory_id,
 215             cross_connector_type_name):
 216         """
 217         Instructs creation of a connection between the given connectors
 218         of different testbed instances
 219         """
 220         raise NotImplementedError
 221
 222     def defer_add_trace(self, guid, trace_id):
 223         """Instructs the addition of a trace"""
 224         raise NotImplementedError
 225
 226     def defer_add_address(self, guid, address, netprefix, broadcast):
 227         """Instructs the addition of an address"""
 228         raise NotImplementedError
 229
 230     def defer_add_route(self, guid, destination, netprefix, nexthop):
 231         """Instructs the addition of a route"""
 232         raise NotImplementedError
 233
 234     def do_setup(self):
 235         """After do_setup the testbed initial configuration is done"""
 236         raise NotImplementedError
 237
 238     def do_create(self):
 239         """
 240         After do_create all instructed elements are created and
 241         attributes setted
 242         """
 243         raise NotImplementedError
 244
 245     def do_connect_init(self):
 246         """
 247         After do_connect_init all internal connections between testbed elements
 248         are initiated
 249         """
 250         raise NotImplementedError
 251
 252     def do_connect_compl(self):
 253         """
 254         After do_connect all internal connections between testbed elements
 255         are completed
 256         """
 257         raise NotImplementedError
 258
 259     def do_preconfigure(self):
 260         """
 261         Done just before resolving netrefs, after connection, before cross connections,
 262         useful for early stages of configuration, for setting up stuff that might be
 263         required for netref resolution.
 264         """
 265         raise NotImplementedError
 266
 267     def do_configure(self):
 268         """After do_configure elements are configured"""
 269         raise NotImplementedError
 270
 271     def do_prestart(self):
 272         """Before do_start elements are prestart-configured"""
 273         raise NotImplementedError
 274
 275     def do_cross_connect_init(self, cross_data):
 276         """
 277         After do_cross_connect_init initiation of all external connections
 278         between different testbed elements is performed
 279         """
 280         raise NotImplementedError
 281
 282     def do_cross_connect_compl(self, cross_data):
 283         """
 284         After do_cross_connect_compl completion of all external connections
 285         between different testbed elements is performed
 286         """
 287         raise NotImplementedError
 288
 289     def start(self):
 290         raise NotImplementedError
 291
 292     def stop(self):
 293         raise NotImplementedError
 294
 295     def set(self, guid, name, value, time = TIME_NOW):
 296         raise NotImplementedError
 297
 298     def get(self, guid, name, time = TIME_NOW):
 299         raise NotImplementedError
 300
 301     def get_route(self, guid, index, attribute):
 302         """
 303         Params:
 304
 305             guid: guid of box to query
 306             index: number of routing entry to fetch
 307             attribute: one of Destination, NextHop, NetPrefix
 308         """
 309         raise NotImplementedError
 310
 311     def get_address(self, guid, index, attribute='Address'):
 312         """
 313         Params:
 314
 315             guid: guid of box to query
 316             index: number of inteface to select
 317             attribute: one of Address, NetPrefix, Broadcast
 318         """
 319         raise NotImplementedError
 320
 321     def get_attribute_list(self, guid):
 322         raise NotImplementedError
 323
 324     def action(self, time, guid, action):
 325         raise NotImplementedError
 326
 327     def status(self, guid):
 328         raise NotImplementedError
 329
 330     def trace(self, guid, trace_id, attribute='value'):
 331         raise NotImplementedError
 332
 333     def shutdown(self):
 334         raise NotImplementedError
 335
 336 class ExperimentController(object):
 337     def __init__(self, experiment_xml, root_dir):
 338         self._experiment_xml = experiment_xml
 339         self._testbeds = dict()
 340         self._deployment_config = dict()
 341         self._netrefs = collections.defaultdict(set)
 342         self._testbed_netrefs = collections.defaultdict(set)
 343         self._cross_data = dict()
 344         self._root_dir = root_dir
 345         self._netreffed_testbeds = set()
 346         self._guids_in_testbed_cache = dict()
 347
 348         self.persist_experiment_xml()
 349
 350     @property
 351     def experiment_xml(self):
 352         return self._experiment_xml
 353
 354     def persist_experiment_xml(self):
 355         xml_path = os.path.join(self._root_dir, "experiment.xml")
 356         f = open(xml_path, "w")
 357         f.write(self._experiment_xml)
 358         f.close()
 359
 360     def trace(self, testbed_guid, guid, trace_id, attribute='value'):
 361         return self._testbeds[testbed_guid].trace(guid, trace_id, attribute)
 362
 363     @staticmethod
 364     def _parallel(callables):
 365         excs = []
 366         def wrap(callable):
 367             @functools.wraps(callable)
 368             def wrapped(*p, **kw):
 369                 try:
 370                     callable(*p, **kw)
 371                 except:
 372                     import traceback
 373                     traceback.print_exc(file=sys.stderr)
 374                     excs.append(sys.exc_info())
 375             return wrapped
 376         threads = [ threading.Thread(target=wrap(callable)) for callable in callables ]
 377         for thread in threads:
 378             thread.start()
 379         for thread in threads:
 380             thread.join()
 381         for exc in excs:
 382             eTyp, eVal, eLoc = exc
 383             raise eTyp, eVal, eLoc
 384
 385     def start(self):
 386         parser = XmlExperimentParser()
 387         data = parser.from_xml_to_data(self._experiment_xml)
 388
 389         self._init_testbed_controllers(data)
 390
 391         # persist testbed connection data, for potential recovery
 392         self._persist_testbed_proxies()
 393
 394         def steps_to_configure(self, allowed_guids):
 395             # perform setup in parallel for all test beds,
 396             # wait for all threads to finish
 397             self._parallel([testbed.do_setup
 398                             for guid,testbed in self._testbeds.iteritems()
 399                             if guid in allowed_guids])
 400
 401             # perform create-connect in parallel, wait
 402             # (internal connections only)
 403             self._parallel([testbed.do_create
 404                             for guid,testbed in self._testbeds.iteritems()
 405                             if guid in allowed_guids])
 406
 407             self._parallel([testbed.do_connect_init
 408                             for guid,testbed in self._testbeds.iteritems()
 409                             if guid in allowed_guids])
 410
 411             self._parallel([testbed.do_connect_compl
 412                             for guid,testbed in self._testbeds.iteritems()
 413                             if guid in allowed_guids])
 414
 415             self._parallel([testbed.do_preconfigure
 416                             for guid,testbed in self._testbeds.iteritems()
 417                             if guid in allowed_guids])
 418
 419         steps_to_configure(self, self._testbeds)
 420
 421         if self._netreffed_testbeds:
 422             # initally resolve netrefs
 423             self.do_netrefs(data, fail_if_undefined=False)
 424
 425             # rinse and repeat, for netreffed testbeds
 426             netreffed_testbeds = set(self._netreffed_testbeds)
 427
 428             self._init_testbed_controllers(data)
 429
 430             # persist testbed connection data, for potential recovery
 431             self._persist_testbed_proxies()
 432
 433             # configure dependant testbeds
 434             steps_to_configure(self, netreffed_testbeds)
 435
 436         # final netref step, fail if anything's left unresolved
 437         self.do_netrefs(data, fail_if_undefined=True)
 438
 439         self._program_testbed_cross_connections(data)
 440
 441         # perform do_configure in parallel for al testbeds
 442         # (it's internal configuration for each)
 443         self._parallel([testbed.do_configure
 444                         for testbed in self._testbeds.itervalues()])
 445
 446
 447         #print >>sys.stderr, "DO IT"
 448         #import time
 449         #time.sleep(60)
 450
 451         # cross-connect (cannot be done in parallel)
 452         for guid, testbed in self._testbeds.iteritems():
 453             cross_data = self._get_cross_data(guid)
 454             testbed.do_cross_connect_init(cross_data)
 455         for guid, testbed in self._testbeds.iteritems():
 456             cross_data = self._get_cross_data(guid)
 457             testbed.do_cross_connect_compl(cross_data)
 458
 459         # Last chance to configure (parallel on all testbeds)
 460         self._parallel([testbed.do_prestart
 461                         for testbed in self._testbeds.itervalues()])
 462
 463         # start experiment (parallel start on all testbeds)
 464         self._parallel([testbed.start
 465                         for testbed in self._testbeds.itervalues()])
 466
 467     def _persist_testbed_proxies(self):
 468         TRANSIENT = ('Recover',)
 469
 470         # persist access configuration for all testbeds, so that
 471         # recovery mode can reconnect to them if it becomes necessary
 472         conf = ConfigParser.RawConfigParser()
 473         for testbed_guid, testbed_config in self._deployment_config.iteritems():
 474             testbed_guid = str(testbed_guid)
 475             conf.add_section(testbed_guid)
 476             for attr in testbed_config.attributes_list:
 477                 if attr not in TRANSIENT:
 478                     conf.set(testbed_guid, attr,
 479                         testbed_config.get_attribute_value(attr))
 480
 481         f = open(os.path.join(self._root_dir, 'deployment_config.ini'), 'w')
 482         conf.write(f)
 483         f.close()
 484
 485     def _load_testbed_proxies(self):
 486         TYPEMAP = {
 487             STRING : 'get',
 488             INTEGER : 'getint',
 489             FLOAT : 'getfloat',
 490             BOOLEAN : 'getboolean',
 491         }
 492
 493         # deferred import because proxy needs
 494         # our class definitions to define proxies
 495         import nepi.util.proxy as proxy
 496
 497         conf = ConfigParser.RawConfigParser()
 498         conf.read(os.path.join(self._root_dir, 'deployment_config.ini'))
 499         for testbed_guid in conf.sections():
 500             testbed_config = proxy.AccessConfiguration()
 501             for attr in conf.options(testbed_guid):
 502                 testbed_config.set_attribute_value(attr,
 503                     conf.get(testbed_guid, attr) )
 504
 505             testbed_guid = str(testbed_guid)
 506             conf.add_section(testbed_guid)
 507             for attr in testbed_config.attributes_list:
 508                 if attr not in TRANSIENT:
 509                     getter = getattr(conf, TYPEMAP.get(
 510                         testbed_config.get_attribute_type(attr),
 511                         'get') )
 512                     testbed_config.set_attribute_value(
 513                         testbed_guid, attr, getter(attr))
 514
 515     def _unpersist_testbed_proxies(self):
 516         try:
 517             os.remove(os.path.join(self._root_dir, 'deployment_config.ini'))
 518         except:
 519             # Just print exceptions, this is just cleanup
 520             import traceback
 521             traceback.print_exc(file=sys.stderr)
 522
 523     def stop(self):
 524        for testbed in self._testbeds.values():
 525            testbed.stop()
 526        self._unpersist_testbed_proxies()
 527
 528     def recover(self):
 529         # reload perviously persisted testbed access configurations
 530         self._load_testbed_proxies()
 531
 532         # recreate testbed proxies by reconnecting only
 533         self._init_testbed_controllers(recover = True)
 534
 535         # another time, for netrefs
 536         self._init_testbed_controllers(recover = True)
 537
 538     def is_finished(self, guid):
 539         for testbed in self._testbeds.values():
 540             if guid in testbed.guids:
 541                 return testbed.status(guid) == STATUS_FINISHED
 542         raise RuntimeError("No element exists with guid %d" % guid)
 543
 544     def set(self, testbed_guid, guid, name, value, time = TIME_NOW):
 545         testbed = self._testbeds[testbed_guid]
 546         testbed.set(guid, name, value, time)
 547
 548     def get(self, testbed_guid, guid, name, time = TIME_NOW):
 549         testbed = self._testbeds[testbed_guid]
 550         return testbed.get(guid, name, time)
 551
 552     def shutdown(self):
 553         for testbed in self._testbeds.values():
 554             testbed.shutdown()
 555
 556     def _guids_in_testbed(self, testbed_guid):
 557         if testbed_guid not in self._testbeds:
 558             return set()
 559         if testbed_guid not in self._guids_in_testbed_cache:
 560             self._guids_in_testbed_cache[testbed_guid] = \
 561                 set(self._testbeds[testbed_guid].guids)
 562         return self._guids_in_testbed_cache[testbed_guid]
 563
 564     @staticmethod
 565     def _netref_component_split(component):
 566         match = COMPONENT_PATTERN.match(component)
 567         if match:
 568             return match.group("kind"), match.group("index")
 569         else:
 570             return component, None
 571
 572     _NETREF_COMPONENT_GETTERS = {
 573         'addr':
 574             lambda testbed, guid, index, name:
 575                 testbed.get_address(guid, int(index), name),
 576         'route' :
 577             lambda testbed, guid, index, name:
 578                 testbed.get_route(guid, int(index), name),
 579         'trace' :
 580             lambda testbed, guid, index, name:
 581                 testbed.trace(guid, index, name),
 582         '' :
 583             lambda testbed, guid, index, name:
 584                 testbed.get(guid, name),
 585     }
 586
 587     def resolve_netref_value(self, value, failval = None):
 588         match = ATTRIBUTE_PATTERN_BASE.search(value)
 589         if match:
 590             label = match.group("label")
 591             if label.startswith('GUID-'):
 592                 ref_guid = int(label[5:])
 593                 if ref_guid:
 594                     expr = match.group("expr")
 595                     component = (match.group("component") or "")[1:] # skip the dot
 596                     attribute = match.group("attribute")
 597
 598                     # split compound components into component kind and index
 599                     # eg: 'addr[0]' -> ('addr', '0')
 600                     component, component_index = self._netref_component_split(component)
 601
 602                     # find object and resolve expression
 603                     for ref_testbed_guid, ref_testbed in self._testbeds.iteritems():
 604                         if component not in self._NETREF_COMPONENT_GETTERS:
 605                             raise ValueError, "Malformed netref: %r - unknown component" % (expr,)
 606                         elif ref_guid not in self._guids_in_testbed(ref_testbed_guid):
 607                             pass
 608                         else:
 609                             ref_value = self._NETREF_COMPONENT_GETTERS[component](
 610                                 ref_testbed, ref_guid, component_index, attribute)
 611                             if ref_value:
 612                                 return value.replace(match.group(), ref_value)
 613         # couldn't find value
 614         return failval
 615
 616     def do_netrefs(self, data, fail_if_undefined = False):
 617         # element netrefs
 618         for (testbed_guid, guid), attrs in self._netrefs.items():
 619             testbed = self._testbeds.get(testbed_guid)
 620             if testbed is not None:
 621                 for name in set(attrs):
 622                     value = testbed.get(guid, name)
 623                     if isinstance(value, basestring):
 624                         ref_value = self.resolve_netref_value(value)
 625                         if ref_value is not None:
 626                             testbed.set(guid, name, ref_value)
 627                             attrs.remove(name)
 628                         elif fail_if_undefined:
 629                             raise ValueError, "Unresolvable netref in: %r=%r" % (name,value,)
 630                 if not attrs:
 631                     del self._netrefs[(testbed_guid, guid)]
 632
 633         # testbed netrefs
 634         for testbed_guid, attrs in self._testbed_netrefs.items():
 635             tb_data = dict(data.get_attribute_data(testbed_guid))
 636             if data:
 637                 for name in set(attrs):
 638                     value = tb_data.get(name)
 639                     if isinstance(value, basestring):
 640                         ref_value = self.resolve_netref_value(value)
 641                         if ref_value is not None:
 642                             data.set_attribute_data(testbed_guid, name, ref_value)
 643                             attrs.remove(name)
 644                         elif fail_if_undefined:
 645                             raise ValueError, "Unresolvable netref in: %r" % (value,)
 646                 if not attrs:
 647                     del self._testbed_netrefs[testbed_guid]
 648
 649
 650     def _init_testbed_controllers(self, data, recover = False):
 651         blacklist_testbeds = set(self._testbeds)
 652         element_guids = list()
 653         label_guids = dict()
 654         data_guids = data.guids
 655
 656         # create testbed controllers
 657         for guid in data_guids:
 658             if data.is_testbed_data(guid):
 659                 if guid not in self._testbeds:
 660                     self._create_testbed_controller(guid, data, element_guids,
 661                             recover)
 662             else:
 663                 (testbed_guid, factory_id) = data.get_box_data(guid)
 664                 if testbed_guid not in blacklist_testbeds:
 665                     element_guids.append(guid)
 666                     label = data.get_attribute_data(guid, "label")
 667                     if label is not None:
 668                         if label in label_guids:
 669                             raise RuntimeError, "Label %r is not unique" % (label,)
 670                         label_guids[label] = guid
 671
 672         # replace references to elements labels for its guid
 673         self._resolve_labels(data, data_guids, label_guids)
 674
 675         # program testbed controllers
 676         if not recover:
 677             self._program_testbed_controllers(element_guids, data)
 678
 679     def _resolve_labels(self, data, data_guids, label_guids):
 680         netrefs = self._netrefs
 681         testbed_netrefs = self._testbed_netrefs
 682         for guid in data_guids:
 683             for name, value in data.get_attribute_data(guid):
 684                 if isinstance(value, basestring):
 685                     match = ATTRIBUTE_PATTERN_BASE.search(value)
 686                     if match:
 687                         label = match.group("label")
 688                         if not label.startswith('GUID-'):
 689                             ref_guid = label_guids.get(label)
 690                             if ref_guid is not None:
 691                                 value = ATTRIBUTE_PATTERN_BASE.sub(
 692                                     ATTRIBUTE_PATTERN_GUID_SUB % dict(
 693                                         guid = 'GUID-%d' % (ref_guid,),
 694                                         expr = match.group("expr"),
 695                                         label = label),
 696                                     value)
 697                                 data.set_attribute_data(guid, name, value)
 698
 699                                 # memorize which guid-attribute pairs require
 700                                 # postprocessing, to avoid excessive controller-testbed
 701                                 # communication at configuration time
 702                                 # (which could require high-latency network I/O)
 703                                 if not data.is_testbed_data(guid):
 704                                     (testbed_guid, factory_id) = data.get_box_data(guid)
 705                                     netrefs[(testbed_guid, guid)].add(name)
 706                                 else:
 707                                     testbed_netrefs[guid].add(name)
 708
 709     def _create_testbed_controller(self, guid, data, element_guids, recover):
 710         (testbed_id, testbed_version) = data.get_testbed_data(guid)
 711         deployment_config = self._deployment_config.get(guid)
 712
 713         # deferred import because proxy needs
 714         # our class definitions to define proxies
 715         import nepi.util.proxy as proxy
 716
 717         if deployment_config is None:
 718             # need to create one
 719             deployment_config = proxy.AccessConfiguration()
 720
 721             for (name, value) in data.get_attribute_data(guid):
 722                 if value is not None and deployment_config.has_attribute(name):
 723                     # if any deployment config attribute has a netref, we can't
 724                     # create this controller yet
 725                     if isinstance(value, basestring) and ATTRIBUTE_PATTERN_BASE.search(value):
 726                         # remember to re-issue this one
 727                         self._netreffed_testbeds.add(guid)
 728                         return
 729
 730                     # copy deployment config attribute
 731                     deployment_config.set_attribute_value(name, value)
 732
 733             # commit config
 734             self._deployment_config[guid] = deployment_config
 735
 736         if deployment_config is not None:
 737             # force recovery mode
 738             deployment_config.set_attribute_value("recover",recover)
 739
 740         testbed = proxy.create_testbed_controller(testbed_id,
 741                 testbed_version, deployment_config)
 742         for (name, value) in data.get_attribute_data(guid):
 743             testbed.defer_configure(name, value)
 744         self._testbeds[guid] = testbed
 745         if guid in self._netreffed_testbeds:
 746             self._netreffed_testbeds.remove(guid)
 747
 748     def _program_testbed_controllers(self, element_guids, data):
 749         for guid in element_guids:
 750             (testbed_guid, factory_id) = data.get_box_data(guid)
 751             testbed = self._testbeds.get(testbed_guid)
 752             if testbed:
 753                 testbed.defer_create(guid, factory_id)
 754                 for (name, value) in data.get_attribute_data(guid):
 755                     # Try to resolve create-time netrefs, if possible
 756                     if isinstance(value, basestring) and ATTRIBUTE_PATTERN_BASE.search(value):
 757                         try:
 758                             nuvalue = self.resolve_netref_value(value)
 759                         except:
 760                             # Any trouble means we're not in shape to resolve the netref yet
 761                             nuvalue = None
 762                         if nuvalue is not None:
 763                             # Only if we succeed we remove the netref deferral entry
 764                             value = nuvalue
 765                             data.set_attribute_data(guid, name, value)
 766                             if (testbed_guid, guid) in self._netrefs:
 767                                 self._netrefs[(testbed_guid, guid)].discard(name)
 768                     testbed.defer_create_set(guid, name, value)
 769
 770         for guid in element_guids:
 771             (testbed_guid, factory_id) = data.get_box_data(guid)
 772             testbed = self._testbeds.get(testbed_guid)
 773             if testbed:
 774                 for (connector_type_name, cross_guid, cross_connector_type_name) \
 775                         in data.get_connection_data(guid):
 776                     (testbed_guid, factory_id) = data.get_box_data(guid)
 777                     (cross_testbed_guid, cross_factory_id) = data.get_box_data(
 778                             cross_guid)
 779                     if testbed_guid == cross_testbed_guid:
 780                         testbed.defer_connect(guid, connector_type_name,
 781                                 cross_guid, cross_connector_type_name)
 782                 for trace_id in data.get_trace_data(guid):
 783                     testbed.defer_add_trace(guid, trace_id)
 784                 for (autoconf, address, netprefix, broadcast) in \
 785                         data.get_address_data(guid):
 786                     if address != None:
 787                         testbed.defer_add_address(guid, address, netprefix,
 788                                 broadcast)
 789                 for (destination, netprefix, nexthop) in data.get_route_data(guid):
 790                     testbed.defer_add_route(guid, destination, netprefix, nexthop)
 791
 792     def _program_testbed_cross_connections(self, data):
 793         data_guids = data.guids
 794
 795         for guid in data_guids:
 796             if not data.is_testbed_data(guid):
 797                 (testbed_guid, factory_id) = data.get_box_data(guid)
 798                 testbed = self._testbeds.get(testbed_guid)
 799                 if testbed:
 800                     for (connector_type_name, cross_guid, cross_connector_type_name) \
 801                             in data.get_connection_data(guid):
 802                         (testbed_guid, factory_id) = data.get_box_data(guid)
 803                         (cross_testbed_guid, cross_factory_id) = data.get_box_data(
 804                                 cross_guid)
 805                         if testbed_guid != cross_testbed_guid:
 806                             cross_testbed = self._testbeds[cross_testbed_guid]
 807                             cross_testbed_id = cross_testbed.testbed_id
 808                             testbed.defer_cross_connect(guid, connector_type_name, cross_guid,
 809                                     cross_testbed_guid, cross_testbed_id, cross_factory_id,
 810                                     cross_connector_type_name)
 811                             # save cross data for later
 812                             self._add_crossdata(testbed_guid, guid, cross_testbed_guid,
 813                                     cross_guid)
 814
 815     def _add_crossdata(self, testbed_guid, guid, cross_testbed_guid, cross_guid):
 816         if testbed_guid not in self._cross_data:
 817             self._cross_data[testbed_guid] = dict()
 818         if cross_testbed_guid not in self._cross_data[testbed_guid]:
 819             self._cross_data[testbed_guid][cross_testbed_guid] = set()
 820         self._cross_data[testbed_guid][cross_testbed_guid].add(cross_guid)
 821
 822     def _get_cross_data(self, testbed_guid):
 823         cross_data = dict()
 824         if not testbed_guid in self._cross_data:
 825             return cross_data
 826         for cross_testbed_guid, guid_list in \
 827                 self._cross_data[testbed_guid].iteritems():
 828             cross_data[cross_testbed_guid] = dict()
 829             cross_testbed = self._testbeds[cross_testbed_guid]
 830             for cross_guid in guid_list:
 831                 elem_cross_data = dict(
 832                     _guid = cross_guid,
 833                     _testbed_guid = cross_testbed_guid,
 834                     _testbed_id = cross_testbed.testbed_id,
 835                     _testbed_version = cross_testbed.testbed_version)
 836                 cross_data[cross_testbed_guid][cross_guid] = elem_cross_data
 837                 attributes_list = cross_testbed.get_attribute_list(cross_guid)
 838                 for attr_name in attributes_list:
 839                     attr_value = cross_testbed.get(cross_guid, attr_name)
 840                     elem_cross_data[attr_name] = attr_value
 841         return cross_data
 842