src/nepi/core/execute.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from nepi.core.attributes import Attribute, AttributesMap
   5 from nepi.core.connector import ConnectorTypeBase
   6 from nepi.util import validation
   7 from nepi.util.constants import STATUS_FINISHED, TIME_NOW
   8 from nepi.util.parser._xml import XmlExperimentParser
   9 import sys
  10 import re
  11 import threading
  12 import ConfigParser
  13 import os
  14 import collections
  15 import functools
  16
  17 ATTRIBUTE_PATTERN_BASE = re.compile(r"\{#\[(?P<label>[-a-zA-Z0-9._]*)\](?P<expr>(?P<component>\.addr\[[0-9]+\]|\.route\[[0-9]+\]|\.trace\[[0-9]+\])?.\[(?P<attribute>[-a-zA-Z0-9._]*)\])#}")
  18 ATTRIBUTE_PATTERN_GUID_SUB = r"{#[%(guid)s]%(expr)s#}"
  19 COMPONENT_PATTERN = re.compile(r"(?P<kind>[a-z]*)\[(?P<index>.*)\]")
  20
  21 class ConnectorType(ConnectorTypeBase):
  22     def __init__(self, testbed_id, factory_id, name, max = -1, min = 0):
  23         super(ConnectorType, self).__init__(testbed_id, factory_id, name, max, min)
  24         # from_connections -- connections where the other connector is the "From"
  25         # to_connections -- connections where the other connector is the "To"
  26         # keys in the dictionary correspond to the
  27         # connector_type_id for possible connections. The value is a tuple:
  28         # (can_cross, connect)
  29         # can_cross: indicates if the connection is allowed accros different
  30         #    testbed instances
  31         # code: is the connection function to be invoked when the elements
  32         #    are connected
  33         self._from_connections = dict()
  34         self._to_connections = dict()
  35
  36     def add_from_connection(self, testbed_id, factory_id, name, can_cross,
  37             init_code, compl_code):
  38         type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  39         self._from_connections[type_id] = (can_cross, init_code, compl_code)
  40
  41     def add_to_connection(self, testbed_id, factory_id, name, can_cross,
  42             init_code, compl_code):
  43         type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  44         self._to_connections[type_id] = (can_cross, init_code, compl_code)
  45
  46     def can_connect(self, testbed_id, factory_id, name, count,
  47             must_cross):
  48         connector_type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  49         for lookup_type_id in self._type_resolution_order(connector_type_id):
  50             if lookup_type_id in self._from_connections:
  51                 (can_cross, init_code, compl_code) = self._from_connections[lookup_type_id]
  52             elif lookup_type_id in self._to_connections:
  53                 (can_cross, init_code, compl_code) = self._to_connections[lookup_type_id]
  54             else:
  55                 # keep trying
  56                 continue
  57             return not must_cross or can_cross
  58         else:
  59             return False
  60
  61     def _connect_to_code(self, testbed_id, factory_id, name,
  62             must_cross):
  63         connector_type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  64         for lookup_type_id in self._type_resolution_order(connector_type_id):
  65             if lookup_type_id in self._to_connections:
  66                 (can_cross, init_code, compl_code) = self._to_connections[lookup_type_id]
  67                 if not must_cross or can_cross:
  68                     return (init_code, compl_code)
  69         else:
  70             return (False, False)
  71
  72     def connect_to_init_code(self, testbed_id, factory_id, name, must_cross):
  73         return self._connect_to_code(testbed_id, factory_id, name, must_cross)[0]
  74
  75     def connect_to_compl_code(self, testbed_id, factory_id, name, must_cross):
  76         return self._connect_to_code(testbed_id, factory_id, name, must_cross)[1]
  77
  78 class Factory(AttributesMap):
  79     def __init__(self, factory_id, create_function, start_function,
  80             stop_function, status_function,
  81             configure_function, preconfigure_function,
  82             prestart_function,
  83             allow_addresses = False, has_addresses = False,
  84             allow_routes = False, has_routes = False):
  85         super(Factory, self).__init__()
  86         self._factory_id = factory_id
  87         self._allow_addresses = bool(allow_addresses)
  88         self._allow_routes = bool(allow_routes)
  89         self._has_addresses = bool(has_addresses) or self._allow_addresses
  90         self._has_routes = bool(has_routes) or self._allow_routes
  91         self._create_function = create_function
  92         self._start_function = start_function
  93         self._stop_function = stop_function
  94         self._status_function = status_function
  95         self._configure_function = configure_function
  96         self._preconfigure_function = preconfigure_function
  97         self._prestart_function = prestart_function
  98         self._connector_types = dict()
  99         self._traces = list()
 100         self._tags = list()
 101         self._box_attributes = AttributesMap()
 102
 103     @property
 104     def factory_id(self):
 105         return self._factory_id
 106
 107     @property
 108     def allow_addresses(self):
 109         return self._allow_addresses
 110
 111     @property
 112     def allow_routes(self):
 113         return self._allow_routes
 114
 115     @property
 116     def has_addresses(self):
 117         return self._has_addresses
 118
 119     @property
 120     def has_routes(self):
 121         return self._has_routes
 122
 123     @property
 124     def box_attributes(self):
 125         return self._box_attributes
 126
 127     @property
 128     def create_function(self):
 129         return self._create_function
 130
 131     @property
 132     def prestart_function(self):
 133         return self._prestart_function
 134
 135     @property
 136     def start_function(self):
 137         return self._start_function
 138
 139     @property
 140     def stop_function(self):
 141         return self._stop_function
 142
 143     @property
 144     def status_function(self):
 145         return self._status_function
 146
 147     @property
 148     def configure_function(self):
 149         return self._configure_function
 150
 151     @property
 152     def preconfigure_function(self):
 153         return self._preconfigure_function
 154
 155     @property
 156     def traces(self):
 157         return self._traces
 158
 159     @property
 160     def tags(self):
 161         return self._tags
 162
 163     def connector_type(self, name):
 164         return self._connector_types[name]
 165
 166     def add_connector_type(self, connector_type):
 167         self._connector_types[connector_type.name] = connector_type
 168
 169     def add_trace(self, trace_id):
 170         self._traces.append(trace_id)
 171
 172     def add_tag(self, tag_id):
 173         self._tags.append(tag_id)
 174
 175     def add_box_attribute(self, name, help, type, value = None, range = None,
 176         allowed = None, flags = Attribute.NoFlags, validation_function = None,
 177         category = None):
 178         self._box_attributes.add_attribute(name, help, type, value, range,
 179                 allowed, flags, validation_function, category)
 180
 181 class TestbedController(object):
 182     def __init__(self, testbed_id, testbed_version):
 183         self._testbed_id = testbed_id
 184         self._testbed_version = testbed_version
 185
 186     @property
 187     def testbed_id(self):
 188         return self._testbed_id
 189
 190     @property
 191     def testbed_version(self):
 192         return self._testbed_version
 193
 194     @property
 195     def guids(self):
 196         raise NotImplementedError
 197
 198     def defer_configure(self, name, value):
 199         """Instructs setting a configuartion attribute for the testbed instance"""
 200         raise NotImplementedError
 201
 202     def defer_create(self, guid, factory_id):
 203         """Instructs creation of element """
 204         raise NotImplementedError
 205
 206     def defer_create_set(self, guid, name, value):
 207         """Instructs setting an initial attribute on an element"""
 208         raise NotImplementedError
 209
 210     def defer_factory_set(self, guid, name, value):
 211         """Instructs setting an attribute on a factory"""
 212         raise NotImplementedError
 213
 214     def defer_connect(self, guid1, connector_type_name1, guid2,
 215             connector_type_name2):
 216         """Instructs creation of a connection between the given connectors"""
 217         raise NotImplementedError
 218
 219     def defer_cross_connect(self,
 220             guid, connector_type_name,
 221             cross_guid, cross_testbed_guid,
 222             cross_testbed_id, cross_factory_id,
 223             cross_connector_type_name):
 224         """
 225         Instructs creation of a connection between the given connectors
 226         of different testbed instances
 227         """
 228         raise NotImplementedError
 229
 230     def defer_add_trace(self, guid, trace_id):
 231         """Instructs the addition of a trace"""
 232         raise NotImplementedError
 233
 234     def defer_add_address(self, guid, address, netprefix, broadcast):
 235         """Instructs the addition of an address"""
 236         raise NotImplementedError
 237
 238     def defer_add_route(self, guid, destination, netprefix, nexthop):
 239         """Instructs the addition of a route"""
 240         raise NotImplementedError
 241
 242     def do_setup(self):
 243         """After do_setup the testbed initial configuration is done"""
 244         raise NotImplementedError
 245
 246     def do_create(self):
 247         """
 248         After do_create all instructed elements are created and
 249         attributes setted
 250         """
 251         raise NotImplementedError
 252
 253     def do_connect_init(self):
 254         """
 255         After do_connect_init all internal connections between testbed elements
 256         are initiated
 257         """
 258         raise NotImplementedError
 259
 260     def do_connect_compl(self):
 261         """
 262         After do_connect all internal connections between testbed elements
 263         are completed
 264         """
 265         raise NotImplementedError
 266
 267     def do_preconfigure(self):
 268         """
 269         Done just before resolving netrefs, after connection, before cross connections,
 270         useful for early stages of configuration, for setting up stuff that might be
 271         required for netref resolution.
 272         """
 273         raise NotImplementedError
 274
 275     def do_configure(self):
 276         """After do_configure elements are configured"""
 277         raise NotImplementedError
 278
 279     def do_prestart(self):
 280         """Before do_start elements are prestart-configured"""
 281         raise NotImplementedError
 282
 283     def do_cross_connect_init(self, cross_data):
 284         """
 285         After do_cross_connect_init initiation of all external connections
 286         between different testbed elements is performed
 287         """
 288         raise NotImplementedError
 289
 290     def do_cross_connect_compl(self, cross_data):
 291         """
 292         After do_cross_connect_compl completion of all external connections
 293         between different testbed elements is performed
 294         """
 295         raise NotImplementedError
 296
 297     def start(self):
 298         raise NotImplementedError
 299
 300     def stop(self):
 301         raise NotImplementedError
 302
 303     def set(self, guid, name, value, time = TIME_NOW):
 304         raise NotImplementedError
 305
 306     def get(self, guid, name, time = TIME_NOW):
 307         raise NotImplementedError
 308
 309     def get_route(self, guid, index, attribute):
 310         """
 311         Params:
 312
 313             guid: guid of box to query
 314             index: number of routing entry to fetch
 315             attribute: one of Destination, NextHop, NetPrefix
 316         """
 317         raise NotImplementedError
 318
 319     def get_address(self, guid, index, attribute='Address'):
 320         """
 321         Params:
 322
 323             guid: guid of box to query
 324             index: number of inteface to select
 325             attribute: one of Address, NetPrefix, Broadcast
 326         """
 327         raise NotImplementedError
 328
 329     def get_attribute_list(self, guid):
 330         raise NotImplementedError
 331
 332     def get_factory_id(self, guid):
 333         raise NotImplementedError
 334
 335     def action(self, time, guid, action):
 336         raise NotImplementedError
 337
 338     def status(self, guid):
 339         raise NotImplementedError
 340
 341     def trace(self, guid, trace_id, attribute='value'):
 342         raise NotImplementedError
 343
 344     def shutdown(self):
 345         raise NotImplementedError
 346
 347 class ExperimentController(object):
 348     def __init__(self, experiment_xml, root_dir):
 349         self._experiment_xml = experiment_xml
 350         self._testbeds = dict()
 351         self._deployment_config = dict()
 352         self._netrefs = collections.defaultdict(set)
 353         self._testbed_netrefs = collections.defaultdict(set)
 354         self._cross_data = dict()
 355         self._root_dir = root_dir
 356         self._netreffed_testbeds = set()
 357         self._guids_in_testbed_cache = dict()
 358
 359         self.persist_experiment_xml()
 360
 361     @property
 362     def experiment_xml(self):
 363         return self._experiment_xml
 364
 365     @property
 366     def guids(self):
 367         guids = list()
 368         for testbed_guid in self._testbeds.keys():
 369             _guids = self._guids_in_testbed(testbed_guid)
 370             if _guids:
 371                 guids.extend(_guids)
 372         return guids
 373
 374     def persist_experiment_xml(self):
 375         xml_path = os.path.join(self._root_dir, "experiment.xml")
 376         f = open(xml_path, "w")
 377         f.write(self._experiment_xml)
 378         f.close()
 379
 380     def trace(self, guid, trace_id, attribute='value'):
 381         testbed = self._testbed_for_guid(guid)
 382         if testbed != None:
 383             return testbed.trace(guid, trace_id, attribute)
 384         raise RuntimeError("No element exists with guid %d" % guid)
 385
 386     @staticmethod
 387     def _parallel(callables):
 388         excs = []
 389         def wrap(callable):
 390             @functools.wraps(callable)
 391             def wrapped(*p, **kw):
 392                 try:
 393                     callable(*p, **kw)
 394                 except:
 395                     import traceback
 396                     traceback.print_exc(file=sys.stderr)
 397                     excs.append(sys.exc_info())
 398             return wrapped
 399         threads = [ threading.Thread(target=wrap(callable)) for callable in callables ]
 400         for thread in threads:
 401             thread.start()
 402         for thread in threads:
 403             thread.join()
 404         for exc in excs:
 405             eTyp, eVal, eLoc = exc
 406             raise eTyp, eVal, eLoc
 407
 408     def start(self):
 409         parser = XmlExperimentParser()
 410         data = parser.from_xml_to_data(self._experiment_xml)
 411
 412         self._init_testbed_controllers(data)
 413
 414         # persist testbed connection data, for potential recovery
 415         self._persist_testbed_proxies()
 416
 417         def steps_to_configure(self, allowed_guids):
 418             # perform setup in parallel for all test beds,
 419             # wait for all threads to finish
 420             self._parallel([testbed.do_setup
 421                             for guid,testbed in self._testbeds.iteritems()
 422                             if guid in allowed_guids])
 423
 424             # perform create-connect in parallel, wait
 425             # (internal connections only)
 426             self._parallel([testbed.do_create
 427                             for guid,testbed in self._testbeds.iteritems()
 428                             if guid in allowed_guids])
 429
 430             self._parallel([testbed.do_connect_init
 431                             for guid,testbed in self._testbeds.iteritems()
 432                             if guid in allowed_guids])
 433
 434             self._parallel([testbed.do_connect_compl
 435                             for guid,testbed in self._testbeds.iteritems()
 436                             if guid in allowed_guids])
 437
 438             self._parallel([testbed.do_preconfigure
 439                             for guid,testbed in self._testbeds.iteritems()
 440                             if guid in allowed_guids])
 441             self._clear_caches()
 442
 443         steps_to_configure(self, self._testbeds)
 444
 445         if self._netreffed_testbeds:
 446             # initally resolve netrefs
 447             self.do_netrefs(data, fail_if_undefined=False)
 448
 449             # rinse and repeat, for netreffed testbeds
 450             netreffed_testbeds = set(self._netreffed_testbeds)
 451
 452             self._init_testbed_controllers(data)
 453
 454             # persist testbed connection data, for potential recovery
 455             self._persist_testbed_proxies()
 456
 457             # configure dependant testbeds
 458             steps_to_configure(self, netreffed_testbeds)
 459
 460         # final netref step, fail if anything's left unresolved
 461         self.do_netrefs(data, fail_if_undefined=True)
 462
 463         self._program_testbed_cross_connections(data)
 464
 465         # perform do_configure in parallel for al testbeds
 466         # (it's internal configuration for each)
 467         self._parallel([testbed.do_configure
 468                         for testbed in self._testbeds.itervalues()])
 469
 470         self._clear_caches()
 471
 472         #print >>sys.stderr, "DO IT"
 473         #import time
 474         #time.sleep(60)
 475
 476         # cross-connect (cannot be done in parallel)
 477         for guid, testbed in self._testbeds.iteritems():
 478             cross_data = self._get_cross_data(guid)
 479             testbed.do_cross_connect_init(cross_data)
 480         for guid, testbed in self._testbeds.iteritems():
 481             cross_data = self._get_cross_data(guid)
 482             testbed.do_cross_connect_compl(cross_data)
 483
 484         self._clear_caches()
 485
 486         # Last chance to configure (parallel on all testbeds)
 487         self._parallel([testbed.do_prestart
 488                         for testbed in self._testbeds.itervalues()])
 489
 490         self._clear_caches()
 491
 492         # start experiment (parallel start on all testbeds)
 493         self._parallel([testbed.start
 494                         for testbed in self._testbeds.itervalues()])
 495
 496         self._clear_caches()
 497
 498     def _clear_caches(self):
 499         # Cleaning cache for safety.
 500         self._guids_in_testbed_cache = dict()
 501
 502     def _persist_testbed_proxies(self):
 503         TRANSIENT = ('Recover',)
 504
 505         # persist access configuration for all testbeds, so that
 506         # recovery mode can reconnect to them if it becomes necessary
 507         conf = ConfigParser.RawConfigParser()
 508         for testbed_guid, testbed_config in self._deployment_config.iteritems():
 509             testbed_guid = str(testbed_guid)
 510             conf.add_section(testbed_guid)
 511             for attr in testbed_config.attributes_list:
 512                 if attr not in TRANSIENT:
 513                     conf.set(testbed_guid, attr,
 514                         testbed_config.get_attribute_value(attr))
 515
 516         f = open(os.path.join(self._root_dir, 'deployment_config.ini'), 'w')
 517         conf.write(f)
 518         f.close()
 519
 520     def _load_testbed_proxies(self):
 521         TYPEMAP = {
 522             STRING : 'get',
 523             INTEGER : 'getint',
 524             FLOAT : 'getfloat',
 525             BOOLEAN : 'getboolean',
 526         }
 527
 528         # deferred import because proxy needs
 529         # our class definitions to define proxies
 530         import nepi.util.proxy as proxy
 531
 532         conf = ConfigParser.RawConfigParser()
 533         conf.read(os.path.join(self._root_dir, 'deployment_config.ini'))
 534         for testbed_guid in conf.sections():
 535             testbed_config = proxy.AccessConfiguration()
 536             for attr in conf.options(testbed_guid):
 537                 testbed_config.set_attribute_value(attr,
 538                     conf.get(testbed_guid, attr) )
 539
 540             testbed_guid = str(testbed_guid)
 541             conf.add_section(testbed_guid)
 542             for attr in testbed_config.attributes_list:
 543                 if attr not in TRANSIENT:
 544                     getter = getattr(conf, TYPEMAP.get(
 545                         testbed_config.get_attribute_type(attr),
 546                         'get') )
 547                     testbed_config.set_attribute_value(
 548                         testbed_guid, attr, getter(attr))
 549
 550     def _unpersist_testbed_proxies(self):
 551         try:
 552             os.remove(os.path.join(self._root_dir, 'deployment_config.ini'))
 553         except:
 554             # Just print exceptions, this is just cleanup
 555             import traceback
 556             traceback.print_exc(file=sys.stderr)
 557
 558     def stop(self):
 559        for testbed in self._testbeds.values():
 560            testbed.stop()
 561        self._unpersist_testbed_proxies()
 562
 563     def recover(self):
 564         # reload perviously persisted testbed access configurations
 565         self._load_testbed_proxies()
 566
 567         # recreate testbed proxies by reconnecting only
 568         self._init_testbed_controllers(recover = True)
 569
 570         # another time, for netrefs
 571         self._init_testbed_controllers(recover = True)
 572
 573     def is_finished(self, guid):
 574         testbed = self._testbed_for_guid(guid)
 575         if testbed != None:
 576             return testbed.status(guid) == STATUS_FINISHED
 577         raise RuntimeError("No element exists with guid %d" % guid)
 578
 579     def set(self, guid, name, value, time = TIME_NOW):
 580         testbed = self._testbed_for_guid(guid)
 581         if testbed != None:
 582             testbed.set(guid, name, value, time)
 583         else:
 584             raise RuntimeError("No element exists with guid %d" % guid)
 585
 586     def get(self, guid, name, time = TIME_NOW):
 587         testbed = self._testbed_for_guid(guid)
 588         if testbed != None:
 589             return testbed.get(guid, name, time)
 590         raise RuntimeError("No element exists with guid %d" % guid)
 591
 592     def get_factory_id(self, guid):
 593         testbed = self._testbed_for_guid(guid)
 594         if testbed != None:
 595             return testbed.get_factory_id(guid)
 596         raise RuntimeError("No element exists with guid %d" % guid)
 597
 598     def get_testbed_id(self, guid):
 599         testbed = self._testbed_for_guid(guid)
 600         if testbed != None:
 601             return testbed.testbed_id
 602         raise RuntimeError("No element exists with guid %d" % guid)
 603
 604     def get_testbed_version(self, guid):
 605         testbed = self._testbed_for_guid(guid)
 606         if testbed != None:
 607             return testbed.testbed_version
 608         raise RuntimeError("No element exists with guid %d" % guid)
 609
 610     def shutdown(self):
 611         exceptions = list()
 612         for testbed in self._testbeds.values():
 613             try:
 614                 testbed.shutdown()
 615             except:
 616                 exceptions.append(sys.exc_info())
 617         for exc_info in exceptions:
 618             raise exc_info[0], exc_info[1], exc_info[2]
 619
 620     def _testbed_for_guid(self, guid):
 621         for testbed_guid in self._testbeds.keys():
 622             if guid in self._guids_in_testbed(testbed_guid):
 623                 return self._testbeds[testbed_guid]
 624         return None
 625
 626     def _guids_in_testbed(self, testbed_guid):
 627         if testbed_guid not in self._testbeds:
 628             return set()
 629         if testbed_guid not in self._guids_in_testbed_cache:
 630             self._guids_in_testbed_cache[testbed_guid] = \
 631                 set(self._testbeds[testbed_guid].guids)
 632         return self._guids_in_testbed_cache[testbed_guid]
 633
 634     @staticmethod
 635     def _netref_component_split(component):
 636         match = COMPONENT_PATTERN.match(component)
 637         if match:
 638             return match.group("kind"), match.group("index")
 639         else:
 640             return component, None
 641
 642     _NETREF_COMPONENT_GETTERS = {
 643         'addr':
 644             lambda testbed, guid, index, name:
 645                 testbed.get_address(guid, int(index), name),
 646         'route' :
 647             lambda testbed, guid, index, name:
 648                 testbed.get_route(guid, int(index), name),
 649         'trace' :
 650             lambda testbed, guid, index, name:
 651                 testbed.trace(guid, index, name),
 652         '' :
 653             lambda testbed, guid, index, name:
 654                 testbed.get(guid, name),
 655     }
 656
 657     def resolve_netref_value(self, value, failval = None):
 658         match = ATTRIBUTE_PATTERN_BASE.search(value)
 659         if match:
 660             label = match.group("label")
 661             if label.startswith('GUID-'):
 662                 ref_guid = int(label[5:])
 663                 if ref_guid:
 664                     expr = match.group("expr")
 665                     component = (match.group("component") or "")[1:] # skip the dot
 666                     attribute = match.group("attribute")
 667
 668                     # split compound components into component kind and index
 669                     # eg: 'addr[0]' -> ('addr', '0')
 670                     component, component_index = self._netref_component_split(component)
 671
 672                     # find object and resolve expression
 673                     for ref_testbed_guid, ref_testbed in self._testbeds.iteritems():
 674                         if component not in self._NETREF_COMPONENT_GETTERS:
 675                             raise ValueError, "Malformed netref: %r - unknown component" % (expr,)
 676                         elif ref_guid not in self._guids_in_testbed(ref_testbed_guid):
 677                             pass
 678                         else:
 679                             ref_value = self._NETREF_COMPONENT_GETTERS[component](
 680                                 ref_testbed, ref_guid, component_index, attribute)
 681                             if ref_value:
 682                                 return value.replace(match.group(), ref_value)
 683         # couldn't find value
 684         return failval
 685
 686     def do_netrefs(self, data, fail_if_undefined = False):
 687         # element netrefs
 688         for (testbed_guid, guid), attrs in self._netrefs.items():
 689             testbed = self._testbeds.get(testbed_guid)
 690             if testbed is not None:
 691                 for name in set(attrs):
 692                     value = testbed.get(guid, name)
 693                     if isinstance(value, basestring):
 694                         ref_value = self.resolve_netref_value(value)
 695                         if ref_value is not None:
 696                             testbed.set(guid, name, ref_value)
 697                             attrs.remove(name)
 698                         elif fail_if_undefined:
 699                             raise ValueError, "Unresolvable netref in: %r=%r" % (name,value,)
 700                 if not attrs:
 701                     del self._netrefs[(testbed_guid, guid)]
 702
 703         # testbed netrefs
 704         for testbed_guid, attrs in self._testbed_netrefs.items():
 705             tb_data = dict(data.get_attribute_data(testbed_guid))
 706             if data:
 707                 for name in set(attrs):
 708                     value = tb_data.get(name)
 709                     if isinstance(value, basestring):
 710                         ref_value = self.resolve_netref_value(value)
 711                         if ref_value is not None:
 712                             data.set_attribute_data(testbed_guid, name, ref_value)
 713                             attrs.remove(name)
 714                         elif fail_if_undefined:
 715                             raise ValueError, "Unresolvable netref in: %r" % (value,)
 716                 if not attrs:
 717                     del self._testbed_netrefs[testbed_guid]
 718
 719
 720     def _init_testbed_controllers(self, data, recover = False):
 721         blacklist_testbeds = set(self._testbeds)
 722         element_guids = list()
 723         label_guids = dict()
 724         data_guids = data.guids
 725
 726         # create testbed controllers
 727         for guid in data_guids:
 728             if data.is_testbed_data(guid):
 729                 if guid not in self._testbeds:
 730                     self._create_testbed_controller(guid, data, element_guids,
 731                             recover)
 732             else:
 733                 (testbed_guid, factory_id) = data.get_box_data(guid)
 734                 if testbed_guid not in blacklist_testbeds:
 735                     element_guids.append(guid)
 736                     label = data.get_attribute_data(guid, "label")
 737                     if label is not None:
 738                         if label in label_guids:
 739                             raise RuntimeError, "Label %r is not unique" % (label,)
 740                         label_guids[label] = guid
 741
 742         # replace references to elements labels for its guid
 743         self._resolve_labels(data, data_guids, label_guids)
 744
 745         # program testbed controllers
 746         if not recover:
 747             self._program_testbed_controllers(element_guids, data)
 748
 749     def _resolve_labels(self, data, data_guids, label_guids):
 750         netrefs = self._netrefs
 751         testbed_netrefs = self._testbed_netrefs
 752         for guid in data_guids:
 753             for name, value in data.get_attribute_data(guid):
 754                 if isinstance(value, basestring):
 755                     match = ATTRIBUTE_PATTERN_BASE.search(value)
 756                     if match:
 757                         label = match.group("label")
 758                         if not label.startswith('GUID-'):
 759                             ref_guid = label_guids.get(label)
 760                             if ref_guid is not None:
 761                                 value = ATTRIBUTE_PATTERN_BASE.sub(
 762                                     ATTRIBUTE_PATTERN_GUID_SUB % dict(
 763                                         guid = 'GUID-%d' % (ref_guid,),
 764                                         expr = match.group("expr"),
 765                                         label = label),
 766                                     value)
 767                                 data.set_attribute_data(guid, name, value)
 768
 769                                 # memorize which guid-attribute pairs require
 770                                 # postprocessing, to avoid excessive controller-testbed
 771                                 # communication at configuration time
 772                                 # (which could require high-latency network I/O)
 773                                 if not data.is_testbed_data(guid):
 774                                     (testbed_guid, factory_id) = data.get_box_data(guid)
 775                                     netrefs[(testbed_guid, guid)].add(name)
 776                                 else:
 777                                     testbed_netrefs[guid].add(name)
 778
 779     def _create_testbed_controller(self, guid, data, element_guids, recover):
 780         (testbed_id, testbed_version) = data.get_testbed_data(guid)
 781         deployment_config = self._deployment_config.get(guid)
 782
 783         # deferred import because proxy needs
 784         # our class definitions to define proxies
 785         import nepi.util.proxy as proxy
 786
 787         if deployment_config is None:
 788             # need to create one
 789             deployment_config = proxy.AccessConfiguration()
 790
 791             for (name, value) in data.get_attribute_data(guid):
 792                 if value is not None and deployment_config.has_attribute(name):
 793                     # if any deployment config attribute has a netref, we can't
 794                     # create this controller yet
 795                     if isinstance(value, basestring) and ATTRIBUTE_PATTERN_BASE.search(value):
 796                         # remember to re-issue this one
 797                         self._netreffed_testbeds.add(guid)
 798                         return
 799
 800                     # copy deployment config attribute
 801                     deployment_config.set_attribute_value(name, value)
 802
 803             # commit config
 804             self._deployment_config[guid] = deployment_config
 805
 806         if deployment_config is not None:
 807             # force recovery mode
 808             deployment_config.set_attribute_value("recover",recover)
 809
 810         testbed = proxy.create_testbed_controller(testbed_id,
 811                 testbed_version, deployment_config)
 812         for (name, value) in data.get_attribute_data(guid):
 813             testbed.defer_configure(name, value)
 814         self._testbeds[guid] = testbed
 815         if guid in self._netreffed_testbeds:
 816             self._netreffed_testbeds.remove(guid)
 817
 818     def _program_testbed_controllers(self, element_guids, data):
 819         for guid in element_guids:
 820             (testbed_guid, factory_id) = data.get_box_data(guid)
 821             testbed = self._testbeds.get(testbed_guid)
 822             if testbed:
 823                 testbed.defer_create(guid, factory_id)
 824                 for (name, value) in data.get_attribute_data(guid):
 825                     # Try to resolve create-time netrefs, if possible
 826                     if isinstance(value, basestring) and ATTRIBUTE_PATTERN_BASE.search(value):
 827                         try:
 828                             nuvalue = self.resolve_netref_value(value)
 829                         except:
 830                             # Any trouble means we're not in shape to resolve the netref yet
 831                             nuvalue = None
 832                         if nuvalue is not None:
 833                             # Only if we succeed we remove the netref deferral entry
 834                             value = nuvalue
 835                             data.set_attribute_data(guid, name, value)
 836                             if (testbed_guid, guid) in self._netrefs:
 837                                 self._netrefs[(testbed_guid, guid)].discard(name)
 838                     testbed.defer_create_set(guid, name, value)
 839
 840         for guid in element_guids:
 841             (testbed_guid, factory_id) = data.get_box_data(guid)
 842             testbed = self._testbeds.get(testbed_guid)
 843             if testbed:
 844                 for (connector_type_name, cross_guid, cross_connector_type_name) \
 845                         in data.get_connection_data(guid):
 846                     (testbed_guid, factory_id) = data.get_box_data(guid)
 847                     (cross_testbed_guid, cross_factory_id) = data.get_box_data(
 848                             cross_guid)
 849                     if testbed_guid == cross_testbed_guid:
 850                         testbed.defer_connect(guid, connector_type_name,
 851                                 cross_guid, cross_connector_type_name)
 852                 for trace_id in data.get_trace_data(guid):
 853                     testbed.defer_add_trace(guid, trace_id)
 854                 for (autoconf, address, netprefix, broadcast) in \
 855                         data.get_address_data(guid):
 856                     if address != None:
 857                         testbed.defer_add_address(guid, address, netprefix,
 858                                 broadcast)
 859                 for (destination, netprefix, nexthop) in data.get_route_data(guid):
 860                     testbed.defer_add_route(guid, destination, netprefix, nexthop)
 861
 862     def _program_testbed_cross_connections(self, data):
 863         data_guids = data.guids
 864
 865         for guid in data_guids:
 866             if not data.is_testbed_data(guid):
 867                 (testbed_guid, factory_id) = data.get_box_data(guid)
 868                 testbed = self._testbeds.get(testbed_guid)
 869                 if testbed:
 870                     for (connector_type_name, cross_guid, cross_connector_type_name) \
 871                             in data.get_connection_data(guid):
 872                         (testbed_guid, factory_id) = data.get_box_data(guid)
 873                         (cross_testbed_guid, cross_factory_id) = data.get_box_data(
 874                                 cross_guid)
 875                         if testbed_guid != cross_testbed_guid:
 876                             cross_testbed = self._testbeds[cross_testbed_guid]
 877                             cross_testbed_id = cross_testbed.testbed_id
 878                             testbed.defer_cross_connect(guid, connector_type_name, cross_guid,
 879                                     cross_testbed_guid, cross_testbed_id, cross_factory_id,
 880                                     cross_connector_type_name)
 881                             # save cross data for later
 882                             self._add_crossdata(testbed_guid, guid, cross_testbed_guid,
 883                                     cross_guid)
 884
 885     def _add_crossdata(self, testbed_guid, guid, cross_testbed_guid, cross_guid):
 886         if testbed_guid not in self._cross_data:
 887             self._cross_data[testbed_guid] = dict()
 888         if cross_testbed_guid not in self._cross_data[testbed_guid]:
 889             self._cross_data[testbed_guid][cross_testbed_guid] = set()
 890         self._cross_data[testbed_guid][cross_testbed_guid].add(cross_guid)
 891
 892     def _get_cross_data(self, testbed_guid):
 893         cross_data = dict()
 894         if not testbed_guid in self._cross_data:
 895             return cross_data
 896         for cross_testbed_guid, guid_list in \
 897                 self._cross_data[testbed_guid].iteritems():
 898             cross_data[cross_testbed_guid] = dict()
 899             cross_testbed = self._testbeds[cross_testbed_guid]
 900             for cross_guid in guid_list:
 901                 elem_cross_data = dict(
 902                     _guid = cross_guid,
 903                     _testbed_guid = cross_testbed_guid,
 904                     _testbed_id = cross_testbed.testbed_id,
 905                     _testbed_version = cross_testbed.testbed_version)
 906                 cross_data[cross_testbed_guid][cross_guid] = elem_cross_data
 907                 attributes_list = cross_testbed.get_attribute_list(cross_guid)
 908                 for attr_name in attributes_list:
 909                     attr_value = cross_testbed.get(cross_guid, attr_name)
 910                     elem_cross_data[attr_name] = attr_value
 911         return cross_data
 912