src/nepi/core/execute.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from nepi.core.attributes import Attribute, AttributesMap
   5 from nepi.core.connector import ConnectorTypeBase
   6 from nepi.util import validation
   7 from nepi.util.constants import ApplicationStatus as AS, TIME_NOW
   8 from nepi.util.parser._xml import XmlExperimentParser
   9 import sys
  10 import re
  11 import threading
  12 import ConfigParser
  13 import os
  14 import collections
  15 import functools
  16
  17 ATTRIBUTE_PATTERN_BASE = re.compile(r"\{#\[(?P<label>[-a-zA-Z0-9._]*)\](?P<expr>(?P<component>\.addr\[[0-9]+\]|\.route\[[0-9]+\]|\.trace\[[0-9]+\])?.\[(?P<attribute>[-a-zA-Z0-9._]*)\])#}")
  18 ATTRIBUTE_PATTERN_GUID_SUB = r"{#[%(guid)s]%(expr)s#}"
  19 COMPONENT_PATTERN = re.compile(r"(?P<kind>[a-z]*)\[(?P<index>.*)\]")
  20
  21 class ConnectorType(ConnectorTypeBase):
  22     def __init__(self, testbed_id, factory_id, name, max = -1, min = 0):
  23         super(ConnectorType, self).__init__(testbed_id, factory_id, name, max, min)
  24         # from_connections -- connections where the other connector is the "From"
  25         # to_connections -- connections where the other connector is the "To"
  26         # keys in the dictionary correspond to the
  27         # connector_type_id for possible connections. The value is a tuple:
  28         # (can_cross, connect)
  29         # can_cross: indicates if the connection is allowed accros different
  30         #    testbed instances
  31         # code: is the connection function to be invoked when the elements
  32         #    are connected
  33         self._from_connections = dict()
  34         self._to_connections = dict()
  35
  36     def add_from_connection(self, testbed_id, factory_id, name, can_cross,
  37             init_code, compl_code):
  38         type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  39         self._from_connections[type_id] = (can_cross, init_code, compl_code)
  40
  41     def add_to_connection(self, testbed_id, factory_id, name, can_cross,
  42             init_code, compl_code):
  43         type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  44         self._to_connections[type_id] = (can_cross, init_code, compl_code)
  45
  46     def can_connect(self, testbed_id, factory_id, name, count,
  47             must_cross):
  48         connector_type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  49         for lookup_type_id in self._type_resolution_order(connector_type_id):
  50             if lookup_type_id in self._from_connections:
  51                 (can_cross, init_code, compl_code) = self._from_connections[lookup_type_id]
  52             elif lookup_type_id in self._to_connections:
  53                 (can_cross, init_code, compl_code) = self._to_connections[lookup_type_id]
  54             else:
  55                 # keep trying
  56                 continue
  57             return not must_cross or can_cross
  58         else:
  59             return False
  60
  61     def _connect_to_code(self, testbed_id, factory_id, name,
  62             must_cross):
  63         connector_type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  64         for lookup_type_id in self._type_resolution_order(connector_type_id):
  65             if lookup_type_id in self._to_connections:
  66                 (can_cross, init_code, compl_code) = self._to_connections[lookup_type_id]
  67                 if not must_cross or can_cross:
  68                     return (init_code, compl_code)
  69         else:
  70             return (False, False)
  71
  72     def connect_to_init_code(self, testbed_id, factory_id, name, must_cross):
  73         return self._connect_to_code(testbed_id, factory_id, name, must_cross)[0]
  74
  75     def connect_to_compl_code(self, testbed_id, factory_id, name, must_cross):
  76         return self._connect_to_code(testbed_id, factory_id, name, must_cross)[1]
  77
  78 class Factory(AttributesMap):
  79     def __init__(self, factory_id, create_function, start_function,
  80             stop_function, status_function,
  81             configure_function, preconfigure_function,
  82             prestart_function,
  83             allow_addresses = False, has_addresses = False,
  84             allow_routes = False, has_routes = False):
  85         super(Factory, self).__init__()
  86         self._factory_id = factory_id
  87         self._allow_addresses = bool(allow_addresses)
  88         self._allow_routes = bool(allow_routes)
  89         self._has_addresses = bool(has_addresses) or self._allow_addresses
  90         self._has_routes = bool(has_routes) or self._allow_routes
  91         self._create_function = create_function
  92         self._start_function = start_function
  93         self._stop_function = stop_function
  94         self._status_function = status_function
  95         self._configure_function = configure_function
  96         self._preconfigure_function = preconfigure_function
  97         self._prestart_function = prestart_function
  98         self._connector_types = dict()
  99         self._traces = list()
 100         self._tags = list()
 101         self._box_attributes = AttributesMap()
 102
 103     @property
 104     def factory_id(self):
 105         return self._factory_id
 106
 107     @property
 108     def allow_addresses(self):
 109         return self._allow_addresses
 110
 111     @property
 112     def allow_routes(self):
 113         return self._allow_routes
 114
 115     @property
 116     def has_addresses(self):
 117         return self._has_addresses
 118
 119     @property
 120     def has_routes(self):
 121         return self._has_routes
 122
 123     @property
 124     def box_attributes(self):
 125         return self._box_attributes
 126
 127     @property
 128     def create_function(self):
 129         return self._create_function
 130
 131     @property
 132     def prestart_function(self):
 133         return self._prestart_function
 134
 135     @property
 136     def start_function(self):
 137         return self._start_function
 138
 139     @property
 140     def stop_function(self):
 141         return self._stop_function
 142
 143     @property
 144     def status_function(self):
 145         return self._status_function
 146
 147     @property
 148     def configure_function(self):
 149         return self._configure_function
 150
 151     @property
 152     def preconfigure_function(self):
 153         return self._preconfigure_function
 154
 155     @property
 156     def traces(self):
 157         return self._traces
 158
 159     @property
 160     def tags(self):
 161         return self._tags
 162
 163     def connector_type(self, name):
 164         return self._connector_types[name]
 165
 166     def add_connector_type(self, connector_type):
 167         self._connector_types[connector_type.name] = connector_type
 168
 169     def add_trace(self, trace_id):
 170         self._traces.append(trace_id)
 171
 172     def add_tag(self, tag_id):
 173         self._tags.append(tag_id)
 174
 175     def add_box_attribute(self, name, help, type, value = None, range = None,
 176         allowed = None, flags = Attribute.NoFlags, validation_function = None,
 177         category = None):
 178         self._box_attributes.add_attribute(name, help, type, value, range,
 179                 allowed, flags, validation_function, category)
 180
 181 class TestbedController(object):
 182     def __init__(self, testbed_id, testbed_version):
 183         self._testbed_id = testbed_id
 184         self._testbed_version = testbed_version
 185
 186     @property
 187     def testbed_id(self):
 188         return self._testbed_id
 189
 190     @property
 191     def testbed_version(self):
 192         return self._testbed_version
 193
 194     @property
 195     def guids(self):
 196         raise NotImplementedError
 197
 198     def defer_configure(self, name, value):
 199         """Instructs setting a configuartion attribute for the testbed instance"""
 200         raise NotImplementedError
 201
 202     def defer_create(self, guid, factory_id):
 203         """Instructs creation of element """
 204         raise NotImplementedError
 205
 206     def defer_create_set(self, guid, name, value):
 207         """Instructs setting an initial attribute on an element"""
 208         raise NotImplementedError
 209
 210     def defer_factory_set(self, guid, name, value):
 211         """Instructs setting an attribute on a factory"""
 212         raise NotImplementedError
 213
 214     def defer_connect(self, guid1, connector_type_name1, guid2,
 215             connector_type_name2):
 216         """Instructs creation of a connection between the given connectors"""
 217         raise NotImplementedError
 218
 219     def defer_cross_connect(self,
 220             guid, connector_type_name,
 221             cross_guid, cross_testbed_guid,
 222             cross_testbed_id, cross_factory_id,
 223             cross_connector_type_name):
 224         """
 225         Instructs creation of a connection between the given connectors
 226         of different testbed instances
 227         """
 228         raise NotImplementedError
 229
 230     def defer_add_trace(self, guid, trace_id):
 231         """Instructs the addition of a trace"""
 232         raise NotImplementedError
 233
 234     def defer_add_address(self, guid, address, netprefix, broadcast):
 235         """Instructs the addition of an address"""
 236         raise NotImplementedError
 237
 238     def defer_add_route(self, guid, destination, netprefix, nexthop, metric = 0):
 239         """Instructs the addition of a route"""
 240         raise NotImplementedError
 241
 242     def do_setup(self):
 243         """After do_setup the testbed initial configuration is done"""
 244         raise NotImplementedError
 245
 246     def do_create(self):
 247         """
 248         After do_create all instructed elements are created and
 249         attributes setted
 250         """
 251         raise NotImplementedError
 252
 253     def do_connect_init(self):
 254         """
 255         After do_connect_init all internal connections between testbed elements
 256         are initiated
 257         """
 258         raise NotImplementedError
 259
 260     def do_connect_compl(self):
 261         """
 262         After do_connect all internal connections between testbed elements
 263         are completed
 264         """
 265         raise NotImplementedError
 266
 267     def do_preconfigure(self):
 268         """
 269         Done just before resolving netrefs, after connection, before cross connections,
 270         useful for early stages of configuration, for setting up stuff that might be
 271         required for netref resolution.
 272         """
 273         raise NotImplementedError
 274
 275     def do_configure(self):
 276         """After do_configure elements are configured"""
 277         raise NotImplementedError
 278
 279     def do_prestart(self):
 280         """Before do_start elements are prestart-configured"""
 281         raise NotImplementedError
 282
 283     def do_cross_connect_init(self, cross_data):
 284         """
 285         After do_cross_connect_init initiation of all external connections
 286         between different testbed elements is performed
 287         """
 288         raise NotImplementedError
 289
 290     def do_cross_connect_compl(self, cross_data):
 291         """
 292         After do_cross_connect_compl completion of all external connections
 293         between different testbed elements is performed
 294         """
 295         raise NotImplementedError
 296
 297     def start(self):
 298         raise NotImplementedError
 299
 300     def stop(self):
 301         raise NotImplementedError
 302
 303     def set(self, guid, name, value, time = TIME_NOW):
 304         raise NotImplementedError
 305
 306     def get(self, guid, name, time = TIME_NOW):
 307         raise NotImplementedError
 308
 309     def get_route(self, guid, index, attribute):
 310         """
 311         Params:
 312
 313             guid: guid of box to query
 314             index: number of routing entry to fetch
 315             attribute: one of Destination, NextHop, NetPrefix
 316         """
 317         raise NotImplementedError
 318
 319     def get_address(self, guid, index, attribute='Address'):
 320         """
 321         Params:
 322
 323             guid: guid of box to query
 324             index: number of inteface to select
 325             attribute: one of Address, NetPrefix, Broadcast
 326         """
 327         raise NotImplementedError
 328
 329     def get_attribute_list(self, guid):
 330         raise NotImplementedError
 331
 332     def get_factory_id(self, guid):
 333         raise NotImplementedError
 334
 335     def action(self, time, guid, action):
 336         raise NotImplementedError
 337
 338     def status(self, guid):
 339         raise NotImplementedError
 340
 341     def trace(self, guid, trace_id, attribute='value'):
 342         raise NotImplementedError
 343
 344     def traces_info(self):
 345         """ dictionary of dictionaries:
 346             traces_info = dict({
 347                 guid = dict({
 348                     trace_id = dict({
 349                             host = host,
 350                             filepath = filepath,
 351                             filesize = size in bytes,
 352                         })
 353                 })
 354             })"""
 355         raise NotImplementedError
 356
 357     def shutdown(self):
 358         raise NotImplementedError
 359
 360 class ExperimentController(object):
 361     def __init__(self, experiment_xml, root_dir):
 362         self._experiment_xml = experiment_xml
 363         self._testbeds = dict()
 364         self._deployment_config = dict()
 365         self._netrefs = collections.defaultdict(set)
 366         self._testbed_netrefs = collections.defaultdict(set)
 367         self._cross_data = dict()
 368         self._root_dir = root_dir
 369         self._netreffed_testbeds = set()
 370         self._guids_in_testbed_cache = dict()
 371
 372         self.persist_experiment_xml()
 373
 374     @property
 375     def experiment_xml(self):
 376         return self._experiment_xml
 377
 378     @property
 379     def guids(self):
 380         guids = list()
 381         for testbed_guid in self._testbeds.keys():
 382             _guids = self._guids_in_testbed(testbed_guid)
 383             if _guids:
 384                 guids.extend(_guids)
 385         return guids
 386
 387     def persist_experiment_xml(self):
 388         xml_path = os.path.join(self._root_dir, "experiment.xml")
 389         f = open(xml_path, "w")
 390         f.write(self._experiment_xml)
 391         f.close()
 392
 393     def trace(self, guid, trace_id, attribute='value'):
 394         testbed = self._testbed_for_guid(guid)
 395         if testbed != None:
 396             return testbed.trace(guid, trace_id, attribute)
 397         raise RuntimeError("No element exists with guid %d" % guid)
 398
 399     def traces_info(self):
 400         traces_info = dict()
 401         for guid, testbed in self._testbeds.iteritems():
 402             tinfo = testbed.traces_info()
 403             if tinfo:
 404                 traces_info[guid] = testbed.traces_info()
 405         return traces_info
 406
 407     @staticmethod
 408     def _parallel(callables):
 409         excs = []
 410         def wrap(callable):
 411             @functools.wraps(callable)
 412             def wrapped(*p, **kw):
 413                 try:
 414                     callable(*p, **kw)
 415                 except:
 416                     import traceback
 417                     traceback.print_exc(file=sys.stderr)
 418                     excs.append(sys.exc_info())
 419             return wrapped
 420         threads = [ threading.Thread(target=wrap(callable)) for callable in callables ]
 421         for thread in threads:
 422             thread.start()
 423         for thread in threads:
 424             thread.join()
 425         for exc in excs:
 426             eTyp, eVal, eLoc = exc
 427             raise eTyp, eVal, eLoc
 428
 429     def start(self):
 430         parser = XmlExperimentParser()
 431         data = parser.from_xml_to_data(self._experiment_xml)
 432
 433         self._init_testbed_controllers(data)
 434
 435         # persist testbed connection data, for potential recovery
 436         self._persist_testbed_proxies()
 437
 438         def steps_to_configure(self, allowed_guids):
 439             # perform setup in parallel for all test beds,
 440             # wait for all threads to finish
 441             self._parallel([testbed.do_setup
 442                             for guid,testbed in self._testbeds.iteritems()
 443                             if guid in allowed_guids])
 444
 445             # perform create-connect in parallel, wait
 446             # (internal connections only)
 447             self._parallel([testbed.do_create
 448                             for guid,testbed in self._testbeds.iteritems()
 449                             if guid in allowed_guids])
 450
 451             self._parallel([testbed.do_connect_init
 452                             for guid,testbed in self._testbeds.iteritems()
 453                             if guid in allowed_guids])
 454
 455             self._parallel([testbed.do_connect_compl
 456                             for guid,testbed in self._testbeds.iteritems()
 457                             if guid in allowed_guids])
 458
 459             self._parallel([testbed.do_preconfigure
 460                             for guid,testbed in self._testbeds.iteritems()
 461                             if guid in allowed_guids])
 462             self._clear_caches()
 463
 464         steps_to_configure(self, self._testbeds)
 465
 466         if self._netreffed_testbeds:
 467             # initally resolve netrefs
 468             self.do_netrefs(data, fail_if_undefined=False)
 469
 470             # rinse and repeat, for netreffed testbeds
 471             netreffed_testbeds = set(self._netreffed_testbeds)
 472
 473             self._init_testbed_controllers(data)
 474
 475             # persist testbed connection data, for potential recovery
 476             self._persist_testbed_proxies()
 477
 478             # configure dependant testbeds
 479             steps_to_configure(self, netreffed_testbeds)
 480
 481         # final netref step, fail if anything's left unresolved
 482         self.do_netrefs(data, fail_if_undefined=True)
 483
 484         self._program_testbed_cross_connections(data)
 485
 486         # perform do_configure in parallel for al testbeds
 487         # (it's internal configuration for each)
 488         self._parallel([testbed.do_configure
 489                         for testbed in self._testbeds.itervalues()])
 490
 491         self._clear_caches()
 492
 493         #print >>sys.stderr, "DO IT"
 494         #import time
 495         #time.sleep(60)
 496
 497         # cross-connect (cannot be done in parallel)
 498         for guid, testbed in self._testbeds.iteritems():
 499             cross_data = self._get_cross_data(guid)
 500             testbed.do_cross_connect_init(cross_data)
 501         for guid, testbed in self._testbeds.iteritems():
 502             cross_data = self._get_cross_data(guid)
 503             testbed.do_cross_connect_compl(cross_data)
 504
 505         self._clear_caches()
 506
 507         # Last chance to configure (parallel on all testbeds)
 508         self._parallel([testbed.do_prestart
 509                         for testbed in self._testbeds.itervalues()])
 510
 511         self._clear_caches()
 512
 513         # start experiment (parallel start on all testbeds)
 514         self._parallel([testbed.start
 515                         for testbed in self._testbeds.itervalues()])
 516
 517         self._clear_caches()
 518
 519     def _clear_caches(self):
 520         # Cleaning cache for safety.
 521         self._guids_in_testbed_cache = dict()
 522
 523     def _persist_testbed_proxies(self):
 524         TRANSIENT = ('Recover',)
 525
 526         # persist access configuration for all testbeds, so that
 527         # recovery mode can reconnect to them if it becomes necessary
 528         conf = ConfigParser.RawConfigParser()
 529         for testbed_guid, testbed_config in self._deployment_config.iteritems():
 530             testbed_guid = str(testbed_guid)
 531             conf.add_section(testbed_guid)
 532             for attr in testbed_config.attributes_list:
 533                 if attr not in TRANSIENT:
 534                     conf.set(testbed_guid, attr,
 535                         testbed_config.get_attribute_value(attr))
 536
 537         f = open(os.path.join(self._root_dir, 'deployment_config.ini'), 'w')
 538         conf.write(f)
 539         f.close()
 540
 541     def _load_testbed_proxies(self):
 542         TYPEMAP = {
 543             STRING : 'get',
 544             INTEGER : 'getint',
 545             FLOAT : 'getfloat',
 546             BOOLEAN : 'getboolean',
 547         }
 548
 549         # deferred import because proxy needs
 550         # our class definitions to define proxies
 551         import nepi.util.proxy as proxy
 552
 553         conf = ConfigParser.RawConfigParser()
 554         conf.read(os.path.join(self._root_dir, 'deployment_config.ini'))
 555         for testbed_guid in conf.sections():
 556             testbed_config = proxy.AccessConfiguration()
 557             for attr in conf.options(testbed_guid):
 558                 testbed_config.set_attribute_value(attr,
 559                     conf.get(testbed_guid, attr) )
 560
 561             testbed_guid = str(testbed_guid)
 562             conf.add_section(testbed_guid)
 563             for attr in testbed_config.attributes_list:
 564                 if attr not in TRANSIENT:
 565                     getter = getattr(conf, TYPEMAP.get(
 566                         testbed_config.get_attribute_type(attr),
 567                         'get') )
 568                     testbed_config.set_attribute_value(
 569                         testbed_guid, attr, getter(attr))
 570
 571     def _unpersist_testbed_proxies(self):
 572         try:
 573             os.remove(os.path.join(self._root_dir, 'deployment_config.ini'))
 574         except:
 575             # Just print exceptions, this is just cleanup
 576             import traceback
 577             ######## BUG ##########
 578             #BUG: If the next line is uncomented pyQt explodes when shutting down the experiment !!!!!!!!
 579             #traceback.print_exc(file=sys.stderr)
 580
 581     def stop(self):
 582        for testbed in self._testbeds.values():
 583            testbed.stop()
 584        self._unpersist_testbed_proxies()
 585
 586     def recover(self):
 587         # reload perviously persisted testbed access configurations
 588         self._load_testbed_proxies()
 589
 590         # recreate testbed proxies by reconnecting only
 591         self._init_testbed_controllers(recover = True)
 592
 593         # another time, for netrefs
 594         self._init_testbed_controllers(recover = True)
 595
 596     def is_finished(self, guid):
 597         testbed = self._testbed_for_guid(guid)
 598         if testbed != None:
 599             return testbed.status(guid) == AS.STATUS_FINISHED
 600         raise RuntimeError("No element exists with guid %d" % guid)
 601
 602     def set(self, guid, name, value, time = TIME_NOW):
 603         testbed = self._testbed_for_guid(guid)
 604         if testbed != None:
 605             testbed.set(guid, name, value, time)
 606         else:
 607             raise RuntimeError("No element exists with guid %d" % guid)
 608
 609     def get(self, guid, name, time = TIME_NOW):
 610         testbed = self._testbed_for_guid(guid)
 611         if testbed != None:
 612             return testbed.get(guid, name, time)
 613         raise RuntimeError("No element exists with guid %d" % guid)
 614
 615     def get_deferred(self, guid, name, time = TIME_NOW):
 616         testbed = self._testbed_for_guid(guid)
 617         if testbed != None:
 618             return testbed.get_deferred(guid, name, time)
 619         raise RuntimeError("No element exists with guid %d" % guid)
 620
 621     def get_factory_id(self, guid):
 622         testbed = self._testbed_for_guid(guid)
 623         if testbed != None:
 624             return testbed.get_factory_id(guid)
 625         raise RuntimeError("No element exists with guid %d" % guid)
 626
 627     def get_testbed_id(self, guid):
 628         testbed = self._testbed_for_guid(guid)
 629         if testbed != None:
 630             return testbed.testbed_id
 631         raise RuntimeError("No element exists with guid %d" % guid)
 632
 633     def get_testbed_version(self, guid):
 634         testbed = self._testbed_for_guid(guid)
 635         if testbed != None:
 636             return testbed.testbed_version
 637         raise RuntimeError("No element exists with guid %d" % guid)
 638
 639     def shutdown(self):
 640         exceptions = list()
 641         for testbed in self._testbeds.values():
 642             try:
 643                 testbed.shutdown()
 644             except:
 645                 exceptions.append(sys.exc_info())
 646         for exc_info in exceptions:
 647             raise exc_info[0], exc_info[1], exc_info[2]
 648
 649     def _testbed_for_guid(self, guid):
 650         for testbed_guid in self._testbeds.keys():
 651             if guid in self._guids_in_testbed(testbed_guid):
 652                 return self._testbeds[testbed_guid]
 653         return None
 654
 655     def _guids_in_testbed(self, testbed_guid):
 656         if testbed_guid not in self._testbeds:
 657             return set()
 658         if testbed_guid not in self._guids_in_testbed_cache:
 659             self._guids_in_testbed_cache[testbed_guid] = \
 660                 set(self._testbeds[testbed_guid].guids)
 661         return self._guids_in_testbed_cache[testbed_guid]
 662
 663     @staticmethod
 664     def _netref_component_split(component):
 665         match = COMPONENT_PATTERN.match(component)
 666         if match:
 667             return match.group("kind"), match.group("index")
 668         else:
 669             return component, None
 670
 671     _NETREF_COMPONENT_GETTERS = {
 672         'addr':
 673             lambda testbed, guid, index, name:
 674                 testbed.get_address(guid, int(index), name),
 675         'route' :
 676             lambda testbed, guid, index, name:
 677                 testbed.get_route(guid, int(index), name),
 678         'trace' :
 679             lambda testbed, guid, index, name:
 680                 testbed.trace(guid, index, name),
 681         '' :
 682             lambda testbed, guid, index, name:
 683                 testbed.get(guid, name),
 684     }
 685
 686     def resolve_netref_value(self, value, failval = None):
 687         match = ATTRIBUTE_PATTERN_BASE.search(value)
 688         if match:
 689             label = match.group("label")
 690             if label.startswith('GUID-'):
 691                 ref_guid = int(label[5:])
 692                 if ref_guid:
 693                     expr = match.group("expr")
 694                     component = (match.group("component") or "")[1:] # skip the dot
 695                     attribute = match.group("attribute")
 696
 697                     # split compound components into component kind and index
 698                     # eg: 'addr[0]' -> ('addr', '0')
 699                     component, component_index = self._netref_component_split(component)
 700
 701                     # find object and resolve expression
 702                     for ref_testbed_guid, ref_testbed in self._testbeds.iteritems():
 703                         if component not in self._NETREF_COMPONENT_GETTERS:
 704                             raise ValueError, "Malformed netref: %r - unknown component" % (expr,)
 705                         elif ref_guid not in self._guids_in_testbed(ref_testbed_guid):
 706                             pass
 707                         else:
 708                             ref_value = self._NETREF_COMPONENT_GETTERS[component](
 709                                 ref_testbed, ref_guid, component_index, attribute)
 710                             if ref_value:
 711                                 return value.replace(match.group(), ref_value)
 712         # couldn't find value
 713         return failval
 714
 715     def do_netrefs(self, data, fail_if_undefined = False):
 716         # element netrefs
 717         for (testbed_guid, guid), attrs in self._netrefs.items():
 718             testbed = self._testbeds.get(testbed_guid)
 719             if testbed is not None:
 720                 for name in set(attrs):
 721                     value = testbed.get(guid, name)
 722                     if isinstance(value, basestring):
 723                         ref_value = self.resolve_netref_value(value)
 724                         if ref_value is not None:
 725                             testbed.set(guid, name, ref_value)
 726                             attrs.remove(name)
 727                         elif fail_if_undefined:
 728                             raise ValueError, "Unresolvable netref in: %r=%r" % (name,value,)
 729                 if not attrs:
 730                     del self._netrefs[(testbed_guid, guid)]
 731
 732         # testbed netrefs
 733         for testbed_guid, attrs in self._testbed_netrefs.items():
 734             tb_data = dict(data.get_attribute_data(testbed_guid))
 735             if data:
 736                 for name in set(attrs):
 737                     value = tb_data.get(name)
 738                     if isinstance(value, basestring):
 739                         ref_value = self.resolve_netref_value(value)
 740                         if ref_value is not None:
 741                             data.set_attribute_data(testbed_guid, name, ref_value)
 742                             attrs.remove(name)
 743                         elif fail_if_undefined:
 744                             raise ValueError, "Unresolvable netref in: %r" % (value,)
 745                 if not attrs:
 746                     del self._testbed_netrefs[testbed_guid]
 747
 748
 749     def _init_testbed_controllers(self, data, recover = False):
 750         blacklist_testbeds = set(self._testbeds)
 751         element_guids = list()
 752         label_guids = dict()
 753         data_guids = data.guids
 754
 755         # create testbed controllers
 756         for guid in data_guids:
 757             if data.is_testbed_data(guid):
 758                 if guid not in self._testbeds:
 759                     self._create_testbed_controller(guid, data, element_guids,
 760                             recover)
 761             else:
 762                 (testbed_guid, factory_id) = data.get_box_data(guid)
 763                 if testbed_guid not in blacklist_testbeds:
 764                     element_guids.append(guid)
 765                     label = data.get_attribute_data(guid, "label")
 766                     if label is not None:
 767                         if label in label_guids:
 768                             raise RuntimeError, "Label %r is not unique" % (label,)
 769                         label_guids[label] = guid
 770
 771         # replace references to elements labels for its guid
 772         self._resolve_labels(data, data_guids, label_guids)
 773
 774         # program testbed controllers
 775         if not recover:
 776             self._program_testbed_controllers(element_guids, data)
 777
 778     def _resolve_labels(self, data, data_guids, label_guids):
 779         netrefs = self._netrefs
 780         testbed_netrefs = self._testbed_netrefs
 781         for guid in data_guids:
 782             for name, value in data.get_attribute_data(guid):
 783                 if isinstance(value, basestring):
 784                     match = ATTRIBUTE_PATTERN_BASE.search(value)
 785                     if match:
 786                         label = match.group("label")
 787                         if not label.startswith('GUID-'):
 788                             ref_guid = label_guids.get(label)
 789                             if ref_guid is not None:
 790                                 value = ATTRIBUTE_PATTERN_BASE.sub(
 791                                     ATTRIBUTE_PATTERN_GUID_SUB % dict(
 792                                         guid = 'GUID-%d' % (ref_guid,),
 793                                         expr = match.group("expr"),
 794                                         label = label),
 795                                     value)
 796                                 data.set_attribute_data(guid, name, value)
 797
 798                                 # memorize which guid-attribute pairs require
 799                                 # postprocessing, to avoid excessive controller-testbed
 800                                 # communication at configuration time
 801                                 # (which could require high-latency network I/O)
 802                                 if not data.is_testbed_data(guid):
 803                                     (testbed_guid, factory_id) = data.get_box_data(guid)
 804                                     netrefs[(testbed_guid, guid)].add(name)
 805                                 else:
 806                                     testbed_netrefs[guid].add(name)
 807
 808     def _create_testbed_controller(self, guid, data, element_guids, recover):
 809         (testbed_id, testbed_version) = data.get_testbed_data(guid)
 810         deployment_config = self._deployment_config.get(guid)
 811
 812         # deferred import because proxy needs
 813         # our class definitions to define proxies
 814         import nepi.util.proxy as proxy
 815
 816         if deployment_config is None:
 817             # need to create one
 818             deployment_config = proxy.AccessConfiguration()
 819
 820             for (name, value) in data.get_attribute_data(guid):
 821                 if value is not None and deployment_config.has_attribute(name):
 822                     # if any deployment config attribute has a netref, we can't
 823                     # create this controller yet
 824                     if isinstance(value, basestring) and ATTRIBUTE_PATTERN_BASE.search(value):
 825                         # remember to re-issue this one
 826                         self._netreffed_testbeds.add(guid)
 827                         return
 828
 829                     # copy deployment config attribute
 830                     deployment_config.set_attribute_value(name, value)
 831
 832             # commit config
 833             self._deployment_config[guid] = deployment_config
 834
 835         if deployment_config is not None:
 836             # force recovery mode
 837             deployment_config.set_attribute_value("recover",recover)
 838
 839         testbed = proxy.create_testbed_controller(testbed_id,
 840                 testbed_version, deployment_config)
 841         for (name, value) in data.get_attribute_data(guid):
 842             testbed.defer_configure(name, value)
 843         self._testbeds[guid] = testbed
 844         if guid in self._netreffed_testbeds:
 845             self._netreffed_testbeds.remove(guid)
 846
 847     def _program_testbed_controllers(self, element_guids, data):
 848         for guid in element_guids:
 849             (testbed_guid, factory_id) = data.get_box_data(guid)
 850             testbed = self._testbeds.get(testbed_guid)
 851             if testbed:
 852                 testbed.defer_create(guid, factory_id)
 853                 for (name, value) in data.get_attribute_data(guid):
 854                     # Try to resolve create-time netrefs, if possible
 855                     if isinstance(value, basestring) and ATTRIBUTE_PATTERN_BASE.search(value):
 856                         try:
 857                             nuvalue = self.resolve_netref_value(value)
 858                         except:
 859                             # Any trouble means we're not in shape to resolve the netref yet
 860                             nuvalue = None
 861                         if nuvalue is not None:
 862                             # Only if we succeed we remove the netref deferral entry
 863                             value = nuvalue
 864                             data.set_attribute_data(guid, name, value)
 865                             if (testbed_guid, guid) in self._netrefs:
 866                                 self._netrefs[(testbed_guid, guid)].discard(name)
 867                     testbed.defer_create_set(guid, name, value)
 868
 869         for guid in element_guids:
 870             (testbed_guid, factory_id) = data.get_box_data(guid)
 871             testbed = self._testbeds.get(testbed_guid)
 872             if testbed:
 873                 for (connector_type_name, cross_guid, cross_connector_type_name) \
 874                         in data.get_connection_data(guid):
 875                     (testbed_guid, factory_id) = data.get_box_data(guid)
 876                     (cross_testbed_guid, cross_factory_id) = data.get_box_data(
 877                             cross_guid)
 878                     if testbed_guid == cross_testbed_guid:
 879                         testbed.defer_connect(guid, connector_type_name,
 880                                 cross_guid, cross_connector_type_name)
 881                 for trace_id in data.get_trace_data(guid):
 882                     testbed.defer_add_trace(guid, trace_id)
 883                 for (address, netprefix, broadcast) in \
 884                         data.get_address_data(guid):
 885                     if address != None:
 886                         testbed.defer_add_address(guid, address, netprefix,
 887                                 broadcast)
 888                 for (destination, netprefix, nexthop, metric) in data.get_route_data(guid):
 889                     testbed.defer_add_route(guid, destination, netprefix, nexthop, metric)
 890
 891     def _program_testbed_cross_connections(self, data):
 892         data_guids = data.guids
 893
 894         for guid in data_guids:
 895             if not data.is_testbed_data(guid):
 896                 (testbed_guid, factory_id) = data.get_box_data(guid)
 897                 testbed = self._testbeds.get(testbed_guid)
 898                 if testbed:
 899                     for (connector_type_name, cross_guid, cross_connector_type_name) \
 900                             in data.get_connection_data(guid):
 901                         (testbed_guid, factory_id) = data.get_box_data(guid)
 902                         (cross_testbed_guid, cross_factory_id) = data.get_box_data(
 903                                 cross_guid)
 904                         if testbed_guid != cross_testbed_guid:
 905                             cross_testbed = self._testbeds[cross_testbed_guid]
 906                             cross_testbed_id = cross_testbed.testbed_id
 907                             testbed.defer_cross_connect(guid, connector_type_name, cross_guid,
 908                                     cross_testbed_guid, cross_testbed_id, cross_factory_id,
 909                                     cross_connector_type_name)
 910                             # save cross data for later
 911                             self._add_crossdata(testbed_guid, guid, cross_testbed_guid,
 912                                     cross_guid)
 913
 914     def _add_crossdata(self, testbed_guid, guid, cross_testbed_guid, cross_guid):
 915         if testbed_guid not in self._cross_data:
 916             self._cross_data[testbed_guid] = dict()
 917         if cross_testbed_guid not in self._cross_data[testbed_guid]:
 918             self._cross_data[testbed_guid][cross_testbed_guid] = set()
 919         self._cross_data[testbed_guid][cross_testbed_guid].add(cross_guid)
 920
 921     def _get_cross_data(self, testbed_guid):
 922         cross_data = dict()
 923         if not testbed_guid in self._cross_data:
 924             return cross_data
 925         for cross_testbed_guid, guid_list in \
 926                 self._cross_data[testbed_guid].iteritems():
 927             cross_data[cross_testbed_guid] = dict()
 928             cross_testbed = self._testbeds[cross_testbed_guid]
 929             for cross_guid in guid_list:
 930                 elem_cross_data = dict(
 931                     _guid = cross_guid,
 932                     _testbed_guid = cross_testbed_guid,
 933                     _testbed_id = cross_testbed.testbed_id,
 934                     _testbed_version = cross_testbed.testbed_version)
 935                 cross_data[cross_testbed_guid][cross_guid] = elem_cross_data
 936                 attributes_list = cross_testbed.get_attribute_list(cross_guid)
 937                 for attr_name in attributes_list:
 938                     attr_value = cross_testbed.get(cross_guid, attr_name)
 939                     elem_cross_data[attr_name] = attr_value
 940         return cross_data
 941