src/nepi/core/execute.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from nepi.core.attributes import Attribute, AttributesMap
   5 from nepi.util import validation
   6 from nepi.util.constants import ApplicationStatus as AS, TIME_NOW
   7 from nepi.util.parser._xml import XmlExperimentParser
   8 import sys
   9 import re
  10 import threading
  11 import ConfigParser
  12 import os
  13 import collections
  14 import functools
  15
  16 ATTRIBUTE_PATTERN_BASE = re.compile(r"\{#\[(?P<label>[-a-zA-Z0-9._]*)\](?P<expr>(?P<component>\.addr\[[0-9]+\]|\.route\[[0-9]+\]|\.trace\[[0-9]+\])?.\[(?P<attribute>[-a-zA-Z0-9._]*)\])#}")
  17 ATTRIBUTE_PATTERN_GUID_SUB = r"{#[%(guid)s]%(expr)s#}"
  18 COMPONENT_PATTERN = re.compile(r"(?P<kind>[a-z]*)\[(?P<index>.*)\]")
  19
  20 class Factory(AttributesMap):
  21     def __init__(self, factory_id, create_function, start_function,
  22             stop_function, status_function,
  23             configure_function, preconfigure_function,
  24             prestart_function,
  25             allow_addresses = False, has_addresses = False,
  26             allow_routes = False, has_routes = False):
  27         super(Factory, self).__init__()
  28         self._factory_id = factory_id
  29         self._allow_addresses = bool(allow_addresses)
  30         self._allow_routes = bool(allow_routes)
  31         self._has_addresses = bool(has_addresses) or self._allow_addresses
  32         self._has_routes = bool(has_routes) or self._allow_routes
  33         self._create_function = create_function
  34         self._start_function = start_function
  35         self._stop_function = stop_function
  36         self._status_function = status_function
  37         self._configure_function = configure_function
  38         self._preconfigure_function = preconfigure_function
  39         self._prestart_function = prestart_function
  40         self._connector_types = dict()
  41         self._traces = list()
  42         self._tags = list()
  43         self._box_attributes = AttributesMap()
  44
  45     @property
  46     def factory_id(self):
  47         return self._factory_id
  48
  49     @property
  50     def allow_addresses(self):
  51         return self._allow_addresses
  52
  53     @property
  54     def allow_routes(self):
  55         return self._allow_routes
  56
  57     @property
  58     def has_addresses(self):
  59         return self._has_addresses
  60
  61     @property
  62     def has_routes(self):
  63         return self._has_routes
  64
  65     @property
  66     def box_attributes(self):
  67         return self._box_attributes
  68
  69     @property
  70     def create_function(self):
  71         return self._create_function
  72
  73     @property
  74     def prestart_function(self):
  75         return self._prestart_function
  76
  77     @property
  78     def start_function(self):
  79         return self._start_function
  80
  81     @property
  82     def stop_function(self):
  83         return self._stop_function
  84
  85     @property
  86     def status_function(self):
  87         return self._status_function
  88
  89     @property
  90     def configure_function(self):
  91         return self._configure_function
  92
  93     @property
  94     def preconfigure_function(self):
  95         return self._preconfigure_function
  96
  97     @property
  98     def traces(self):
  99         return self._traces
 100
 101     @property
 102     def tags(self):
 103         return self._tags
 104
 105     def connector_type(self, name):
 106         return self._connector_types[name]
 107
 108     def add_connector_type(self, connector_type):
 109         self._connector_types[connector_type.name] = connector_type
 110
 111     def add_trace(self, trace_id):
 112         self._traces.append(trace_id)
 113
 114     def add_tag(self, tag_id):
 115         self._tags.append(tag_id)
 116
 117     def add_box_attribute(self, name, help, type, value = None, range = None,
 118         allowed = None, flags = Attribute.NoFlags, validation_function = None,
 119         category = None):
 120         self._box_attributes.add_attribute(name, help, type, value, range,
 121                 allowed, flags, validation_function, category)
 122
 123 class TestbedController(object):
 124     def __init__(self, testbed_id, testbed_version):
 125         self._testbed_id = testbed_id
 126         self._testbed_version = testbed_version
 127
 128     @property
 129     def testbed_id(self):
 130         return self._testbed_id
 131
 132     @property
 133     def testbed_version(self):
 134         return self._testbed_version
 135
 136     @property
 137     def guids(self):
 138         raise NotImplementedError
 139
 140     def defer_configure(self, name, value):
 141         """Instructs setting a configuartion attribute for the testbed instance"""
 142         raise NotImplementedError
 143
 144     def defer_create(self, guid, factory_id):
 145         """Instructs creation of element """
 146         raise NotImplementedError
 147
 148     def defer_create_set(self, guid, name, value):
 149         """Instructs setting an initial attribute on an element"""
 150         raise NotImplementedError
 151
 152     def defer_factory_set(self, guid, name, value):
 153         """Instructs setting an attribute on a factory"""
 154         raise NotImplementedError
 155
 156     def defer_connect(self, guid1, connector_type_name1, guid2,
 157             connector_type_name2):
 158         """Instructs creation of a connection between the given connectors"""
 159         raise NotImplementedError
 160
 161     def defer_cross_connect(self,
 162             guid, connector_type_name,
 163             cross_guid, cross_testbed_guid,
 164             cross_testbed_id, cross_factory_id,
 165             cross_connector_type_name):
 166         """
 167         Instructs creation of a connection between the given connectors
 168         of different testbed instances
 169         """
 170         raise NotImplementedError
 171
 172     def defer_add_trace(self, guid, trace_id):
 173         """Instructs the addition of a trace"""
 174         raise NotImplementedError
 175
 176     def defer_add_address(self, guid, address, netprefix, broadcast):
 177         """Instructs the addition of an address"""
 178         raise NotImplementedError
 179
 180     def defer_add_route(self, guid, destination, netprefix, nexthop, metric = 0):
 181         """Instructs the addition of a route"""
 182         raise NotImplementedError
 183
 184     def do_setup(self):
 185         """After do_setup the testbed initial configuration is done"""
 186         raise NotImplementedError
 187
 188     def do_create(self):
 189         """
 190         After do_create all instructed elements are created and
 191         attributes setted
 192         """
 193         raise NotImplementedError
 194
 195     def do_connect_init(self):
 196         """
 197         After do_connect_init all internal connections between testbed elements
 198         are initiated
 199         """
 200         raise NotImplementedError
 201
 202     def do_connect_compl(self):
 203         """
 204         After do_connect all internal connections between testbed elements
 205         are completed
 206         """
 207         raise NotImplementedError
 208
 209     def do_preconfigure(self):
 210         """
 211         Done just before resolving netrefs, after connection, before cross connections,
 212         useful for early stages of configuration, for setting up stuff that might be
 213         required for netref resolution.
 214         """
 215         raise NotImplementedError
 216
 217     def do_configure(self):
 218         """After do_configure elements are configured"""
 219         raise NotImplementedError
 220
 221     def do_prestart(self):
 222         """Before do_start elements are prestart-configured"""
 223         raise NotImplementedError
 224
 225     def do_cross_connect_init(self, cross_data):
 226         """
 227         After do_cross_connect_init initiation of all external connections
 228         between different testbed elements is performed
 229         """
 230         raise NotImplementedError
 231
 232     def do_cross_connect_compl(self, cross_data):
 233         """
 234         After do_cross_connect_compl completion of all external connections
 235         between different testbed elements is performed
 236         """
 237         raise NotImplementedError
 238
 239     def start(self):
 240         raise NotImplementedError
 241
 242     def stop(self):
 243         raise NotImplementedError
 244
 245     def set(self, guid, name, value, time = TIME_NOW):
 246         raise NotImplementedError
 247
 248     def get(self, guid, name, time = TIME_NOW):
 249         raise NotImplementedError
 250
 251     def get_route(self, guid, index, attribute):
 252         """
 253         Params:
 254
 255             guid: guid of box to query
 256             index: number of routing entry to fetch
 257             attribute: one of Destination, NextHop, NetPrefix
 258         """
 259         raise NotImplementedError
 260
 261     def get_address(self, guid, index, attribute='Address'):
 262         """
 263         Params:
 264
 265             guid: guid of box to query
 266             index: number of inteface to select
 267             attribute: one of Address, NetPrefix, Broadcast
 268         """
 269         raise NotImplementedError
 270
 271     def get_attribute_list(self, guid):
 272         raise NotImplementedError
 273
 274     def get_factory_id(self, guid):
 275         raise NotImplementedError
 276
 277     def action(self, time, guid, action):
 278         raise NotImplementedError
 279
 280     def status(self, guid):
 281         raise NotImplementedError
 282
 283     def trace(self, guid, trace_id, attribute='value'):
 284         raise NotImplementedError
 285
 286     def traces_info(self):
 287         """ dictionary of dictionaries:
 288             traces_info = dict({
 289                 guid = dict({
 290                     trace_id = dict({
 291                             host = host,
 292                             filepath = filepath,
 293                             filesize = size in bytes,
 294                         })
 295                 })
 296             })"""
 297         raise NotImplementedError
 298
 299     def shutdown(self):
 300         raise NotImplementedError
 301
 302 class ExperimentController(object):
 303     def __init__(self, experiment_xml, root_dir):
 304         self._experiment_xml = experiment_xml
 305         self._testbeds = dict()
 306         self._deployment_config = dict()
 307         self._netrefs = collections.defaultdict(set)
 308         self._testbed_netrefs = collections.defaultdict(set)
 309         self._cross_data = dict()
 310         self._root_dir = root_dir
 311         self._netreffed_testbeds = set()
 312         self._guids_in_testbed_cache = dict()
 313
 314         self.persist_experiment_xml()
 315
 316     @property
 317     def experiment_xml(self):
 318         return self._experiment_xml
 319
 320     @property
 321     def guids(self):
 322         guids = list()
 323         for testbed_guid in self._testbeds.keys():
 324             _guids = self._guids_in_testbed(testbed_guid)
 325             if _guids:
 326                 guids.extend(_guids)
 327         return guids
 328
 329     def persist_experiment_xml(self):
 330         xml_path = os.path.join(self._root_dir, "experiment.xml")
 331         f = open(xml_path, "w")
 332         f.write(self._experiment_xml)
 333         f.close()
 334
 335     def trace(self, guid, trace_id, attribute='value'):
 336         testbed = self._testbed_for_guid(guid)
 337         if testbed != None:
 338             return testbed.trace(guid, trace_id, attribute)
 339         raise RuntimeError("No element exists with guid %d" % guid)
 340
 341     def traces_info(self):
 342         traces_info = dict()
 343         for guid, testbed in self._testbeds.iteritems():
 344             tinfo = testbed.traces_info()
 345             if tinfo:
 346                 traces_info[guid] = testbed.traces_info()
 347         return traces_info
 348
 349     @staticmethod
 350     def _parallel(callables):
 351         excs = []
 352         def wrap(callable):
 353             @functools.wraps(callable)
 354             def wrapped(*p, **kw):
 355                 try:
 356                     callable(*p, **kw)
 357                 except:
 358                     import traceback
 359                     traceback.print_exc(file=sys.stderr)
 360                     excs.append(sys.exc_info())
 361             return wrapped
 362         threads = [ threading.Thread(target=wrap(callable)) for callable in callables ]
 363         for thread in threads:
 364             thread.start()
 365         for thread in threads:
 366             thread.join()
 367         for exc in excs:
 368             eTyp, eVal, eLoc = exc
 369             raise eTyp, eVal, eLoc
 370
 371     def start(self):
 372         parser = XmlExperimentParser()
 373         data = parser.from_xml_to_data(self._experiment_xml)
 374
 375         self._init_testbed_controllers(data)
 376
 377         # persist testbed connection data, for potential recovery
 378         self._persist_testbed_proxies()
 379
 380         def steps_to_configure(self, allowed_guids):
 381             # perform setup in parallel for all test beds,
 382             # wait for all threads to finish
 383             self._parallel([testbed.do_setup
 384                             for guid,testbed in self._testbeds.iteritems()
 385                             if guid in allowed_guids])
 386
 387             # perform create-connect in parallel, wait
 388             # (internal connections only)
 389             self._parallel([testbed.do_create
 390                             for guid,testbed in self._testbeds.iteritems()
 391                             if guid in allowed_guids])
 392
 393             self._parallel([testbed.do_connect_init
 394                             for guid,testbed in self._testbeds.iteritems()
 395                             if guid in allowed_guids])
 396
 397             self._parallel([testbed.do_connect_compl
 398                             for guid,testbed in self._testbeds.iteritems()
 399                             if guid in allowed_guids])
 400
 401             self._parallel([testbed.do_preconfigure
 402                             for guid,testbed in self._testbeds.iteritems()
 403                             if guid in allowed_guids])
 404             self._clear_caches()
 405
 406         steps_to_configure(self, self._testbeds)
 407
 408         if self._netreffed_testbeds:
 409             # initally resolve netrefs
 410             self.do_netrefs(data, fail_if_undefined=False)
 411
 412             # rinse and repeat, for netreffed testbeds
 413             netreffed_testbeds = set(self._netreffed_testbeds)
 414
 415             self._init_testbed_controllers(data)
 416
 417             # persist testbed connection data, for potential recovery
 418             self._persist_testbed_proxies()
 419
 420             # configure dependant testbeds
 421             steps_to_configure(self, netreffed_testbeds)
 422
 423         # final netref step, fail if anything's left unresolved
 424         self.do_netrefs(data, fail_if_undefined=True)
 425
 426         self._program_testbed_cross_connections(data)
 427
 428         # perform do_configure in parallel for al testbeds
 429         # (it's internal configuration for each)
 430         self._parallel([testbed.do_configure
 431                         for testbed in self._testbeds.itervalues()])
 432
 433         self._clear_caches()
 434
 435         #print >>sys.stderr, "DO IT"
 436         #import time
 437         #time.sleep(60)
 438
 439         # cross-connect (cannot be done in parallel)
 440         for guid, testbed in self._testbeds.iteritems():
 441             cross_data = self._get_cross_data(guid)
 442             testbed.do_cross_connect_init(cross_data)
 443         for guid, testbed in self._testbeds.iteritems():
 444             cross_data = self._get_cross_data(guid)
 445             testbed.do_cross_connect_compl(cross_data)
 446
 447         self._clear_caches()
 448
 449         # Last chance to configure (parallel on all testbeds)
 450         self._parallel([testbed.do_prestart
 451                         for testbed in self._testbeds.itervalues()])
 452
 453         self._clear_caches()
 454
 455         # start experiment (parallel start on all testbeds)
 456         self._parallel([testbed.start
 457                         for testbed in self._testbeds.itervalues()])
 458
 459         self._clear_caches()
 460
 461     def _clear_caches(self):
 462         # Cleaning cache for safety.
 463         self._guids_in_testbed_cache = dict()
 464
 465     def _persist_testbed_proxies(self):
 466         TRANSIENT = ('Recover',)
 467
 468         # persist access configuration for all testbeds, so that
 469         # recovery mode can reconnect to them if it becomes necessary
 470         conf = ConfigParser.RawConfigParser()
 471         for testbed_guid, testbed_config in self._deployment_config.iteritems():
 472             testbed_guid = str(testbed_guid)
 473             conf.add_section(testbed_guid)
 474             for attr in testbed_config.attributes_list:
 475                 if attr not in TRANSIENT:
 476                     conf.set(testbed_guid, attr,
 477                         testbed_config.get_attribute_value(attr))
 478
 479         f = open(os.path.join(self._root_dir, 'deployment_config.ini'), 'w')
 480         conf.write(f)
 481         f.close()
 482
 483     def _load_testbed_proxies(self):
 484         TYPEMAP = {
 485             STRING : 'get',
 486             INTEGER : 'getint',
 487             FLOAT : 'getfloat',
 488             BOOLEAN : 'getboolean',
 489         }
 490
 491         # deferred import because proxy needs
 492         # our class definitions to define proxies
 493         import nepi.util.proxy as proxy
 494
 495         conf = ConfigParser.RawConfigParser()
 496         conf.read(os.path.join(self._root_dir, 'deployment_config.ini'))
 497         for testbed_guid in conf.sections():
 498             testbed_config = proxy.AccessConfiguration()
 499             for attr in conf.options(testbed_guid):
 500                 testbed_config.set_attribute_value(attr,
 501                     conf.get(testbed_guid, attr) )
 502
 503             testbed_guid = str(testbed_guid)
 504             conf.add_section(testbed_guid)
 505             for attr in testbed_config.attributes_list:
 506                 if attr not in TRANSIENT:
 507                     getter = getattr(conf, TYPEMAP.get(
 508                         testbed_config.get_attribute_type(attr),
 509                         'get') )
 510                     testbed_config.set_attribute_value(
 511                         testbed_guid, attr, getter(attr))
 512
 513     def _unpersist_testbed_proxies(self):
 514         try:
 515             os.remove(os.path.join(self._root_dir, 'deployment_config.ini'))
 516         except:
 517             # Just print exceptions, this is just cleanup
 518             import traceback
 519             ######## BUG ##########
 520             #BUG: If the next line is uncomented pyQt explodes when shutting down the experiment !!!!!!!!
 521             #traceback.print_exc(file=sys.stderr)
 522
 523     def stop(self):
 524        for testbed in self._testbeds.values():
 525            testbed.stop()
 526        self._unpersist_testbed_proxies()
 527
 528     def recover(self):
 529         # reload perviously persisted testbed access configurations
 530         self._load_testbed_proxies()
 531
 532         # recreate testbed proxies by reconnecting only
 533         self._init_testbed_controllers(recover = True)
 534
 535         # another time, for netrefs
 536         self._init_testbed_controllers(recover = True)
 537
 538     def is_finished(self, guid):
 539         testbed = self._testbed_for_guid(guid)
 540         if testbed != None:
 541             return testbed.status(guid) == AS.STATUS_FINISHED
 542         raise RuntimeError("No element exists with guid %d" % guid)
 543
 544     def set(self, guid, name, value, time = TIME_NOW):
 545         testbed = self._testbed_for_guid(guid)
 546         if testbed != None:
 547             testbed.set(guid, name, value, time)
 548         else:
 549             raise RuntimeError("No element exists with guid %d" % guid)
 550
 551     def get(self, guid, name, time = TIME_NOW):
 552         testbed = self._testbed_for_guid(guid)
 553         if testbed != None:
 554             return testbed.get(guid, name, time)
 555         raise RuntimeError("No element exists with guid %d" % guid)
 556
 557     def get_deferred(self, guid, name, time = TIME_NOW):
 558         testbed = self._testbed_for_guid(guid)
 559         if testbed != None:
 560             return testbed.get_deferred(guid, name, time)
 561         raise RuntimeError("No element exists with guid %d" % guid)
 562
 563     def get_factory_id(self, guid):
 564         testbed = self._testbed_for_guid(guid)
 565         if testbed != None:
 566             return testbed.get_factory_id(guid)
 567         raise RuntimeError("No element exists with guid %d" % guid)
 568
 569     def get_testbed_id(self, guid):
 570         testbed = self._testbed_for_guid(guid)
 571         if testbed != None:
 572             return testbed.testbed_id
 573         raise RuntimeError("No element exists with guid %d" % guid)
 574
 575     def get_testbed_version(self, guid):
 576         testbed = self._testbed_for_guid(guid)
 577         if testbed != None:
 578             return testbed.testbed_version
 579         raise RuntimeError("No element exists with guid %d" % guid)
 580
 581     def shutdown(self):
 582         exceptions = list()
 583         for testbed in self._testbeds.values():
 584             try:
 585                 testbed.shutdown()
 586             except:
 587                 exceptions.append(sys.exc_info())
 588         for exc_info in exceptions:
 589             raise exc_info[0], exc_info[1], exc_info[2]
 590
 591     def _testbed_for_guid(self, guid):
 592         for testbed_guid in self._testbeds.keys():
 593             if guid in self._guids_in_testbed(testbed_guid):
 594                 return self._testbeds[testbed_guid]
 595         return None
 596
 597     def _guids_in_testbed(self, testbed_guid):
 598         if testbed_guid not in self._testbeds:
 599             return set()
 600         if testbed_guid not in self._guids_in_testbed_cache:
 601             self._guids_in_testbed_cache[testbed_guid] = \
 602                 set(self._testbeds[testbed_guid].guids)
 603         return self._guids_in_testbed_cache[testbed_guid]
 604
 605     @staticmethod
 606     def _netref_component_split(component):
 607         match = COMPONENT_PATTERN.match(component)
 608         if match:
 609             return match.group("kind"), match.group("index")
 610         else:
 611             return component, None
 612
 613     _NETREF_COMPONENT_GETTERS = {
 614         'addr':
 615             lambda testbed, guid, index, name:
 616                 testbed.get_address(guid, int(index), name),
 617         'route' :
 618             lambda testbed, guid, index, name:
 619                 testbed.get_route(guid, int(index), name),
 620         'trace' :
 621             lambda testbed, guid, index, name:
 622                 testbed.trace(guid, index, name),
 623         '' :
 624             lambda testbed, guid, index, name:
 625                 testbed.get(guid, name),
 626     }
 627
 628     def resolve_netref_value(self, value, failval = None):
 629         match = ATTRIBUTE_PATTERN_BASE.search(value)
 630         if match:
 631             label = match.group("label")
 632             if label.startswith('GUID-'):
 633                 ref_guid = int(label[5:])
 634                 if ref_guid:
 635                     expr = match.group("expr")
 636                     component = (match.group("component") or "")[1:] # skip the dot
 637                     attribute = match.group("attribute")
 638
 639                     # split compound components into component kind and index
 640                     # eg: 'addr[0]' -> ('addr', '0')
 641                     component, component_index = self._netref_component_split(component)
 642
 643                     # find object and resolve expression
 644                     for ref_testbed_guid, ref_testbed in self._testbeds.iteritems():
 645                         if component not in self._NETREF_COMPONENT_GETTERS:
 646                             raise ValueError, "Malformed netref: %r - unknown component" % (expr,)
 647                         elif ref_guid not in self._guids_in_testbed(ref_testbed_guid):
 648                             pass
 649                         else:
 650                             ref_value = self._NETREF_COMPONENT_GETTERS[component](
 651                                 ref_testbed, ref_guid, component_index, attribute)
 652                             if ref_value:
 653                                 return value.replace(match.group(), ref_value)
 654         # couldn't find value
 655         return failval
 656
 657     def do_netrefs(self, data, fail_if_undefined = False):
 658         # element netrefs
 659         for (testbed_guid, guid), attrs in self._netrefs.items():
 660             testbed = self._testbeds.get(testbed_guid)
 661             if testbed is not None:
 662                 for name in set(attrs):
 663                     value = testbed.get(guid, name)
 664                     if isinstance(value, basestring):
 665                         ref_value = self.resolve_netref_value(value)
 666                         if ref_value is not None:
 667                             testbed.set(guid, name, ref_value)
 668                             attrs.remove(name)
 669                         elif fail_if_undefined:
 670                             raise ValueError, "Unresolvable netref in: %r=%r" % (name,value,)
 671                 if not attrs:
 672                     del self._netrefs[(testbed_guid, guid)]
 673
 674         # testbed netrefs
 675         for testbed_guid, attrs in self._testbed_netrefs.items():
 676             tb_data = dict(data.get_attribute_data(testbed_guid))
 677             if data:
 678                 for name in set(attrs):
 679                     value = tb_data.get(name)
 680                     if isinstance(value, basestring):
 681                         ref_value = self.resolve_netref_value(value)
 682                         if ref_value is not None:
 683                             data.set_attribute_data(testbed_guid, name, ref_value)
 684                             attrs.remove(name)
 685                         elif fail_if_undefined:
 686                             raise ValueError, "Unresolvable netref in: %r" % (value,)
 687                 if not attrs:
 688                     del self._testbed_netrefs[testbed_guid]
 689
 690
 691     def _init_testbed_controllers(self, data, recover = False):
 692         blacklist_testbeds = set(self._testbeds)
 693         element_guids = list()
 694         label_guids = dict()
 695         data_guids = data.guids
 696
 697         # create testbed controllers
 698         for guid in data_guids:
 699             if data.is_testbed_data(guid):
 700                 if guid not in self._testbeds:
 701                     self._create_testbed_controller(guid, data, element_guids,
 702                             recover)
 703             else:
 704                 (testbed_guid, factory_id) = data.get_box_data(guid)
 705                 if testbed_guid not in blacklist_testbeds:
 706                     element_guids.append(guid)
 707                     label = data.get_attribute_data(guid, "label")
 708                     if label is not None:
 709                         if label in label_guids:
 710                             raise RuntimeError, "Label %r is not unique" % (label,)
 711                         label_guids[label] = guid
 712
 713         # replace references to elements labels for its guid
 714         self._resolve_labels(data, data_guids, label_guids)
 715
 716         # program testbed controllers
 717         if not recover:
 718             self._program_testbed_controllers(element_guids, data)
 719
 720     def _resolve_labels(self, data, data_guids, label_guids):
 721         netrefs = self._netrefs
 722         testbed_netrefs = self._testbed_netrefs
 723         for guid in data_guids:
 724             for name, value in data.get_attribute_data(guid):
 725                 if isinstance(value, basestring):
 726                     match = ATTRIBUTE_PATTERN_BASE.search(value)
 727                     if match:
 728                         label = match.group("label")
 729                         if not label.startswith('GUID-'):
 730                             ref_guid = label_guids.get(label)
 731                             if ref_guid is not None:
 732                                 value = ATTRIBUTE_PATTERN_BASE.sub(
 733                                     ATTRIBUTE_PATTERN_GUID_SUB % dict(
 734                                         guid = 'GUID-%d' % (ref_guid,),
 735                                         expr = match.group("expr"),
 736                                         label = label),
 737                                     value)
 738                                 data.set_attribute_data(guid, name, value)
 739
 740                                 # memorize which guid-attribute pairs require
 741                                 # postprocessing, to avoid excessive controller-testbed
 742                                 # communication at configuration time
 743                                 # (which could require high-latency network I/O)
 744                                 if not data.is_testbed_data(guid):
 745                                     (testbed_guid, factory_id) = data.get_box_data(guid)
 746                                     netrefs[(testbed_guid, guid)].add(name)
 747                                 else:
 748                                     testbed_netrefs[guid].add(name)
 749
 750     def _create_testbed_controller(self, guid, data, element_guids, recover):
 751         (testbed_id, testbed_version) = data.get_testbed_data(guid)
 752         deployment_config = self._deployment_config.get(guid)
 753
 754         # deferred import because proxy needs
 755         # our class definitions to define proxies
 756         import nepi.util.proxy as proxy
 757
 758         if deployment_config is None:
 759             # need to create one
 760             deployment_config = proxy.AccessConfiguration()
 761
 762             for (name, value) in data.get_attribute_data(guid):
 763                 if value is not None and deployment_config.has_attribute(name):
 764                     # if any deployment config attribute has a netref, we can't
 765                     # create this controller yet
 766                     if isinstance(value, basestring) and ATTRIBUTE_PATTERN_BASE.search(value):
 767                         # remember to re-issue this one
 768                         self._netreffed_testbeds.add(guid)
 769                         return
 770
 771                     # copy deployment config attribute
 772                     deployment_config.set_attribute_value(name, value)
 773
 774             # commit config
 775             self._deployment_config[guid] = deployment_config
 776
 777         if deployment_config is not None:
 778             # force recovery mode
 779             deployment_config.set_attribute_value("recover",recover)
 780
 781         testbed = proxy.create_testbed_controller(testbed_id,
 782                 testbed_version, deployment_config)
 783         for (name, value) in data.get_attribute_data(guid):
 784             testbed.defer_configure(name, value)
 785         self._testbeds[guid] = testbed
 786         if guid in self._netreffed_testbeds:
 787             self._netreffed_testbeds.remove(guid)
 788
 789     def _program_testbed_controllers(self, element_guids, data):
 790         for guid in element_guids:
 791             (testbed_guid, factory_id) = data.get_box_data(guid)
 792             testbed = self._testbeds.get(testbed_guid)
 793             if testbed:
 794                 testbed.defer_create(guid, factory_id)
 795                 for (name, value) in data.get_attribute_data(guid):
 796                     # Try to resolve create-time netrefs, if possible
 797                     if isinstance(value, basestring) and ATTRIBUTE_PATTERN_BASE.search(value):
 798                         try:
 799                             nuvalue = self.resolve_netref_value(value)
 800                         except:
 801                             # Any trouble means we're not in shape to resolve the netref yet
 802                             nuvalue = None
 803                         if nuvalue is not None:
 804                             # Only if we succeed we remove the netref deferral entry
 805                             value = nuvalue
 806                             data.set_attribute_data(guid, name, value)
 807                             if (testbed_guid, guid) in self._netrefs:
 808                                 self._netrefs[(testbed_guid, guid)].discard(name)
 809                     testbed.defer_create_set(guid, name, value)
 810
 811         for guid in element_guids:
 812             (testbed_guid, factory_id) = data.get_box_data(guid)
 813             testbed = self._testbeds.get(testbed_guid)
 814             if testbed:
 815                 for (connector_type_name, cross_guid, cross_connector_type_name) \
 816                         in data.get_connection_data(guid):
 817                     (testbed_guid, factory_id) = data.get_box_data(guid)
 818                     (cross_testbed_guid, cross_factory_id) = data.get_box_data(
 819                             cross_guid)
 820                     if testbed_guid == cross_testbed_guid:
 821                         testbed.defer_connect(guid, connector_type_name,
 822                                 cross_guid, cross_connector_type_name)
 823                 for trace_id in data.get_trace_data(guid):
 824                     testbed.defer_add_trace(guid, trace_id)
 825                 for (address, netprefix, broadcast) in \
 826                         data.get_address_data(guid):
 827                     if address != None:
 828                         testbed.defer_add_address(guid, address, netprefix,
 829                                 broadcast)
 830                 for (destination, netprefix, nexthop, metric) in data.get_route_data(guid):
 831                     testbed.defer_add_route(guid, destination, netprefix, nexthop, metric)
 832
 833     def _program_testbed_cross_connections(self, data):
 834         data_guids = data.guids
 835
 836         for guid in data_guids:
 837             if not data.is_testbed_data(guid):
 838                 (testbed_guid, factory_id) = data.get_box_data(guid)
 839                 testbed = self._testbeds.get(testbed_guid)
 840                 if testbed:
 841                     for (connector_type_name, cross_guid, cross_connector_type_name) \
 842                             in data.get_connection_data(guid):
 843                         (testbed_guid, factory_id) = data.get_box_data(guid)
 844                         (cross_testbed_guid, cross_factory_id) = data.get_box_data(
 845                                 cross_guid)
 846                         if testbed_guid != cross_testbed_guid:
 847                             cross_testbed = self._testbeds[cross_testbed_guid]
 848                             cross_testbed_id = cross_testbed.testbed_id
 849                             testbed.defer_cross_connect(guid, connector_type_name, cross_guid,
 850                                     cross_testbed_guid, cross_testbed_id, cross_factory_id,
 851                                     cross_connector_type_name)
 852                             # save cross data for later
 853                             self._add_crossdata(testbed_guid, guid, cross_testbed_guid,
 854                                     cross_guid)
 855
 856     def _add_crossdata(self, testbed_guid, guid, cross_testbed_guid, cross_guid):
 857         if testbed_guid not in self._cross_data:
 858             self._cross_data[testbed_guid] = dict()
 859         if cross_testbed_guid not in self._cross_data[testbed_guid]:
 860             self._cross_data[testbed_guid][cross_testbed_guid] = set()
 861         self._cross_data[testbed_guid][cross_testbed_guid].add(cross_guid)
 862
 863     def _get_cross_data(self, testbed_guid):
 864         cross_data = dict()
 865         if not testbed_guid in self._cross_data:
 866             return cross_data
 867         for cross_testbed_guid, guid_list in \
 868                 self._cross_data[testbed_guid].iteritems():
 869             cross_data[cross_testbed_guid] = dict()
 870             cross_testbed = self._testbeds[cross_testbed_guid]
 871             for cross_guid in guid_list:
 872                 elem_cross_data = dict(
 873                     _guid = cross_guid,
 874                     _testbed_guid = cross_testbed_guid,
 875                     _testbed_id = cross_testbed.testbed_id,
 876                     _testbed_version = cross_testbed.testbed_version)
 877                 cross_data[cross_testbed_guid][cross_guid] = elem_cross_data
 878                 attributes_list = cross_testbed.get_attribute_list(cross_guid)
 879                 for attr_name in attributes_list:
 880                     attr_value = cross_testbed.get(cross_guid, attr_name)
 881                     elem_cross_data[attr_name] = attr_value
 882         return cross_data
 883