src/nepi/core/execute.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from nepi.core.attributes import Attribute, AttributesMap
   5 from nepi.util import validation
   6 from nepi.util.constants import ApplicationStatus as AS, TIME_NOW
   7 from nepi.util.parser._xml import XmlExperimentParser
   8 import sys
   9 import re
  10 import threading
  11 import ConfigParser
  12 import os
  13 import collections
  14 import functools
  15
  16 ATTRIBUTE_PATTERN_BASE = re.compile(r"\{#\[(?P<label>[-a-zA-Z0-9._]*)\](?P<expr>(?P<component>\.addr\[[0-9]+\]|\.route\[[0-9]+\]|\.trace\[[0-9]+\])?.\[(?P<attribute>[-a-zA-Z0-9._]*)\])#}")
  17 ATTRIBUTE_PATTERN_GUID_SUB = r"{#[%(guid)s]%(expr)s#}"
  18 COMPONENT_PATTERN = re.compile(r"(?P<kind>[a-z]*)\[(?P<index>.*)\]")
  19
  20 class TestbedController(object):
  21     def __init__(self, testbed_id, testbed_version):
  22         self._testbed_id = testbed_id
  23         self._testbed_version = testbed_version
  24
  25     @property
  26     def testbed_id(self):
  27         return self._testbed_id
  28
  29     @property
  30     def testbed_version(self):
  31         return self._testbed_version
  32
  33     @property
  34     def guids(self):
  35         raise NotImplementedError
  36
  37     def defer_configure(self, name, value):
  38         """Instructs setting a configuartion attribute for the testbed instance"""
  39         raise NotImplementedError
  40
  41     def defer_create(self, guid, factory_id):
  42         """Instructs creation of element """
  43         raise NotImplementedError
  44
  45     def defer_create_set(self, guid, name, value):
  46         """Instructs setting an initial attribute on an element"""
  47         raise NotImplementedError
  48
  49     def defer_factory_set(self, guid, name, value):
  50         """Instructs setting an attribute on a factory"""
  51         raise NotImplementedError
  52
  53     def defer_connect(self, guid1, connector_type_name1, guid2,
  54             connector_type_name2):
  55         """Instructs creation of a connection between the given connectors"""
  56         raise NotImplementedError
  57
  58     def defer_cross_connect(self,
  59             guid, connector_type_name,
  60             cross_guid, cross_testbed_guid,
  61             cross_testbed_id, cross_factory_id,
  62             cross_connector_type_name):
  63         """
  64         Instructs creation of a connection between the given connectors
  65         of different testbed instances
  66         """
  67         raise NotImplementedError
  68
  69     def defer_add_trace(self, guid, trace_id):
  70         """Instructs the addition of a trace"""
  71         raise NotImplementedError
  72
  73     def defer_add_address(self, guid, address, netprefix, broadcast):
  74         """Instructs the addition of an address"""
  75         raise NotImplementedError
  76
  77     def defer_add_route(self, guid, destination, netprefix, nexthop, metric = 0):
  78         """Instructs the addition of a route"""
  79         raise NotImplementedError
  80
  81     def do_setup(self):
  82         """After do_setup the testbed initial configuration is done"""
  83         raise NotImplementedError
  84
  85     def do_create(self):
  86         """
  87         After do_create all instructed elements are created and
  88         attributes setted
  89         """
  90         raise NotImplementedError
  91
  92     def do_connect_init(self):
  93         """
  94         After do_connect_init all internal connections between testbed elements
  95         are initiated
  96         """
  97         raise NotImplementedError
  98
  99     def do_connect_compl(self):
 100         """
 101         After do_connect all internal connections between testbed elements
 102         are completed
 103         """
 104         raise NotImplementedError
 105
 106     def do_preconfigure(self):
 107         """
 108         Done just before resolving netrefs, after connection, before cross connections,
 109         useful for early stages of configuration, for setting up stuff that might be
 110         required for netref resolution.
 111         """
 112         raise NotImplementedError
 113
 114     def do_configure(self):
 115         """After do_configure elements are configured"""
 116         raise NotImplementedError
 117
 118     def do_prestart(self):
 119         """Before do_start elements are prestart-configured"""
 120         raise NotImplementedError
 121
 122     def do_cross_connect_init(self, cross_data):
 123         """
 124         After do_cross_connect_init initiation of all external connections
 125         between different testbed elements is performed
 126         """
 127         raise NotImplementedError
 128
 129     def do_cross_connect_compl(self, cross_data):
 130         """
 131         After do_cross_connect_compl completion of all external connections
 132         between different testbed elements is performed
 133         """
 134         raise NotImplementedError
 135
 136     def start(self):
 137         raise NotImplementedError
 138
 139     def stop(self):
 140         raise NotImplementedError
 141
 142     def set(self, guid, name, value, time = TIME_NOW):
 143         raise NotImplementedError
 144
 145     def get(self, guid, name, time = TIME_NOW):
 146         raise NotImplementedError
 147
 148     def get_route(self, guid, index, attribute):
 149         """
 150         Params:
 151
 152             guid: guid of box to query
 153             index: number of routing entry to fetch
 154             attribute: one of Destination, NextHop, NetPrefix
 155         """
 156         raise NotImplementedError
 157
 158     def get_address(self, guid, index, attribute='Address'):
 159         """
 160         Params:
 161
 162             guid: guid of box to query
 163             index: number of inteface to select
 164             attribute: one of Address, NetPrefix, Broadcast
 165         """
 166         raise NotImplementedError
 167
 168     def get_attribute_list(self, guid, filter_flags = None):
 169         raise NotImplementedError
 170
 171     def get_factory_id(self, guid):
 172         raise NotImplementedError
 173
 174     def action(self, time, guid, action):
 175         raise NotImplementedError
 176
 177     def status(self, guid):
 178         raise NotImplementedError
 179
 180     def trace(self, guid, trace_id, attribute='value'):
 181         raise NotImplementedError
 182
 183     def traces_info(self):
 184         """ dictionary of dictionaries:
 185             traces_info = dict({
 186                 guid = dict({
 187                     trace_id = dict({
 188                             host = host,
 189                             filepath = filepath,
 190                             filesize = size in bytes,
 191                         })
 192                 })
 193             })"""
 194         raise NotImplementedError
 195
 196     def shutdown(self):
 197         raise NotImplementedError
 198
 199 class ExperimentController(object):
 200     def __init__(self, experiment_xml, root_dir):
 201         self._experiment_xml = experiment_xml
 202         self._testbeds = dict()
 203         self._deployment_config = dict()
 204         self._netrefs = collections.defaultdict(set)
 205         self._testbed_netrefs = collections.defaultdict(set)
 206         self._cross_data = dict()
 207         self._root_dir = root_dir
 208         self._netreffed_testbeds = set()
 209         self._guids_in_testbed_cache = dict()
 210
 211         self.persist_experiment_xml()
 212
 213     @property
 214     def experiment_xml(self):
 215         return self._experiment_xml
 216
 217     @property
 218     def guids(self):
 219         guids = list()
 220         for testbed_guid in self._testbeds.keys():
 221             _guids = self._guids_in_testbed(testbed_guid)
 222             if _guids:
 223                 guids.extend(_guids)
 224         return guids
 225
 226     def persist_experiment_xml(self):
 227         xml_path = os.path.join(self._root_dir, "experiment.xml")
 228         f = open(xml_path, "w")
 229         f.write(self._experiment_xml)
 230         f.close()
 231
 232     def trace(self, guid, trace_id, attribute='value'):
 233         testbed = self._testbed_for_guid(guid)
 234         if testbed != None:
 235             return testbed.trace(guid, trace_id, attribute)
 236         raise RuntimeError("No element exists with guid %d" % guid)
 237
 238     def traces_info(self):
 239         traces_info = dict()
 240         for guid, testbed in self._testbeds.iteritems():
 241             tinfo = testbed.traces_info()
 242             if tinfo:
 243                 traces_info[guid] = testbed.traces_info()
 244         return traces_info
 245
 246     @staticmethod
 247     def _parallel(callables):
 248         excs = []
 249         def wrap(callable):
 250             @functools.wraps(callable)
 251             def wrapped(*p, **kw):
 252                 try:
 253                     callable(*p, **kw)
 254                 except:
 255                     import traceback
 256                     traceback.print_exc(file=sys.stderr)
 257                     excs.append(sys.exc_info())
 258             return wrapped
 259         threads = [ threading.Thread(target=wrap(callable)) for callable in callables ]
 260         for thread in threads:
 261             thread.start()
 262         for thread in threads:
 263             thread.join()
 264         for exc in excs:
 265             eTyp, eVal, eLoc = exc
 266             raise eTyp, eVal, eLoc
 267
 268     def start(self):
 269         parser = XmlExperimentParser()
 270         data = parser.from_xml_to_data(self._experiment_xml)
 271
 272         self._init_testbed_controllers(data)
 273
 274         # persist testbed connection data, for potential recovery
 275         self._persist_testbed_proxies()
 276
 277         def steps_to_configure(self, allowed_guids):
 278             # perform setup in parallel for all test beds,
 279             # wait for all threads to finish
 280             self._parallel([testbed.do_setup
 281                             for guid,testbed in self._testbeds.iteritems()
 282                             if guid in allowed_guids])
 283
 284             # perform create-connect in parallel, wait
 285             # (internal connections only)
 286             self._parallel([testbed.do_create
 287                             for guid,testbed in self._testbeds.iteritems()
 288                             if guid in allowed_guids])
 289
 290             self._parallel([testbed.do_connect_init
 291                             for guid,testbed in self._testbeds.iteritems()
 292                             if guid in allowed_guids])
 293
 294             self._parallel([testbed.do_connect_compl
 295                             for guid,testbed in self._testbeds.iteritems()
 296                             if guid in allowed_guids])
 297
 298             self._parallel([testbed.do_preconfigure
 299                             for guid,testbed in self._testbeds.iteritems()
 300                             if guid in allowed_guids])
 301             self._clear_caches()
 302
 303         steps_to_configure(self, self._testbeds)
 304
 305         if self._netreffed_testbeds:
 306             # initally resolve netrefs
 307             self.do_netrefs(data, fail_if_undefined=False)
 308
 309             # rinse and repeat, for netreffed testbeds
 310             netreffed_testbeds = set(self._netreffed_testbeds)
 311
 312             self._init_testbed_controllers(data)
 313
 314             # persist testbed connection data, for potential recovery
 315             self._persist_testbed_proxies()
 316
 317             # configure dependant testbeds
 318             steps_to_configure(self, netreffed_testbeds)
 319
 320         # final netref step, fail if anything's left unresolved
 321         self.do_netrefs(data, fail_if_undefined=True)
 322
 323         self._program_testbed_cross_connections(data)
 324
 325         # perform do_configure in parallel for al testbeds
 326         # (it's internal configuration for each)
 327         self._parallel([testbed.do_configure
 328                         for testbed in self._testbeds.itervalues()])
 329
 330         self._clear_caches()
 331
 332         #print >>sys.stderr, "DO IT"
 333         #import time
 334         #time.sleep(60)
 335
 336         # cross-connect (cannot be done in parallel)
 337         for guid, testbed in self._testbeds.iteritems():
 338             cross_data = self._get_cross_data(guid)
 339             testbed.do_cross_connect_init(cross_data)
 340         for guid, testbed in self._testbeds.iteritems():
 341             cross_data = self._get_cross_data(guid)
 342             testbed.do_cross_connect_compl(cross_data)
 343
 344         self._clear_caches()
 345
 346         # Last chance to configure (parallel on all testbeds)
 347         self._parallel([testbed.do_prestart
 348                         for testbed in self._testbeds.itervalues()])
 349
 350         self._clear_caches()
 351
 352         # start experiment (parallel start on all testbeds)
 353         self._parallel([testbed.start
 354                         for testbed in self._testbeds.itervalues()])
 355
 356         self._clear_caches()
 357
 358     def _clear_caches(self):
 359         # Cleaning cache for safety.
 360         self._guids_in_testbed_cache = dict()
 361
 362     def _persist_testbed_proxies(self):
 363         TRANSIENT = ('Recover',)
 364
 365         # persist access configuration for all testbeds, so that
 366         # recovery mode can reconnect to them if it becomes necessary
 367         conf = ConfigParser.RawConfigParser()
 368         for testbed_guid, testbed_config in self._deployment_config.iteritems():
 369             testbed_guid = str(testbed_guid)
 370             conf.add_section(testbed_guid)
 371             for attr in testbed_config.get_attribute_list():
 372                 if attr not in TRANSIENT:
 373                     conf.set(testbed_guid, attr,
 374                         testbed_config.get_attribute_value(attr))
 375
 376         f = open(os.path.join(self._root_dir, 'deployment_config.ini'), 'w')
 377         conf.write(f)
 378         f.close()
 379
 380     def _load_testbed_proxies(self):
 381         TYPEMAP = {
 382             STRING : 'get',
 383             INTEGER : 'getint',
 384             FLOAT : 'getfloat',
 385             BOOLEAN : 'getboolean',
 386         }
 387
 388         # deferred import because proxy needs
 389         # our class definitions to define proxies
 390         import nepi.util.proxy as proxy
 391
 392         conf = ConfigParser.RawConfigParser()
 393         conf.read(os.path.join(self._root_dir, 'deployment_config.ini'))
 394         for testbed_guid in conf.sections():
 395             testbed_config = proxy.AccessConfiguration()
 396             for attr in conf.options(testbed_guid):
 397                 testbed_config.set_attribute_value(attr,
 398                     conf.get(testbed_guid, attr) )
 399
 400             testbed_guid = str(testbed_guid)
 401             conf.add_section(testbed_guid)
 402             for attr in testbed_config.get_attribute_list():
 403                 if attr not in TRANSIENT:
 404                     getter = getattr(conf, TYPEMAP.get(
 405                         testbed_config.get_attribute_type(attr),
 406                         'get') )
 407                     testbed_config.set_attribute_value(
 408                         testbed_guid, attr, getter(attr))
 409
 410     def _unpersist_testbed_proxies(self):
 411         try:
 412             os.remove(os.path.join(self._root_dir, 'deployment_config.ini'))
 413         except:
 414             # Just print exceptions, this is just cleanup
 415             import traceback
 416             ######## BUG ##########
 417             #BUG: If the next line is uncomented pyQt explodes when shutting down the experiment !!!!!!!!
 418             #traceback.print_exc(file=sys.stderr)
 419
 420     def stop(self):
 421        for testbed in self._testbeds.values():
 422            testbed.stop()
 423        self._unpersist_testbed_proxies()
 424
 425     def recover(self):
 426         # reload perviously persisted testbed access configurations
 427         self._load_testbed_proxies()
 428
 429         # recreate testbed proxies by reconnecting only
 430         self._init_testbed_controllers(recover = True)
 431
 432         # another time, for netrefs
 433         self._init_testbed_controllers(recover = True)
 434
 435     def is_finished(self, guid):
 436         testbed = self._testbed_for_guid(guid)
 437         if testbed != None:
 438             return testbed.status(guid) == AS.STATUS_FINISHED
 439         raise RuntimeError("No element exists with guid %d" % guid)
 440
 441     def set(self, guid, name, value, time = TIME_NOW):
 442         testbed = self._testbed_for_guid(guid)
 443         if testbed != None:
 444             testbed.set(guid, name, value, time)
 445         else:
 446             raise RuntimeError("No element exists with guid %d" % guid)
 447
 448     def get(self, guid, name, time = TIME_NOW):
 449         testbed = self._testbed_for_guid(guid)
 450         if testbed != None:
 451             return testbed.get(guid, name, time)
 452         raise RuntimeError("No element exists with guid %d" % guid)
 453
 454     def get_deferred(self, guid, name, time = TIME_NOW):
 455         testbed = self._testbed_for_guid(guid)
 456         if testbed != None:
 457             return testbed.get_deferred(guid, name, time)
 458         raise RuntimeError("No element exists with guid %d" % guid)
 459
 460     def get_factory_id(self, guid):
 461         testbed = self._testbed_for_guid(guid)
 462         if testbed != None:
 463             return testbed.get_factory_id(guid)
 464         raise RuntimeError("No element exists with guid %d" % guid)
 465
 466     def get_testbed_id(self, guid):
 467         testbed = self._testbed_for_guid(guid)
 468         if testbed != None:
 469             return testbed.testbed_id
 470         raise RuntimeError("No element exists with guid %d" % guid)
 471
 472     def get_testbed_version(self, guid):
 473         testbed = self._testbed_for_guid(guid)
 474         if testbed != None:
 475             return testbed.testbed_version
 476         raise RuntimeError("No element exists with guid %d" % guid)
 477
 478     def shutdown(self):
 479         exceptions = list()
 480         for testbed in self._testbeds.values():
 481             try:
 482                 testbed.shutdown()
 483             except:
 484                 exceptions.append(sys.exc_info())
 485         for exc_info in exceptions:
 486             raise exc_info[0], exc_info[1], exc_info[2]
 487
 488     def _testbed_for_guid(self, guid):
 489         for testbed_guid in self._testbeds.keys():
 490             if guid in self._guids_in_testbed(testbed_guid):
 491                 return self._testbeds[testbed_guid]
 492         return None
 493
 494     def _guids_in_testbed(self, testbed_guid):
 495         if testbed_guid not in self._testbeds:
 496             return set()
 497         if testbed_guid not in self._guids_in_testbed_cache:
 498             self._guids_in_testbed_cache[testbed_guid] = \
 499                 set(self._testbeds[testbed_guid].guids)
 500         return self._guids_in_testbed_cache[testbed_guid]
 501
 502     @staticmethod
 503     def _netref_component_split(component):
 504         match = COMPONENT_PATTERN.match(component)
 505         if match:
 506             return match.group("kind"), match.group("index")
 507         else:
 508             return component, None
 509
 510     _NETREF_COMPONENT_GETTERS = {
 511         'addr':
 512             lambda testbed, guid, index, name:
 513                 testbed.get_address(guid, int(index), name),
 514         'route' :
 515             lambda testbed, guid, index, name:
 516                 testbed.get_route(guid, int(index), name),
 517         'trace' :
 518             lambda testbed, guid, index, name:
 519                 testbed.trace(guid, index, name),
 520         '' :
 521             lambda testbed, guid, index, name:
 522                 testbed.get(guid, name),
 523     }
 524
 525     def resolve_netref_value(self, value, failval = None):
 526         match = ATTRIBUTE_PATTERN_BASE.search(value)
 527         if match:
 528             label = match.group("label")
 529             if label.startswith('GUID-'):
 530                 ref_guid = int(label[5:])
 531                 if ref_guid:
 532                     expr = match.group("expr")
 533                     component = (match.group("component") or "")[1:] # skip the dot
 534                     attribute = match.group("attribute")
 535
 536                     # split compound components into component kind and index
 537                     # eg: 'addr[0]' -> ('addr', '0')
 538                     component, component_index = self._netref_component_split(component)
 539
 540                     # find object and resolve expression
 541                     for ref_testbed_guid, ref_testbed in self._testbeds.iteritems():
 542                         if component not in self._NETREF_COMPONENT_GETTERS:
 543                             raise ValueError, "Malformed netref: %r - unknown component" % (expr,)
 544                         elif ref_guid not in self._guids_in_testbed(ref_testbed_guid):
 545                             pass
 546                         else:
 547                             ref_value = self._NETREF_COMPONENT_GETTERS[component](
 548                                 ref_testbed, ref_guid, component_index, attribute)
 549                             if ref_value:
 550                                 return value.replace(match.group(), ref_value)
 551         # couldn't find value
 552         return failval
 553
 554     def do_netrefs(self, data, fail_if_undefined = False):
 555         # element netrefs
 556         for (testbed_guid, guid), attrs in self._netrefs.items():
 557             testbed = self._testbeds.get(testbed_guid)
 558             if testbed is not None:
 559                 for name in set(attrs):
 560                     value = testbed.get(guid, name)
 561                     if isinstance(value, basestring):
 562                         ref_value = self.resolve_netref_value(value)
 563                         if ref_value is not None:
 564                             testbed.set(guid, name, ref_value)
 565                             attrs.remove(name)
 566                         elif fail_if_undefined:
 567                             raise ValueError, "Unresolvable netref in: %r=%r" % (name,value,)
 568                 if not attrs:
 569                     del self._netrefs[(testbed_guid, guid)]
 570
 571         # testbed netrefs
 572         for testbed_guid, attrs in self._testbed_netrefs.items():
 573             tb_data = dict(data.get_attribute_data(testbed_guid))
 574             if data:
 575                 for name in set(attrs):
 576                     value = tb_data.get(name)
 577                     if isinstance(value, basestring):
 578                         ref_value = self.resolve_netref_value(value)
 579                         if ref_value is not None:
 580                             data.set_attribute_data(testbed_guid, name, ref_value)
 581                             attrs.remove(name)
 582                         elif fail_if_undefined:
 583                             raise ValueError, "Unresolvable netref in: %r" % (value,)
 584                 if not attrs:
 585                     del self._testbed_netrefs[testbed_guid]
 586
 587
 588     def _init_testbed_controllers(self, data, recover = False):
 589         blacklist_testbeds = set(self._testbeds)
 590         element_guids = list()
 591         label_guids = dict()
 592         data_guids = data.guids
 593
 594         # create testbed controllers
 595         for guid in data_guids:
 596             if data.is_testbed_data(guid):
 597                 if guid not in self._testbeds:
 598                     self._create_testbed_controller(guid, data, element_guids,
 599                             recover)
 600             else:
 601                 (testbed_guid, factory_id) = data.get_box_data(guid)
 602                 if testbed_guid not in blacklist_testbeds:
 603                     element_guids.append(guid)
 604                     label = data.get_attribute_data(guid, "label")
 605                     if label is not None:
 606                         if label in label_guids:
 607                             raise RuntimeError, "Label %r is not unique" % (label,)
 608                         label_guids[label] = guid
 609
 610         # replace references to elements labels for its guid
 611         self._resolve_labels(data, data_guids, label_guids)
 612
 613         # program testbed controllers
 614         if not recover:
 615             self._program_testbed_controllers(element_guids, data)
 616
 617     def _resolve_labels(self, data, data_guids, label_guids):
 618         netrefs = self._netrefs
 619         testbed_netrefs = self._testbed_netrefs
 620         for guid in data_guids:
 621             for name, value in data.get_attribute_data(guid):
 622                 if isinstance(value, basestring):
 623                     match = ATTRIBUTE_PATTERN_BASE.search(value)
 624                     if match:
 625                         label = match.group("label")
 626                         if not label.startswith('GUID-'):
 627                             ref_guid = label_guids.get(label)
 628                             if ref_guid is not None:
 629                                 value = ATTRIBUTE_PATTERN_BASE.sub(
 630                                     ATTRIBUTE_PATTERN_GUID_SUB % dict(
 631                                         guid = 'GUID-%d' % (ref_guid,),
 632                                         expr = match.group("expr"),
 633                                         label = label),
 634                                     value)
 635                                 data.set_attribute_data(guid, name, value)
 636
 637                                 # memorize which guid-attribute pairs require
 638                                 # postprocessing, to avoid excessive controller-testbed
 639                                 # communication at configuration time
 640                                 # (which could require high-latency network I/O)
 641                                 if not data.is_testbed_data(guid):
 642                                     (testbed_guid, factory_id) = data.get_box_data(guid)
 643                                     netrefs[(testbed_guid, guid)].add(name)
 644                                 else:
 645                                     testbed_netrefs[guid].add(name)
 646
 647     def _create_testbed_controller(self, guid, data, element_guids, recover):
 648         (testbed_id, testbed_version) = data.get_testbed_data(guid)
 649         deployment_config = self._deployment_config.get(guid)
 650
 651         # deferred import because proxy needs
 652         # our class definitions to define proxies
 653         import nepi.util.proxy as proxy
 654
 655         if deployment_config is None:
 656             # need to create one
 657             deployment_config = proxy.AccessConfiguration()
 658
 659             for (name, value) in data.get_attribute_data(guid):
 660                 if value is not None and deployment_config.has_attribute(name):
 661                     # if any deployment config attribute has a netref, we can't
 662                     # create this controller yet
 663                     if isinstance(value, basestring) and ATTRIBUTE_PATTERN_BASE.search(value):
 664                         # remember to re-issue this one
 665                         self._netreffed_testbeds.add(guid)
 666                         return
 667
 668                     # copy deployment config attribute
 669                     deployment_config.set_attribute_value(name, value)
 670
 671             # commit config
 672             self._deployment_config[guid] = deployment_config
 673
 674         if deployment_config is not None:
 675             # force recovery mode
 676             deployment_config.set_attribute_value("recover",recover)
 677
 678         testbed = proxy.create_testbed_controller(testbed_id,
 679                 testbed_version, deployment_config)
 680         for (name, value) in data.get_attribute_data(guid):
 681             testbed.defer_configure(name, value)
 682         self._testbeds[guid] = testbed
 683         if guid in self._netreffed_testbeds:
 684             self._netreffed_testbeds.remove(guid)
 685
 686     def _program_testbed_controllers(self, element_guids, data):
 687         for guid in element_guids:
 688             (testbed_guid, factory_id) = data.get_box_data(guid)
 689             testbed = self._testbeds.get(testbed_guid)
 690             if testbed:
 691                 testbed.defer_create(guid, factory_id)
 692                 for (name, value) in data.get_attribute_data(guid):
 693                     # Try to resolve create-time netrefs, if possible
 694                     if isinstance(value, basestring) and ATTRIBUTE_PATTERN_BASE.search(value):
 695                         try:
 696                             nuvalue = self.resolve_netref_value(value)
 697                         except:
 698                             # Any trouble means we're not in shape to resolve the netref yet
 699                             nuvalue = None
 700                         if nuvalue is not None:
 701                             # Only if we succeed we remove the netref deferral entry
 702                             value = nuvalue
 703                             data.set_attribute_data(guid, name, value)
 704                             if (testbed_guid, guid) in self._netrefs:
 705                                 self._netrefs[(testbed_guid, guid)].discard(name)
 706                     testbed.defer_create_set(guid, name, value)
 707
 708         for guid in element_guids:
 709             (testbed_guid, factory_id) = data.get_box_data(guid)
 710             testbed = self._testbeds.get(testbed_guid)
 711             if testbed:
 712                 for (connector_type_name, cross_guid, cross_connector_type_name) \
 713                         in data.get_connection_data(guid):
 714                     (testbed_guid, factory_id) = data.get_box_data(guid)
 715                     (cross_testbed_guid, cross_factory_id) = data.get_box_data(
 716                             cross_guid)
 717                     if testbed_guid == cross_testbed_guid:
 718                         testbed.defer_connect(guid, connector_type_name,
 719                                 cross_guid, cross_connector_type_name)
 720                 for trace_id in data.get_trace_data(guid):
 721                     testbed.defer_add_trace(guid, trace_id)
 722                 for (address, netprefix, broadcast) in \
 723                         data.get_address_data(guid):
 724                     if address != None:
 725                         testbed.defer_add_address(guid, address, netprefix,
 726                                 broadcast)
 727                 for (destination, netprefix, nexthop, metric) in data.get_route_data(guid):
 728                     testbed.defer_add_route(guid, destination, netprefix, nexthop, metric)
 729
 730     def _program_testbed_cross_connections(self, data):
 731         data_guids = data.guids
 732
 733         for guid in data_guids:
 734             if not data.is_testbed_data(guid):
 735                 (testbed_guid, factory_id) = data.get_box_data(guid)
 736                 testbed = self._testbeds.get(testbed_guid)
 737                 if testbed:
 738                     for (connector_type_name, cross_guid, cross_connector_type_name) \
 739                             in data.get_connection_data(guid):
 740                         (testbed_guid, factory_id) = data.get_box_data(guid)
 741                         (cross_testbed_guid, cross_factory_id) = data.get_box_data(
 742                                 cross_guid)
 743                         if testbed_guid != cross_testbed_guid:
 744                             cross_testbed = self._testbeds[cross_testbed_guid]
 745                             cross_testbed_id = cross_testbed.testbed_id
 746                             testbed.defer_cross_connect(guid, connector_type_name, cross_guid,
 747                                     cross_testbed_guid, cross_testbed_id, cross_factory_id,
 748                                     cross_connector_type_name)
 749                             # save cross data for later
 750                             self._add_crossdata(testbed_guid, guid, cross_testbed_guid,
 751                                     cross_guid)
 752
 753     def _add_crossdata(self, testbed_guid, guid, cross_testbed_guid, cross_guid):
 754         if testbed_guid not in self._cross_data:
 755             self._cross_data[testbed_guid] = dict()
 756         if cross_testbed_guid not in self._cross_data[testbed_guid]:
 757             self._cross_data[testbed_guid][cross_testbed_guid] = set()
 758         self._cross_data[testbed_guid][cross_testbed_guid].add(cross_guid)
 759
 760     def _get_cross_data(self, testbed_guid):
 761         cross_data = dict()
 762         if not testbed_guid in self._cross_data:
 763             return cross_data
 764         for cross_testbed_guid, guid_list in \
 765                 self._cross_data[testbed_guid].iteritems():
 766             cross_data[cross_testbed_guid] = dict()
 767             cross_testbed = self._testbeds[cross_testbed_guid]
 768             for cross_guid in guid_list:
 769                 elem_cross_data = dict(
 770                     _guid = cross_guid,
 771                     _testbed_guid = cross_testbed_guid,
 772                     _testbed_id = cross_testbed.testbed_id,
 773                     _testbed_version = cross_testbed.testbed_version)
 774                 cross_data[cross_testbed_guid][cross_guid] = elem_cross_data
 775                 attribute_list = cross_testbed.get_attribute_list(cross_guid,
 776                         filter_flags = Attribute.DesignOnly)
 777                 for attr_name in attribute_list:
 778                     attr_value = cross_testbed.get(cross_guid, attr_name)
 779                     elem_cross_data[attr_name] = attr_value
 780         return cross_data
 781