src/nepi/core/execute.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from nepi.core.attributes import Attribute, AttributesMap
   5 from nepi.core.connector import ConnectorTypeBase
   6 from nepi.util import proxy, validation
   7 from nepi.util.constants import STATUS_FINISHED, TIME_NOW
   8 from nepi.util.parser._xml import XmlExperimentParser
   9 import sys
  10 import re
  11 import threading
  12 import ConfigParser
  13 import os
  14
  15 ATTRIBUTE_PATTERN_BASE = re.compile(r"\{#\[(?P<label>[-a-zA-Z0-9._]*)\](?P<expr>(?P<component>\.addr\[[0-9]+\]|\.route\[[0-9]+\]|\.trace\[[0-9]+\]|).\[(?P<attribute>[-a-zA-Z0-9._]*)\])#}")
  16 ATTRIBUTE_PATTERN_GUID_SUB = r"{#[%(guid)s]%(expr)s#}"
  17 COMPONENT_PATTERN = re.compile(r"(?P<kind>[a-z]*)\[(?P<index>.*)\]")
  18
  19 class ConnectorType(ConnectorTypeBase):
  20     def __init__(self, testbed_id, factory_id, name, max = -1, min = 0):
  21         super(ConnectorType, self).__init__(testbed_id, factory_id, name, max, min)
  22         # from_connections -- connections where the other connector is the "From"
  23         # to_connections -- connections where the other connector is the "To"
  24         # keys in the dictionary correspond to the
  25         # connector_type_id for possible connections. The value is a tuple:
  26         # (can_cross, connect)
  27         # can_cross: indicates if the connection is allowed accros different
  28         #    testbed instances
  29         # code: is the connection function to be invoked when the elements
  30         #    are connected
  31         self._from_connections = dict()
  32         self._to_connections = dict()
  33
  34     def add_from_connection(self, testbed_id, factory_id, name, can_cross,
  35             init_code, compl_code):
  36         type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  37         self._from_connections[type_id] = (can_cross, init_code, compl_code)
  38
  39     def add_to_connection(self, testbed_id, factory_id, name, can_cross,
  40             init_code, compl_code):
  41         type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  42         self._to_connections[type_id] = (can_cross, init_code, compl_code)
  43
  44     def can_connect(self, testbed_id, factory_id, name, count,
  45             must_cross = False):
  46         connector_type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  47         for lookup_type_id in self._type_resolution_order(connector_type_id):
  48             if lookup_type_id in self._from_connections:
  49                 (can_cross, init_code, compl_code) = self._from_connections[lookup_type_id]
  50             elif lookup_type_id in self._to_connections:
  51                 (can_cross, init_code, compl_code) = self._to_connections[lookup_type_id]
  52             else:
  53                 # keey trying
  54                 continue
  55             return not must_cross or can_cross
  56         else:
  57             return False
  58
  59     def _connect_to_code(self, testbed_id, factory_id, name):
  60         connector_type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  61         for lookup_type_id in self._type_resolution_order(connector_type_id):
  62             if lookup_type_id in self._to_connections:
  63                 (can_cross, init_code, compl_code) = self._to_connections[lookup_type_id]
  64                 return (init_code, compl_code)
  65         else:
  66             return (False, False)
  67
  68     def connect_to_init_code(self, testbed_id, factory_id, name):
  69         return self._connect_to_code(testbed_id, factory_id, name)[0]
  70
  71     def connect_to_compl_code(self, testbed_id, factory_id, name):
  72         return self._connect_to_code(testbed_id, factory_id, name)[1]
  73
  74 class Factory(AttributesMap):
  75     def __init__(self, factory_id, create_function, start_function,
  76             stop_function, status_function,
  77             configure_function, preconfigure_function,
  78             allow_addresses = False, has_addresses = False,
  79             allow_routes = False, has_routes = False):
  80         super(Factory, self).__init__()
  81         self._factory_id = factory_id
  82         self._allow_addresses = bool(allow_addresses)
  83         self._allow_routes = bool(allow_routes)
  84         self._has_addresses = bool(has_addresses) or self._allow_addresses
  85         self._has_routes = bool(has_routes) or self._allow_routes
  86         self._create_function = create_function
  87         self._start_function = start_function
  88         self._stop_function = stop_function
  89         self._status_function = status_function
  90         self._configure_function = configure_function
  91         self._preconfigure_function = preconfigure_function
  92         self._connector_types = dict()
  93         self._traces = list()
  94         self._box_attributes = AttributesMap()
  95
  96     @property
  97     def factory_id(self):
  98         return self._factory_id
  99
 100     @property
 101     def allow_addresses(self):
 102         return self._allow_addresses
 103
 104     @property
 105     def allow_routes(self):
 106         return self._allow_routes
 107
 108     @property
 109     def has_addresses(self):
 110         return self._has_addresses
 111
 112     @property
 113     def has_routes(self):
 114         return self._has_routes
 115
 116     @property
 117     def box_attributes(self):
 118         return self._box_attributes
 119
 120     @property
 121     def create_function(self):
 122         return self._create_function
 123
 124     @property
 125     def start_function(self):
 126         return self._start_function
 127
 128     @property
 129     def stop_function(self):
 130         return self._stop_function
 131
 132     @property
 133     def status_function(self):
 134         return self._status_function
 135
 136     @property
 137     def configure_function(self):
 138         return self._configure_function
 139
 140     @property
 141     def preconfigure_function(self):
 142         return self._preconfigure_function
 143
 144     @property
 145     def traces(self):
 146         return self._traces
 147
 148     def connector_type(self, name):
 149         return self._connector_types[name]
 150
 151     def add_connector_type(self, connector_type):
 152         self._connector_types[connector_type.name] = connector_type
 153
 154     def add_trace(self, trace_id):
 155         self._traces.append(trace_id)
 156
 157     def add_box_attribute(self, name, help, type, value = None, range = None,
 158         allowed = None, flags = Attribute.NoFlags, validation_function = None):
 159         self._box_attributes.add_attribute(name, help, type, value, range,
 160                 allowed, flags, validation_function)
 161
 162 class TestbedController(object):
 163     def __init__(self, testbed_id, testbed_version):
 164         self._testbed_id = testbed_id
 165         self._testbed_version = testbed_version
 166
 167     @property
 168     def testbed_id(self):
 169         return self._testbed_id
 170
 171     @property
 172     def testbed_version(self):
 173         return self._testbed_version
 174
 175     @property
 176     def guids(self):
 177         raise NotImplementedError
 178
 179     def defer_configure(self, name, value):
 180         """Instructs setting a configuartion attribute for the testbed instance"""
 181         raise NotImplementedError
 182
 183     def defer_create(self, guid, factory_id):
 184         """Instructs creation of element """
 185         raise NotImplementedError
 186
 187     def defer_create_set(self, guid, name, value):
 188         """Instructs setting an initial attribute on an element"""
 189         raise NotImplementedError
 190
 191     def defer_factory_set(self, guid, name, value):
 192         """Instructs setting an attribute on a factory"""
 193         raise NotImplementedError
 194
 195     def defer_connect(self, guid1, connector_type_name1, guid2,
 196             connector_type_name2):
 197         """Instructs creation of a connection between the given connectors"""
 198         raise NotImplementedError
 199
 200     def defer_cross_connect(self, guid, connector_type_name, cross_guid,
 201             cross_testbed_id, cross_factory_id, cross_connector_type_name):
 202         """
 203         Instructs creation of a connection between the given connectors
 204         of different testbed instances
 205         """
 206         raise NotImplementedError
 207
 208     def defer_add_trace(self, guid, trace_id):
 209         """Instructs the addition of a trace"""
 210         raise NotImplementedError
 211
 212     def defer_add_address(self, guid, address, netprefix, broadcast):
 213         """Instructs the addition of an address"""
 214         raise NotImplementedError
 215
 216     def defer_add_route(self, guid, destination, netprefix, nexthop):
 217         """Instructs the addition of a route"""
 218         raise NotImplementedError
 219
 220     def do_setup(self):
 221         """After do_setup the testbed initial configuration is done"""
 222         raise NotImplementedError
 223
 224     def do_create(self):
 225         """
 226         After do_create all instructed elements are created and
 227         attributes setted
 228         """
 229         raise NotImplementedError
 230
 231     def do_connect_init(self):
 232         """
 233         After do_connect_init all internal connections between testbed elements
 234         are initiated
 235         """
 236         raise NotImplementedError
 237
 238     def do_connect_compl(self):
 239         """
 240         After do_connect all internal connections between testbed elements
 241         are completed
 242         """
 243         raise NotImplementedError
 244
 245     def do_configure(self):
 246         """After do_configure elements are configured"""
 247         raise NotImplementedError
 248
 249     def do_cross_connect_init(self, cross_data):
 250         """
 251         After do_cross_connect_init initiation of all external connections
 252         between different testbed elements is performed
 253         """
 254         raise NotImplementedError
 255
 256     def do_cross_connect_compl(self, cross_data):
 257         """
 258         After do_cross_connect_compl completion of all external connections
 259         between different testbed elements is performed
 260         """
 261         raise NotImplementedError
 262
 263     def start(self):
 264         raise NotImplementedError
 265
 266     def stop(self):
 267         raise NotImplementedError
 268
 269     def set(self, guid, name, value, time = TIME_NOW):
 270         raise NotImplementedError
 271
 272     def get(self, guid, name, time = TIME_NOW):
 273         raise NotImplementedError
 274
 275     def get_route(self, guid, index, attribute):
 276         """
 277         Params:
 278
 279             guid: guid of box to query
 280             index: number of routing entry to fetch
 281             attribute: one of Destination, NextHop, NetPrefix
 282         """
 283         raise NotImplementedError
 284
 285     def get_address(self, guid, index, attribute='Address'):
 286         """
 287         Params:
 288
 289             guid: guid of box to query
 290             index: number of inteface to select
 291             attribute: one of Address, NetPrefix, Broadcast
 292         """
 293         raise NotImplementedError
 294
 295     def get_attribute_list(self, guid):
 296         raise NotImplementedError
 297
 298     def action(self, time, guid, action):
 299         raise NotImplementedError
 300
 301     def status(self, guid):
 302         raise NotImplementedError
 303
 304     def trace(self, guid, trace_id, attribute='value'):
 305         raise NotImplementedError
 306
 307     def shutdown(self):
 308         raise NotImplementedError
 309
 310 class ExperimentController(object):
 311     def __init__(self, experiment_xml, root_dir):
 312         self._experiment_xml = experiment_xml
 313         self._testbeds = dict()
 314         self._deployment_config = dict()
 315         self._netrefs = dict()
 316         self._cross_data = dict()
 317         self._root_dir = root_dir
 318         self._netreffed_testbeds = set()
 319
 320         self.persist_experiment_xml()
 321
 322     @property
 323     def experiment_xml(self):
 324         return self._experiment_xml
 325
 326     def persist_experiment_xml(self):
 327         xml_path = os.path.join(self._root_dir, "experiment.xml")
 328         f = open(xml_path, "w")
 329         f.write(self._experiment_xml)
 330         f.close()
 331
 332     def trace(self, testbed_guid, guid, trace_id, attribute='value'):
 333         return self._testbeds[testbed_guid].trace(guid, trace_id, attribute)
 334
 335     @staticmethod
 336     def _parallel(callables):
 337         threads = [ threading.Thread(target=callable) for callable in callables ]
 338         for thread in threads:
 339             thread.start()
 340         for thread in threads:
 341             thread.join()
 342
 343     def start(self):
 344         self._init_testbed_controllers()
 345
 346         # persist testbed connection data, for potential recovery
 347         self._persist_testbed_proxies()
 348
 349         def steps_to_configure(self, allowed_guids):
 350             # perform setup in parallel for all test beds,
 351             # wait for all threads to finish
 352             self._parallel([testbed.do_setup
 353                             for guid,testbed in self._testbeds.iteritems()
 354                             if guid in allowed_guids])
 355
 356             # perform create-connect in parallel, wait
 357             # (internal connections only)
 358             self._parallel([testbed.do_create
 359                             for guid,testbed in self._testbeds.iteritems()
 360                             if guid in allowed_guids])
 361
 362             self._parallel([testbed.do_connect_init
 363                             for guid,testbed in self._testbeds.iteritems()
 364                             if guid in allowed_guids])
 365
 366             self._parallel([testbed.do_connect_compl
 367                             for guid,testbed in self._testbeds.iteritems()
 368                             if guid in allowed_guids])
 369
 370             self._parallel([testbed.do_preconfigure
 371                             for guid,testbed in self._testbeds.iteritems()
 372                             if guid in allowed_guids])
 373
 374         steps_to_configure(self, self._testbeds)
 375
 376         if self._netreffed_testbeds:
 377             # initally resolve netrefs
 378             self.do_netrefs(fail_if_undefined=False)
 379
 380             # rinse and repeat, for netreffed testbeds
 381             netreffed_testbeds = set(self._netreffed_testbeds)
 382
 383             self._init_testbed_controllers()
 384
 385             # persist testbed connection data, for potential recovery
 386             self._persist_testbed_proxies()
 387
 388             # configure dependant testbeds
 389             steps_to_configure(self, netreffed_testbeds)
 390
 391         # final netref step, fail if anything's left unresolved
 392         self.do_netrefs(fail_if_undefined=True)
 393
 394         # perform do_configure in parallel for al testbeds
 395         # (it's internal configuration for each)
 396         self._parallel([testbed.do_configure
 397                         for testbed in self._testbeds.itervalues()])
 398
 399         # cross-connect (cannot be done in parallel)
 400         for guid, testbed in self._testbeds.iteritems():
 401             cross_data = self._get_cross_data(guid)
 402             testbed.do_cross_connect_init(cross_data)
 403         for guid, testbed in self._testbeds.iteritems():
 404             cross_data = self._get_cross_data(guid)
 405             testbed.do_cross_connect_compl(cross_data)
 406
 407         # start experiment (parallel start on all testbeds)
 408         self._parallel([testbed.start
 409                         for testbed in self._testbeds.itervalues()])
 410
 411     def _persist_testbed_proxies(self):
 412         TRANSIENT = ('Recover',)
 413
 414         # persist access configuration for all testbeds, so that
 415         # recovery mode can reconnect to them if it becomes necessary
 416         conf = ConfigParser.RawConfigParser()
 417         for testbed_guid, testbed_config in self._deployment_config.iteritems():
 418             testbed_guid = str(testbed_guid)
 419             conf.add_section(testbed_guid)
 420             for attr in testbed_config.attributes_list:
 421                 if attr not in TRANSIENT:
 422                     conf.set(testbed_guid, attr,
 423                         testbed_config.get_attribute_value(attr))
 424
 425         f = open(os.path.join(self._root_dir, 'deployment_config.ini'), 'w')
 426         conf.write(f)
 427         f.close()
 428
 429     def _load_testbed_proxies(self):
 430         TYPEMAP = {
 431             STRING : 'get',
 432             INTEGER : 'getint',
 433             FLOAT : 'getfloat',
 434             BOOLEAN : 'getboolean',
 435         }
 436
 437         conf = ConfigParser.RawConfigParser()
 438         conf.read(os.path.join(self._root_dir, 'deployment_config.ini'))
 439         for testbed_guid in conf.sections():
 440             testbed_config = proxy.AccessConfiguration()
 441             for attr in conf.options(testbed_guid):
 442                 testbed_config.set_attribute_value(attr,
 443                     conf.get(testbed_guid, attr) )
 444
 445             testbed_guid = str(testbed_guid)
 446             conf.add_section(testbed_guid)
 447             for attr in testbed_config.attributes_list:
 448                 if attr not in TRANSIENT:
 449                     getter = getattr(conf, TYPEMAP.get(
 450                         testbed_config.get_attribute_type(attr),
 451                         'get') )
 452                     testbed_config.set_attribute_value(
 453                         testbed_guid, attr, getter(attr))
 454
 455     def _unpersist_testbed_proxies(self):
 456         try:
 457             os.remove(os.path.join(self._root_dir, 'deployment_config.ini'))
 458         except:
 459             # Just print exceptions, this is just cleanup
 460             import traceback
 461             traceback.print_exc(file=sys.stderr)
 462
 463     def stop(self):
 464        for testbed in self._testbeds.values():
 465            testbed.stop()
 466        self._unpersist_testbed_proxies()
 467
 468     def recover(self):
 469         # reload perviously persisted testbed access configurations
 470         self._load_testbed_proxies()
 471
 472         # recreate testbed proxies by reconnecting only
 473         self._init_testbed_controllers(recover = True)
 474
 475         # another time, for netrefs
 476         self._init_testbed_controllers(recover = True)
 477
 478     def is_finished(self, guid):
 479         for testbed in self._testbeds.values():
 480             for guid_ in testbed.guids:
 481                 if guid_ == guid:
 482                     return testbed.status(guid) == STATUS_FINISHED
 483         raise RuntimeError("No element exists with guid %d" % guid)
 484
 485     def set(self, testbed_guid, guid, name, value, time = TIME_NOW):
 486         testbed = self._testbeds[testbed_guid]
 487         testbed.set(guid, name, value, time)
 488
 489     def get(self, testbed_guid, guid, name, time = TIME_NOW):
 490         testbed = self._testbeds[testbed_guid]
 491         return testbed.get(guid, name, time)
 492
 493     def shutdown(self):
 494        for testbed in self._testbeds.values():
 495            testbed.shutdown()
 496
 497     @staticmethod
 498     def _netref_component_split(component):
 499         match = COMPONENT_PATTERN.match(component)
 500         if match:
 501             return match.group("kind"), match.group("index")
 502         else:
 503             return component, None
 504
 505     def do_netrefs(self, fail_if_undefined = False):
 506         COMPONENT_GETTERS = {
 507             'addr':
 508                 lambda testbed, guid, index, name:
 509                     testbed.get_address(guid, index, name),
 510             'route' :
 511                 lambda testbed, guid, index, name:
 512                     testbed.get_route(guid, index, name),
 513             'trace' :
 514                 lambda testbed, guid, index, name:
 515                     testbed.trace(guid, index, name),
 516             '' :
 517                 lambda testbed, guid, index, name:
 518                     testbed.get(guid, name),
 519         }
 520
 521         for (testbed_guid, guid), attrs in self._netrefs.iteritems():
 522             testbed = self._testbeds[testbed_guid]
 523             for name in attrs:
 524                 value = testbed.get(guid, name)
 525                 if isinstance(value, basestring):
 526                     match = ATTRIBUTE_PATTERN_BASE.search(value)
 527                     if match:
 528                         label = match.group("label")
 529                         if label.startswith('GUID-'):
 530                             ref_guid = int(label[5:])
 531                             if ref_guid:
 532                                 expr = match.group("expr")
 533                                 component = match.group("component")[1:] # skip the dot
 534                                 attribute = match.group("attribute")
 535
 536                                 # split compound components into component kind and index
 537                                 # eg: 'addr[0]' -> ('addr', '0')
 538                                 component, component_index = self._netref_component_split(component)
 539
 540                                 # find object and resolve expression
 541                                 for ref_testbed in self._testbeds.itervalues():
 542                                     if component not in COMPONENT_GETTERS:
 543                                         raise ValueError, "Malformed netref: %r - unknown component" % (expr,)
 544                                     else:
 545                                         ref_value = COMPONENT_GETTERS[component](
 546                                             ref_testbed, ref_guid, component_index, attribute)
 547                                         if ref_value:
 548                                             testbed.set(guid, name,
 549                                                     value.replace(match.group(), ref_value))
 550                                             break
 551                                 else:
 552                                     # couldn't find value
 553                                     if fail_if_undefined:
 554                                         raise ValueError, "Unresolvable GUID: %r, in netref: %r" % (ref_guid, expr)
 555
 556     def _init_testbed_controllers(self, recover = False):
 557         parser = XmlExperimentParser()
 558         data = parser.from_xml_to_data(self._experiment_xml)
 559         blacklist_testbeds = set(self._testbeds)
 560         element_guids = list()
 561         label_guids = dict()
 562         data_guids = data.guids
 563
 564         # create testbed controllers
 565         for guid in data_guids:
 566             if data.is_testbed_data(guid):
 567                 if guid not in self._testbeds:
 568                     self._create_testbed_controller(guid, data, element_guids,
 569                             recover)
 570             else:
 571                 (testbed_guid, factory_id) = data.get_box_data(guid)
 572                 if testbed_guid not in blacklist_testbeds:
 573                     element_guids.append(guid)
 574                     label = data.get_attribute_data(guid, "label")
 575                     if label is not None:
 576                         if label in label_guids:
 577                             raise RuntimeError, "Label %r is not unique" % (label,)
 578                         label_guids[label] = guid
 579
 580         # replace references to elements labels for its guid
 581         self._resolve_labels(data, data_guids, label_guids)
 582
 583         # program testbed controllers
 584         if not recover:
 585             self._program_testbed_controllers(element_guids, data)
 586
 587     def _resolve_labels(self, data, data_guids, label_guids):
 588         netrefs = self._netrefs
 589         for guid in data_guids:
 590             if not data.is_testbed_data(guid):
 591                 for name, value in data.get_attribute_data(guid):
 592                     if isinstance(value, basestring):
 593                         match = ATTRIBUTE_PATTERN_BASE.search(value)
 594                         if match:
 595                             label = match.group("label")
 596                             if not label.startswith('GUID-'):
 597                                 ref_guid = label_guids.get(label)
 598                                 if ref_guid is not None:
 599                                     value = ATTRIBUTE_PATTERN_BASE.sub(
 600                                         ATTRIBUTE_PATTERN_GUID_SUB % dict(
 601                                             guid = 'GUID-%d' % (ref_guid,),
 602                                             expr = match.group("expr"),
 603                                             label = label),
 604                                         value)
 605                                     data.set_attribute_data(guid, name, value)
 606
 607                                     # memorize which guid-attribute pairs require
 608                                     # postprocessing, to avoid excessive controller-testbed
 609                                     # communication at configuration time
 610                                     # (which could require high-latency network I/O)
 611                                     (testbed_guid, factory_id) = data.get_box_data(guid)
 612                                     netrefs.setdefault((testbed_guid, guid), set()).add(name)
 613
 614     def _create_testbed_controller(self, guid, data, element_guids, recover):
 615         (testbed_id, testbed_version) = data.get_testbed_data(guid)
 616         deployment_config = self._deployment_config.get(guid)
 617
 618         if deployment_config is None:
 619             # need to create one
 620             deployment_config = self._deployment_config[guid] = proxy.AccessConfiguration()
 621
 622             for (name, value) in data.get_attribute_data(guid):
 623                 if value is not None and deployment_config.has_attribute(name):
 624                     # if any deployment config attribute has a netref, we can't
 625                     # create this controller yet
 626                     if isinstance(value, basestring) and ATTRIBUTE_PATTERN_BASE.search(value):
 627                         # remember to re-issue this one
 628                         self._netreffed_testbeds.add(guid)
 629                         return
 630
 631                     # copy deployment config attribute
 632                     deployment_config.set_attribute_value(name, value)
 633
 634         if deployment_config is not None:
 635             # force recovery mode
 636             deployment_config.set_attribute_value("recover",recover)
 637
 638         testbed = proxy.create_testbed_controller(testbed_id,
 639                 testbed_version, deployment_config)
 640         for (name, value) in data.get_attribute_data(guid):
 641             testbed.defer_configure(name, value)
 642         self._testbeds[guid] = testbed
 643         if guid in self._netreffed_testbeds:
 644             self._netreffed_testbeds.remove(guid)
 645
 646     def _program_testbed_controllers(self, element_guids, data):
 647         for guid in element_guids:
 648             (testbed_guid, factory_id) = data.get_box_data(guid)
 649             testbed = self._testbeds[testbed_guid]
 650             testbed.defer_create(guid, factory_id)
 651             for (name, value) in data.get_attribute_data(guid):
 652                 testbed.defer_create_set(guid, name, value)
 653
 654         for guid in element_guids:
 655             (testbed_guid, factory_id) = data.get_box_data(guid)
 656             testbed = self._testbeds[testbed_guid]
 657             for (connector_type_name, cross_guid, cross_connector_type_name) \
 658                     in data.get_connection_data(guid):
 659                 (testbed_guid, factory_id) = data.get_box_data(guid)
 660                 (cross_testbed_guid, cross_factory_id) = data.get_box_data(
 661                         cross_guid)
 662                 if testbed_guid == cross_testbed_guid:
 663                     testbed.defer_connect(guid, connector_type_name,
 664                             cross_guid, cross_connector_type_name)
 665                 else:
 666                     cross_testbed = self._testbeds[cross_testbed_guid]
 667                     cross_testbed_id = cross_testbed.testbed_id
 668                     testbed.defer_cross_connect(guid, connector_type_name, cross_guid,
 669                             cross_testbed_guid, cross_testbed_id, cross_factory_id,
 670                             cross_connector_type_name)
 671                     # save cross data for later
 672                     self._add_crossdata(testbed_guid, guid, cross_testbed_guid,
 673                             cross_guid)
 674             for trace_id in data.get_trace_data(guid):
 675                 testbed.defer_add_trace(guid, trace_id)
 676             for (autoconf, address, netprefix, broadcast) in \
 677                     data.get_address_data(guid):
 678                 if address != None:
 679                     testbed.defer_add_address(guid, address, netprefix,
 680                             broadcast)
 681             for (destination, netprefix, nexthop) in data.get_route_data(guid):
 682                 testbed.defer_add_route(guid, destination, netprefix, nexthop)
 683
 684     def _add_crossdata(self, testbed_guid, guid, cross_testbed_guid, cross_guid):
 685         if testbed_guid not in self._cross_data:
 686             self._cross_data[testbed_guid] = dict()
 687         if cross_testbed_guid not in self._cross_data[testbed_guid]:
 688             self._cross_data[testbed_guid][cross_testbed_guid] = set()
 689         self._cross_data[testbed_guid][cross_testbed_guid].add(cross_guid)
 690
 691     def _get_cross_data(self, testbed_guid):
 692         cross_data = dict()
 693         if not testbed_guid in self._cross_data:
 694             return cross_data
 695         for cross_testbed_guid, guid_list in \
 696                 self._cross_data[testbed_guid].iteritems():
 697             cross_data[cross_testbed_guid] = dict()
 698             cross_testbed = self._testbeds[cross_testbed_guid]
 699             for cross_guid in guid_list:
 700                 elem_cross_data = dict()
 701                 cross_data[cross_testbed_guid][cross_guid] = elem_cross_data
 702                 attributes_list = cross_testbed.get_attribute_list(cross_guid)
 703                 for attr_name in attributes_list:
 704                     attr_value = cross_testbed.get(cross_guid, attr_name)
 705                     elem_cross_data[attr_name] = attr_value
 706         return cross_data
 707