src/nepi/core/execute.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from nepi.core.attributes import Attribute, AttributesMap
   5 from nepi.core.connector import ConnectorTypeBase
   6 from nepi.util import validation
   7 from nepi.util.constants import STATUS_FINISHED, TIME_NOW
   8 from nepi.util.parser._xml import XmlExperimentParser
   9 import sys
  10 import re
  11 import threading
  12 import ConfigParser
  13 import os
  14 import collections
  15 import functools
  16
  17 ATTRIBUTE_PATTERN_BASE = re.compile(r"\{#\[(?P<label>[-a-zA-Z0-9._]*)\](?P<expr>(?P<component>\.addr\[[0-9]+\]|\.route\[[0-9]+\]|\.trace\[[0-9]+\])?.\[(?P<attribute>[-a-zA-Z0-9._]*)\])#}")
  18 ATTRIBUTE_PATTERN_GUID_SUB = r"{#[%(guid)s]%(expr)s#}"
  19 COMPONENT_PATTERN = re.compile(r"(?P<kind>[a-z]*)\[(?P<index>.*)\]")
  20
  21 class ConnectorType(ConnectorTypeBase):
  22     def __init__(self, testbed_id, factory_id, name, max = -1, min = 0):
  23         super(ConnectorType, self).__init__(testbed_id, factory_id, name, max, min)
  24         # from_connections -- connections where the other connector is the "From"
  25         # to_connections -- connections where the other connector is the "To"
  26         # keys in the dictionary correspond to the
  27         # connector_type_id for possible connections. The value is a tuple:
  28         # (can_cross, connect)
  29         # can_cross: indicates if the connection is allowed accros different
  30         #    testbed instances
  31         # code: is the connection function to be invoked when the elements
  32         #    are connected
  33         self._from_connections = dict()
  34         self._to_connections = dict()
  35
  36     def add_from_connection(self, testbed_id, factory_id, name, can_cross,
  37             init_code, compl_code):
  38         type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  39         self._from_connections[type_id] = (can_cross, init_code, compl_code)
  40
  41     def add_to_connection(self, testbed_id, factory_id, name, can_cross,
  42             init_code, compl_code):
  43         type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  44         self._to_connections[type_id] = (can_cross, init_code, compl_code)
  45
  46     def can_connect(self, testbed_id, factory_id, name, count,
  47             must_cross):
  48         connector_type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  49         for lookup_type_id in self._type_resolution_order(connector_type_id):
  50             if lookup_type_id in self._from_connections:
  51                 (can_cross, init_code, compl_code) = self._from_connections[lookup_type_id]
  52             elif lookup_type_id in self._to_connections:
  53                 (can_cross, init_code, compl_code) = self._to_connections[lookup_type_id]
  54             else:
  55                 # keep trying
  56                 continue
  57             return not must_cross or can_cross
  58         else:
  59             return False
  60
  61     def _connect_to_code(self, testbed_id, factory_id, name,
  62             must_cross):
  63         connector_type_id = self.make_connector_type_id(testbed_id, factory_id, name)
  64         for lookup_type_id in self._type_resolution_order(connector_type_id):
  65             if lookup_type_id in self._to_connections:
  66                 (can_cross, init_code, compl_code) = self._to_connections[lookup_type_id]
  67                 if not must_cross or can_cross:
  68                     return (init_code, compl_code)
  69         else:
  70             return (False, False)
  71
  72     def connect_to_init_code(self, testbed_id, factory_id, name, must_cross):
  73         return self._connect_to_code(testbed_id, factory_id, name, must_cross)[0]
  74
  75     def connect_to_compl_code(self, testbed_id, factory_id, name, must_cross):
  76         return self._connect_to_code(testbed_id, factory_id, name, must_cross)[1]
  77
  78 class Factory(AttributesMap):
  79     def __init__(self, factory_id, create_function, start_function,
  80             stop_function, status_function,
  81             configure_function, preconfigure_function,
  82             prestart_function,
  83             allow_addresses = False, has_addresses = False,
  84             allow_routes = False, has_routes = False):
  85         super(Factory, self).__init__()
  86         self._factory_id = factory_id
  87         self._allow_addresses = bool(allow_addresses)
  88         self._allow_routes = bool(allow_routes)
  89         self._has_addresses = bool(has_addresses) or self._allow_addresses
  90         self._has_routes = bool(has_routes) or self._allow_routes
  91         self._create_function = create_function
  92         self._start_function = start_function
  93         self._stop_function = stop_function
  94         self._status_function = status_function
  95         self._configure_function = configure_function
  96         self._preconfigure_function = preconfigure_function
  97         self._prestart_function = prestart_function
  98         self._connector_types = dict()
  99         self._traces = list()
 100         self._tags = list()
 101         self._box_attributes = AttributesMap()
 102
 103     @property
 104     def factory_id(self):
 105         return self._factory_id
 106
 107     @property
 108     def allow_addresses(self):
 109         return self._allow_addresses
 110
 111     @property
 112     def allow_routes(self):
 113         return self._allow_routes
 114
 115     @property
 116     def has_addresses(self):
 117         return self._has_addresses
 118
 119     @property
 120     def has_routes(self):
 121         return self._has_routes
 122
 123     @property
 124     def box_attributes(self):
 125         return self._box_attributes
 126
 127     @property
 128     def create_function(self):
 129         return self._create_function
 130
 131     @property
 132     def prestart_function(self):
 133         return self._prestart_function
 134
 135     @property
 136     def start_function(self):
 137         return self._start_function
 138
 139     @property
 140     def stop_function(self):
 141         return self._stop_function
 142
 143     @property
 144     def status_function(self):
 145         return self._status_function
 146
 147     @property
 148     def configure_function(self):
 149         return self._configure_function
 150
 151     @property
 152     def preconfigure_function(self):
 153         return self._preconfigure_function
 154
 155     @property
 156     def traces(self):
 157         return self._traces
 158
 159     @property
 160     def tags(self):
 161         return self._tags
 162
 163     def connector_type(self, name):
 164         return self._connector_types[name]
 165
 166     def add_connector_type(self, connector_type):
 167         self._connector_types[connector_type.name] = connector_type
 168
 169     def add_trace(self, trace_id):
 170         self._traces.append(trace_id)
 171
 172     def add_tag(self, tag_id):
 173         self._tags.append(tag_id)
 174
 175     def add_box_attribute(self, name, help, type, value = None, range = None,
 176         allowed = None, flags = Attribute.NoFlags, validation_function = None,
 177         category = None):
 178         self._box_attributes.add_attribute(name, help, type, value, range,
 179                 allowed, flags, validation_function, category)
 180
 181 class TestbedController(object):
 182     def __init__(self, testbed_id, testbed_version):
 183         self._testbed_id = testbed_id
 184         self._testbed_version = testbed_version
 185
 186     @property
 187     def testbed_id(self):
 188         return self._testbed_id
 189
 190     @property
 191     def testbed_version(self):
 192         return self._testbed_version
 193
 194     @property
 195     def guids(self):
 196         raise NotImplementedError
 197
 198     def defer_configure(self, name, value):
 199         """Instructs setting a configuartion attribute for the testbed instance"""
 200         raise NotImplementedError
 201
 202     def defer_create(self, guid, factory_id):
 203         """Instructs creation of element """
 204         raise NotImplementedError
 205
 206     def defer_create_set(self, guid, name, value):
 207         """Instructs setting an initial attribute on an element"""
 208         raise NotImplementedError
 209
 210     def defer_factory_set(self, guid, name, value):
 211         """Instructs setting an attribute on a factory"""
 212         raise NotImplementedError
 213
 214     def defer_connect(self, guid1, connector_type_name1, guid2,
 215             connector_type_name2):
 216         """Instructs creation of a connection between the given connectors"""
 217         raise NotImplementedError
 218
 219     def defer_cross_connect(self,
 220             guid, connector_type_name,
 221             cross_guid, cross_testbed_guid,
 222             cross_testbed_id, cross_factory_id,
 223             cross_connector_type_name):
 224         """
 225         Instructs creation of a connection between the given connectors
 226         of different testbed instances
 227         """
 228         raise NotImplementedError
 229
 230     def defer_add_trace(self, guid, trace_id):
 231         """Instructs the addition of a trace"""
 232         raise NotImplementedError
 233
 234     def defer_add_address(self, guid, address, netprefix, broadcast):
 235         """Instructs the addition of an address"""
 236         raise NotImplementedError
 237
 238     def defer_add_route(self, guid, destination, netprefix, nexthop):
 239         """Instructs the addition of a route"""
 240         raise NotImplementedError
 241
 242     def do_setup(self):
 243         """After do_setup the testbed initial configuration is done"""
 244         raise NotImplementedError
 245
 246     def do_create(self):
 247         """
 248         After do_create all instructed elements are created and
 249         attributes setted
 250         """
 251         raise NotImplementedError
 252
 253     def do_connect_init(self):
 254         """
 255         After do_connect_init all internal connections between testbed elements
 256         are initiated
 257         """
 258         raise NotImplementedError
 259
 260     def do_connect_compl(self):
 261         """
 262         After do_connect all internal connections between testbed elements
 263         are completed
 264         """
 265         raise NotImplementedError
 266
 267     def do_preconfigure(self):
 268         """
 269         Done just before resolving netrefs, after connection, before cross connections,
 270         useful for early stages of configuration, for setting up stuff that might be
 271         required for netref resolution.
 272         """
 273         raise NotImplementedError
 274
 275     def do_configure(self):
 276         """After do_configure elements are configured"""
 277         raise NotImplementedError
 278
 279     def do_prestart(self):
 280         """Before do_start elements are prestart-configured"""
 281         raise NotImplementedError
 282
 283     def do_cross_connect_init(self, cross_data):
 284         """
 285         After do_cross_connect_init initiation of all external connections
 286         between different testbed elements is performed
 287         """
 288         raise NotImplementedError
 289
 290     def do_cross_connect_compl(self, cross_data):
 291         """
 292         After do_cross_connect_compl completion of all external connections
 293         between different testbed elements is performed
 294         """
 295         raise NotImplementedError
 296
 297     def start(self):
 298         raise NotImplementedError
 299
 300     def stop(self):
 301         raise NotImplementedError
 302
 303     def set(self, guid, name, value, time = TIME_NOW):
 304         raise NotImplementedError
 305
 306     def get(self, guid, name, time = TIME_NOW):
 307         raise NotImplementedError
 308
 309     def get_route(self, guid, index, attribute):
 310         """
 311         Params:
 312
 313             guid: guid of box to query
 314             index: number of routing entry to fetch
 315             attribute: one of Destination, NextHop, NetPrefix
 316         """
 317         raise NotImplementedError
 318
 319     def get_address(self, guid, index, attribute='Address'):
 320         """
 321         Params:
 322
 323             guid: guid of box to query
 324             index: number of inteface to select
 325             attribute: one of Address, NetPrefix, Broadcast
 326         """
 327         raise NotImplementedError
 328
 329     def get_attribute_list(self, guid):
 330         raise NotImplementedError
 331
 332     def get_factory_id(self, guid):
 333         raise NotImplementedError
 334
 335     def action(self, time, guid, action):
 336         raise NotImplementedError
 337
 338     def status(self, guid):
 339         raise NotImplementedError
 340
 341     def trace(self, guid, trace_id, attribute='value'):
 342         raise NotImplementedError
 343
 344     def traces_info(self):
 345         """ dictionary of dictionaries:
 346             traces_info = dict({
 347                 guid = dict({
 348                     trace_id = dict({
 349                             host = host,
 350                             filepath = filepath,
 351                             filesize = size in bytes,
 352                         })
 353                 })
 354             })"""
 355         raise NotImplementedError
 356
 357     def shutdown(self):
 358         raise NotImplementedError
 359
 360 class ExperimentController(object):
 361     def __init__(self, experiment_xml, root_dir):
 362         self._experiment_xml = experiment_xml
 363         self._testbeds = dict()
 364         self._deployment_config = dict()
 365         self._netrefs = collections.defaultdict(set)
 366         self._testbed_netrefs = collections.defaultdict(set)
 367         self._cross_data = dict()
 368         self._root_dir = root_dir
 369         self._netreffed_testbeds = set()
 370         self._guids_in_testbed_cache = dict()
 371
 372         self.persist_experiment_xml()
 373
 374     @property
 375     def experiment_xml(self):
 376         return self._experiment_xml
 377
 378     @property
 379     def guids(self):
 380         guids = list()
 381         for testbed_guid in self._testbeds.keys():
 382             _guids = self._guids_in_testbed(testbed_guid)
 383             if _guids:
 384                 guids.extend(_guids)
 385         return guids
 386
 387     def persist_experiment_xml(self):
 388         xml_path = os.path.join(self._root_dir, "experiment.xml")
 389         f = open(xml_path, "w")
 390         f.write(self._experiment_xml)
 391         f.close()
 392
 393     def trace(self, guid, trace_id, attribute='value'):
 394         testbed = self._testbed_for_guid(guid)
 395         if testbed != None:
 396             return testbed.trace(guid, trace_id, attribute)
 397         raise RuntimeError("No element exists with guid %d" % guid)
 398
 399     def traces_info(self):
 400         traces_info = dict()
 401         for guid, testbed in self._testbeds.iteritems():
 402             traces_info[guid] = testbed.traces_info()
 403         return traces_info
 404
 405     @staticmethod
 406     def _parallel(callables):
 407         excs = []
 408         def wrap(callable):
 409             @functools.wraps(callable)
 410             def wrapped(*p, **kw):
 411                 try:
 412                     callable(*p, **kw)
 413                 except:
 414                     import traceback
 415                     traceback.print_exc(file=sys.stderr)
 416                     excs.append(sys.exc_info())
 417             return wrapped
 418         threads = [ threading.Thread(target=wrap(callable)) for callable in callables ]
 419         for thread in threads:
 420             thread.start()
 421         for thread in threads:
 422             thread.join()
 423         for exc in excs:
 424             eTyp, eVal, eLoc = exc
 425             raise eTyp, eVal, eLoc
 426
 427     def start(self):
 428         parser = XmlExperimentParser()
 429         data = parser.from_xml_to_data(self._experiment_xml)
 430
 431         self._init_testbed_controllers(data)
 432
 433         # persist testbed connection data, for potential recovery
 434         self._persist_testbed_proxies()
 435
 436         def steps_to_configure(self, allowed_guids):
 437             # perform setup in parallel for all test beds,
 438             # wait for all threads to finish
 439             self._parallel([testbed.do_setup
 440                             for guid,testbed in self._testbeds.iteritems()
 441                             if guid in allowed_guids])
 442
 443             # perform create-connect in parallel, wait
 444             # (internal connections only)
 445             self._parallel([testbed.do_create
 446                             for guid,testbed in self._testbeds.iteritems()
 447                             if guid in allowed_guids])
 448
 449             self._parallel([testbed.do_connect_init
 450                             for guid,testbed in self._testbeds.iteritems()
 451                             if guid in allowed_guids])
 452
 453             self._parallel([testbed.do_connect_compl
 454                             for guid,testbed in self._testbeds.iteritems()
 455                             if guid in allowed_guids])
 456
 457             self._parallel([testbed.do_preconfigure
 458                             for guid,testbed in self._testbeds.iteritems()
 459                             if guid in allowed_guids])
 460             self._clear_caches()
 461
 462         steps_to_configure(self, self._testbeds)
 463
 464         if self._netreffed_testbeds:
 465             # initally resolve netrefs
 466             self.do_netrefs(data, fail_if_undefined=False)
 467
 468             # rinse and repeat, for netreffed testbeds
 469             netreffed_testbeds = set(self._netreffed_testbeds)
 470
 471             self._init_testbed_controllers(data)
 472
 473             # persist testbed connection data, for potential recovery
 474             self._persist_testbed_proxies()
 475
 476             # configure dependant testbeds
 477             steps_to_configure(self, netreffed_testbeds)
 478
 479         # final netref step, fail if anything's left unresolved
 480         self.do_netrefs(data, fail_if_undefined=True)
 481
 482         self._program_testbed_cross_connections(data)
 483
 484         # perform do_configure in parallel for al testbeds
 485         # (it's internal configuration for each)
 486         self._parallel([testbed.do_configure
 487                         for testbed in self._testbeds.itervalues()])
 488
 489         self._clear_caches()
 490
 491         #print >>sys.stderr, "DO IT"
 492         #import time
 493         #time.sleep(60)
 494
 495         # cross-connect (cannot be done in parallel)
 496         for guid, testbed in self._testbeds.iteritems():
 497             cross_data = self._get_cross_data(guid)
 498             testbed.do_cross_connect_init(cross_data)
 499         for guid, testbed in self._testbeds.iteritems():
 500             cross_data = self._get_cross_data(guid)
 501             testbed.do_cross_connect_compl(cross_data)
 502
 503         self._clear_caches()
 504
 505         # Last chance to configure (parallel on all testbeds)
 506         self._parallel([testbed.do_prestart
 507                         for testbed in self._testbeds.itervalues()])
 508
 509         self._clear_caches()
 510
 511         # start experiment (parallel start on all testbeds)
 512         self._parallel([testbed.start
 513                         for testbed in self._testbeds.itervalues()])
 514
 515         self._clear_caches()
 516
 517     def _clear_caches(self):
 518         # Cleaning cache for safety.
 519         self._guids_in_testbed_cache = dict()
 520
 521     def _persist_testbed_proxies(self):
 522         TRANSIENT = ('Recover',)
 523
 524         # persist access configuration for all testbeds, so that
 525         # recovery mode can reconnect to them if it becomes necessary
 526         conf = ConfigParser.RawConfigParser()
 527         for testbed_guid, testbed_config in self._deployment_config.iteritems():
 528             testbed_guid = str(testbed_guid)
 529             conf.add_section(testbed_guid)
 530             for attr in testbed_config.attributes_list:
 531                 if attr not in TRANSIENT:
 532                     conf.set(testbed_guid, attr,
 533                         testbed_config.get_attribute_value(attr))
 534
 535         f = open(os.path.join(self._root_dir, 'deployment_config.ini'), 'w')
 536         conf.write(f)
 537         f.close()
 538
 539     def _load_testbed_proxies(self):
 540         TYPEMAP = {
 541             STRING : 'get',
 542             INTEGER : 'getint',
 543             FLOAT : 'getfloat',
 544             BOOLEAN : 'getboolean',
 545         }
 546
 547         # deferred import because proxy needs
 548         # our class definitions to define proxies
 549         import nepi.util.proxy as proxy
 550
 551         conf = ConfigParser.RawConfigParser()
 552         conf.read(os.path.join(self._root_dir, 'deployment_config.ini'))
 553         for testbed_guid in conf.sections():
 554             testbed_config = proxy.AccessConfiguration()
 555             for attr in conf.options(testbed_guid):
 556                 testbed_config.set_attribute_value(attr,
 557                     conf.get(testbed_guid, attr) )
 558
 559             testbed_guid = str(testbed_guid)
 560             conf.add_section(testbed_guid)
 561             for attr in testbed_config.attributes_list:
 562                 if attr not in TRANSIENT:
 563                     getter = getattr(conf, TYPEMAP.get(
 564                         testbed_config.get_attribute_type(attr),
 565                         'get') )
 566                     testbed_config.set_attribute_value(
 567                         testbed_guid, attr, getter(attr))
 568
 569     def _unpersist_testbed_proxies(self):
 570         try:
 571             os.remove(os.path.join(self._root_dir, 'deployment_config.ini'))
 572         except:
 573             # Just print exceptions, this is just cleanup
 574             import traceback
 575             ######## BUG ##########
 576             #BUG: If the next line is uncomented pyQt explodes when shutting down the experiment !!!!!!!!
 577             #traceback.print_exc(file=sys.stderr)
 578
 579     def stop(self):
 580        for testbed in self._testbeds.values():
 581            testbed.stop()
 582        self._unpersist_testbed_proxies()
 583
 584     def recover(self):
 585         # reload perviously persisted testbed access configurations
 586         self._load_testbed_proxies()
 587
 588         # recreate testbed proxies by reconnecting only
 589         self._init_testbed_controllers(recover = True)
 590
 591         # another time, for netrefs
 592         self._init_testbed_controllers(recover = True)
 593
 594     def is_finished(self, guid):
 595         testbed = self._testbed_for_guid(guid)
 596         if testbed != None:
 597             return testbed.status(guid) == STATUS_FINISHED
 598         raise RuntimeError("No element exists with guid %d" % guid)
 599
 600     def set(self, guid, name, value, time = TIME_NOW):
 601         testbed = self._testbed_for_guid(guid)
 602         if testbed != None:
 603             testbed.set(guid, name, value, time)
 604         else:
 605             raise RuntimeError("No element exists with guid %d" % guid)
 606
 607     def get(self, guid, name, time = TIME_NOW):
 608         testbed = self._testbed_for_guid(guid)
 609         if testbed != None:
 610             return testbed.get(guid, name, time)
 611         raise RuntimeError("No element exists with guid %d" % guid)
 612
 613     def get_factory_id(self, guid):
 614         testbed = self._testbed_for_guid(guid)
 615         if testbed != None:
 616             return testbed.get_factory_id(guid)
 617         raise RuntimeError("No element exists with guid %d" % guid)
 618
 619     def get_testbed_id(self, guid):
 620         testbed = self._testbed_for_guid(guid)
 621         if testbed != None:
 622             return testbed.testbed_id
 623         raise RuntimeError("No element exists with guid %d" % guid)
 624
 625     def get_testbed_version(self, guid):
 626         testbed = self._testbed_for_guid(guid)
 627         if testbed != None:
 628             return testbed.testbed_version
 629         raise RuntimeError("No element exists with guid %d" % guid)
 630
 631     def shutdown(self):
 632         exceptions = list()
 633         for testbed in self._testbeds.values():
 634             try:
 635                 testbed.shutdown()
 636             except:
 637                 exceptions.append(sys.exc_info())
 638         for exc_info in exceptions:
 639             raise exc_info[0], exc_info[1], exc_info[2]
 640
 641     def _testbed_for_guid(self, guid):
 642         for testbed_guid in self._testbeds.keys():
 643             if guid in self._guids_in_testbed(testbed_guid):
 644                 return self._testbeds[testbed_guid]
 645         return None
 646
 647     def _guids_in_testbed(self, testbed_guid):
 648         if testbed_guid not in self._testbeds:
 649             return set()
 650         if testbed_guid not in self._guids_in_testbed_cache:
 651             self._guids_in_testbed_cache[testbed_guid] = \
 652                 set(self._testbeds[testbed_guid].guids)
 653         return self._guids_in_testbed_cache[testbed_guid]
 654
 655     @staticmethod
 656     def _netref_component_split(component):
 657         match = COMPONENT_PATTERN.match(component)
 658         if match:
 659             return match.group("kind"), match.group("index")
 660         else:
 661             return component, None
 662
 663     _NETREF_COMPONENT_GETTERS = {
 664         'addr':
 665             lambda testbed, guid, index, name:
 666                 testbed.get_address(guid, int(index), name),
 667         'route' :
 668             lambda testbed, guid, index, name:
 669                 testbed.get_route(guid, int(index), name),
 670         'trace' :
 671             lambda testbed, guid, index, name:
 672                 testbed.trace(guid, index, name),
 673         '' :
 674             lambda testbed, guid, index, name:
 675                 testbed.get(guid, name),
 676     }
 677
 678     def resolve_netref_value(self, value, failval = None):
 679         match = ATTRIBUTE_PATTERN_BASE.search(value)
 680         if match:
 681             label = match.group("label")
 682             if label.startswith('GUID-'):
 683                 ref_guid = int(label[5:])
 684                 if ref_guid:
 685                     expr = match.group("expr")
 686                     component = (match.group("component") or "")[1:] # skip the dot
 687                     attribute = match.group("attribute")
 688
 689                     # split compound components into component kind and index
 690                     # eg: 'addr[0]' -> ('addr', '0')
 691                     component, component_index = self._netref_component_split(component)
 692
 693                     # find object and resolve expression
 694                     for ref_testbed_guid, ref_testbed in self._testbeds.iteritems():
 695                         if component not in self._NETREF_COMPONENT_GETTERS:
 696                             raise ValueError, "Malformed netref: %r - unknown component" % (expr,)
 697                         elif ref_guid not in self._guids_in_testbed(ref_testbed_guid):
 698                             pass
 699                         else:
 700                             ref_value = self._NETREF_COMPONENT_GETTERS[component](
 701                                 ref_testbed, ref_guid, component_index, attribute)
 702                             if ref_value:
 703                                 return value.replace(match.group(), ref_value)
 704         # couldn't find value
 705         return failval
 706
 707     def do_netrefs(self, data, fail_if_undefined = False):
 708         # element netrefs
 709         for (testbed_guid, guid), attrs in self._netrefs.items():
 710             testbed = self._testbeds.get(testbed_guid)
 711             if testbed is not None:
 712                 for name in set(attrs):
 713                     value = testbed.get(guid, name)
 714                     if isinstance(value, basestring):
 715                         ref_value = self.resolve_netref_value(value)
 716                         if ref_value is not None:
 717                             testbed.set(guid, name, ref_value)
 718                             attrs.remove(name)
 719                         elif fail_if_undefined:
 720                             raise ValueError, "Unresolvable netref in: %r=%r" % (name,value,)
 721                 if not attrs:
 722                     del self._netrefs[(testbed_guid, guid)]
 723
 724         # testbed netrefs
 725         for testbed_guid, attrs in self._testbed_netrefs.items():
 726             tb_data = dict(data.get_attribute_data(testbed_guid))
 727             if data:
 728                 for name in set(attrs):
 729                     value = tb_data.get(name)
 730                     if isinstance(value, basestring):
 731                         ref_value = self.resolve_netref_value(value)
 732                         if ref_value is not None:
 733                             data.set_attribute_data(testbed_guid, name, ref_value)
 734                             attrs.remove(name)
 735                         elif fail_if_undefined:
 736                             raise ValueError, "Unresolvable netref in: %r" % (value,)
 737                 if not attrs:
 738                     del self._testbed_netrefs[testbed_guid]
 739
 740
 741     def _init_testbed_controllers(self, data, recover = False):
 742         blacklist_testbeds = set(self._testbeds)
 743         element_guids = list()
 744         label_guids = dict()
 745         data_guids = data.guids
 746
 747         # create testbed controllers
 748         for guid in data_guids:
 749             if data.is_testbed_data(guid):
 750                 if guid not in self._testbeds:
 751                     self._create_testbed_controller(guid, data, element_guids,
 752                             recover)
 753             else:
 754                 (testbed_guid, factory_id) = data.get_box_data(guid)
 755                 if testbed_guid not in blacklist_testbeds:
 756                     element_guids.append(guid)
 757                     label = data.get_attribute_data(guid, "label")
 758                     if label is not None:
 759                         if label in label_guids:
 760                             raise RuntimeError, "Label %r is not unique" % (label,)
 761                         label_guids[label] = guid
 762
 763         # replace references to elements labels for its guid
 764         self._resolve_labels(data, data_guids, label_guids)
 765
 766         # program testbed controllers
 767         if not recover:
 768             self._program_testbed_controllers(element_guids, data)
 769
 770     def _resolve_labels(self, data, data_guids, label_guids):
 771         netrefs = self._netrefs
 772         testbed_netrefs = self._testbed_netrefs
 773         for guid in data_guids:
 774             for name, value in data.get_attribute_data(guid):
 775                 if isinstance(value, basestring):
 776                     match = ATTRIBUTE_PATTERN_BASE.search(value)
 777                     if match:
 778                         label = match.group("label")
 779                         if not label.startswith('GUID-'):
 780                             ref_guid = label_guids.get(label)
 781                             if ref_guid is not None:
 782                                 value = ATTRIBUTE_PATTERN_BASE.sub(
 783                                     ATTRIBUTE_PATTERN_GUID_SUB % dict(
 784                                         guid = 'GUID-%d' % (ref_guid,),
 785                                         expr = match.group("expr"),
 786                                         label = label),
 787                                     value)
 788                                 data.set_attribute_data(guid, name, value)
 789
 790                                 # memorize which guid-attribute pairs require
 791                                 # postprocessing, to avoid excessive controller-testbed
 792                                 # communication at configuration time
 793                                 # (which could require high-latency network I/O)
 794                                 if not data.is_testbed_data(guid):
 795                                     (testbed_guid, factory_id) = data.get_box_data(guid)
 796                                     netrefs[(testbed_guid, guid)].add(name)
 797                                 else:
 798                                     testbed_netrefs[guid].add(name)
 799
 800     def _create_testbed_controller(self, guid, data, element_guids, recover):
 801         (testbed_id, testbed_version) = data.get_testbed_data(guid)
 802         deployment_config = self._deployment_config.get(guid)
 803
 804         # deferred import because proxy needs
 805         # our class definitions to define proxies
 806         import nepi.util.proxy as proxy
 807
 808         if deployment_config is None:
 809             # need to create one
 810             deployment_config = proxy.AccessConfiguration()
 811
 812             for (name, value) in data.get_attribute_data(guid):
 813                 if value is not None and deployment_config.has_attribute(name):
 814                     # if any deployment config attribute has a netref, we can't
 815                     # create this controller yet
 816                     if isinstance(value, basestring) and ATTRIBUTE_PATTERN_BASE.search(value):
 817                         # remember to re-issue this one
 818                         self._netreffed_testbeds.add(guid)
 819                         return
 820
 821                     # copy deployment config attribute
 822                     deployment_config.set_attribute_value(name, value)
 823
 824             # commit config
 825             self._deployment_config[guid] = deployment_config
 826
 827         if deployment_config is not None:
 828             # force recovery mode
 829             deployment_config.set_attribute_value("recover",recover)
 830
 831         testbed = proxy.create_testbed_controller(testbed_id,
 832                 testbed_version, deployment_config)
 833         for (name, value) in data.get_attribute_data(guid):
 834             testbed.defer_configure(name, value)
 835         self._testbeds[guid] = testbed
 836         if guid in self._netreffed_testbeds:
 837             self._netreffed_testbeds.remove(guid)
 838
 839     def _program_testbed_controllers(self, element_guids, data):
 840         for guid in element_guids:
 841             (testbed_guid, factory_id) = data.get_box_data(guid)
 842             testbed = self._testbeds.get(testbed_guid)
 843             if testbed:
 844                 testbed.defer_create(guid, factory_id)
 845                 for (name, value) in data.get_attribute_data(guid):
 846                     # Try to resolve create-time netrefs, if possible
 847                     if isinstance(value, basestring) and ATTRIBUTE_PATTERN_BASE.search(value):
 848                         try:
 849                             nuvalue = self.resolve_netref_value(value)
 850                         except:
 851                             # Any trouble means we're not in shape to resolve the netref yet
 852                             nuvalue = None
 853                         if nuvalue is not None:
 854                             # Only if we succeed we remove the netref deferral entry
 855                             value = nuvalue
 856                             data.set_attribute_data(guid, name, value)
 857                             if (testbed_guid, guid) in self._netrefs:
 858                                 self._netrefs[(testbed_guid, guid)].discard(name)
 859                     testbed.defer_create_set(guid, name, value)
 860
 861         for guid in element_guids:
 862             (testbed_guid, factory_id) = data.get_box_data(guid)
 863             testbed = self._testbeds.get(testbed_guid)
 864             if testbed:
 865                 for (connector_type_name, cross_guid, cross_connector_type_name) \
 866                         in data.get_connection_data(guid):
 867                     (testbed_guid, factory_id) = data.get_box_data(guid)
 868                     (cross_testbed_guid, cross_factory_id) = data.get_box_data(
 869                             cross_guid)
 870                     if testbed_guid == cross_testbed_guid:
 871                         testbed.defer_connect(guid, connector_type_name,
 872                                 cross_guid, cross_connector_type_name)
 873                 for trace_id in data.get_trace_data(guid):
 874                     testbed.defer_add_trace(guid, trace_id)
 875                 for (autoconf, address, netprefix, broadcast) in \
 876                         data.get_address_data(guid):
 877                     if address != None:
 878                         testbed.defer_add_address(guid, address, netprefix,
 879                                 broadcast)
 880                 for (destination, netprefix, nexthop) in data.get_route_data(guid):
 881                     testbed.defer_add_route(guid, destination, netprefix, nexthop)
 882
 883     def _program_testbed_cross_connections(self, data):
 884         data_guids = data.guids
 885
 886         for guid in data_guids:
 887             if not data.is_testbed_data(guid):
 888                 (testbed_guid, factory_id) = data.get_box_data(guid)
 889                 testbed = self._testbeds.get(testbed_guid)
 890                 if testbed:
 891                     for (connector_type_name, cross_guid, cross_connector_type_name) \
 892                             in data.get_connection_data(guid):
 893                         (testbed_guid, factory_id) = data.get_box_data(guid)
 894                         (cross_testbed_guid, cross_factory_id) = data.get_box_data(
 895                                 cross_guid)
 896                         if testbed_guid != cross_testbed_guid:
 897                             cross_testbed = self._testbeds[cross_testbed_guid]
 898                             cross_testbed_id = cross_testbed.testbed_id
 899                             testbed.defer_cross_connect(guid, connector_type_name, cross_guid,
 900                                     cross_testbed_guid, cross_testbed_id, cross_factory_id,
 901                                     cross_connector_type_name)
 902                             # save cross data for later
 903                             self._add_crossdata(testbed_guid, guid, cross_testbed_guid,
 904                                     cross_guid)
 905
 906     def _add_crossdata(self, testbed_guid, guid, cross_testbed_guid, cross_guid):
 907         if testbed_guid not in self._cross_data:
 908             self._cross_data[testbed_guid] = dict()
 909         if cross_testbed_guid not in self._cross_data[testbed_guid]:
 910             self._cross_data[testbed_guid][cross_testbed_guid] = set()
 911         self._cross_data[testbed_guid][cross_testbed_guid].add(cross_guid)
 912
 913     def _get_cross_data(self, testbed_guid):
 914         cross_data = dict()
 915         if not testbed_guid in self._cross_data:
 916             return cross_data
 917         for cross_testbed_guid, guid_list in \
 918                 self._cross_data[testbed_guid].iteritems():
 919             cross_data[cross_testbed_guid] = dict()
 920             cross_testbed = self._testbeds[cross_testbed_guid]
 921             for cross_guid in guid_list:
 922                 elem_cross_data = dict(
 923                     _guid = cross_guid,
 924                     _testbed_guid = cross_testbed_guid,
 925                     _testbed_id = cross_testbed.testbed_id,
 926                     _testbed_version = cross_testbed.testbed_version)
 927                 cross_data[cross_testbed_guid][cross_guid] = elem_cross_data
 928                 attributes_list = cross_testbed.get_attribute_list(cross_guid)
 929                 for attr_name in attributes_list:
 930                     attr_value = cross_testbed.get(cross_guid, attr_name)
 931                     elem_cross_data[attr_name] = attr_value
 932         return cross_data
 933