src/nepi/core/metadata.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from nepi.core.attributes import Attribute, AttributesMap
   5 import sys
   6 import getpass
   7 from nepi.util import validation
   8 from nepi.util.constants import ATTR_NEPI_TESTBED_ENVIRONMENT_SETUP, DeploymentConfiguration
   9
  10 class VersionedMetadataInfo(object):
  11     @property
  12     def connector_types(self):
  13         """ dictionary of dictionaries with allowed connection information.
  14             connector_id: dict({
  15                 "help": help text,
  16                 "name": connector type name,
  17                 "max": maximum number of connections allowed (-1 for no limit),
  18                 "min": minimum number of connections allowed
  19             }),
  20         """
  21         raise NotImplementedError
  22
  23     @property
  24     def connections(self):
  25         """ array of dictionaries with allowed connection information.
  26         dict({
  27             "from": (testbed_id1, factory_id1, connector_type_name1),
  28             "to": (testbed_id2, factory_id2, connector_type_name2),
  29             "init_code": connection function to invoke for connection initiation
  30             "compl_code": connection function to invoke for connection
  31                 completion
  32             "can_cross": whether the connection can be done across testbed
  33                             instances
  34          }),
  35         """
  36         raise NotImplementedError
  37
  38     @property
  39     def attributes(self):
  40         """ dictionary of dictionaries of all available attributes.
  41             attribute_id: dict({
  42                 "name": attribute name,
  43                 "help": help text,
  44                 "type": attribute type,
  45                 "value": default attribute value,
  46                 "range": (maximum, minimun) values else None if not defined,
  47                 "allowed": array of posible values,
  48                 "flags": attributes flags,
  49                 "validation_function": validation function for the attribute
  50             })
  51         """
  52         raise NotImplementedError
  53
  54     @property
  55     def traces(self):
  56         """ dictionary of dictionaries of all available traces.
  57             trace_id: dict({
  58                 "name": trace name,
  59                 "help": help text
  60             })
  61         """
  62         raise NotImplementedError
  63
  64     @property
  65     def create_order(self):
  66         """ list of factory ids that indicates the order in which the elements
  67         should be instantiated.
  68         """
  69         raise NotImplementedError
  70
  71     @property
  72     def configure_order(self):
  73         """ list of factory ids that indicates the order in which the elements
  74         should be configured.
  75         """
  76         raise NotImplementedError
  77
  78     @property
  79     def preconfigure_order(self):
  80         """ list of factory ids that indicates the order in which the elements
  81         should be preconfigured.
  82
  83         Default: same as configure_order
  84         """
  85         return self.configure_order
  86
  87     @property
  88     def start_order(self):
  89         """ list of factory ids that indicates the order in which the elements
  90         should be started.
  91
  92         Default: same as configure_order
  93         """
  94         return self.configure_order
  95
  96     @property
  97     def factories_info(self):
  98         """ dictionary of dictionaries of factory specific information
  99             factory_id: dict({
 100                 "allow_addresses": whether the box allows adding IP addresses,
 101                 "allow_routes": wether the box allows adding routes,
 102                 "has_addresses": whether the box allows obtaining IP addresses,
 103                 "has_routes": wether the box allows obtaining routes,
 104                 "help": help text,
 105                 "category": category the element belongs to,
 106                 "create_function": function for element instantiation,
 107                 "start_function": function for element starting,
 108                 "stop_function": function for element stoping,
 109                 "status_function": function for retrieving element status,
 110                 "preconfigure_function": function for element preconfiguration,
 111                     (just after connections are made,
 112                     just before netrefs are resolved)
 113                 "configure_function": function for element configuration,
 114                 "factory_attributes": list of references to attribute_ids,
 115                 "box_attributes": list of regerences to attribute_ids,
 116                 "traces": list of references to trace_id
 117                 "connector_types": list of references to connector_types
 118            })
 119         """
 120         raise NotImplementedError
 121
 122     @property
 123     def testbed_attributes(self):
 124         """ dictionary of attributes for testbed instance configuration
 125             attributes_id = dict({
 126                 "name": attribute name,
 127                 "help": help text,
 128                 "type": attribute type,
 129                 "value": default attribute value,
 130                 "range": (maximum, minimun) values else None if not defined,
 131                 "allowed": array of posible values,
 132                 "flags": attributes flags,
 133                 "validation_function": validation function for the attribute
 134              })
 135             ]
 136         """
 137         raise NotImplementedError
 138
 139 class Metadata(object):
 140     STANDARD_BOX_ATTRIBUTES = (
 141         ("label", dict(
 142             name = "label",
 143             validation_function = validation.is_string,
 144             type = Attribute.STRING,
 145             flags = Attribute.DesignOnly,
 146             help = "A unique identifier for referring to this box",
 147         )),
 148     )
 149
 150     # Shorthand for DeploymentConfiguration
 151     # Syntactic sugar to shorten stuff
 152     DC = DeploymentConfiguration
 153
 154     STANDARD_TESTBED_ATTRIBUTES = (
 155         ("home_directory", dict(
 156             name = "homeDirectory",
 157             validation_function = validation.is_string,
 158             help = "Path to the directory where traces and other files will be stored",
 159             type = Attribute.STRING,
 160             value = "",
 161             flags = Attribute.DesignOnly,
 162         )),
 163     )
 164
 165     DEPLOYMENT_ATTRIBUTES = (
 166         # TESTBED DEPLOYMENT ATTRIBUTES
 167         (DC.DEPLOYMENT_ENVIRONMENT_SETUP, dict(
 168                 name = DC.DEPLOYMENT_ENVIRONMENT_SETUP,
 169                 validation_function = validation.is_string,
 170                 help = "Shell commands to run before spawning TestbedController processes",
 171                 type = Attribute.STRING,
 172                 flags = Attribute.DesignOnly,
 173         )),
 174         (DC.DEPLOYMENT_MODE, dict(name = DC.DEPLOYMENT_MODE,
 175                 help = "Instance execution mode",
 176                 type = Attribute.ENUM,
 177                 value = DC.MODE_SINGLE_PROCESS,
 178                 allowed = [
 179                     DC.MODE_DAEMON,
 180                     DC.MODE_SINGLE_PROCESS
 181                 ],
 182                 flags = Attribute.DesignOnly,
 183                 validation_function = validation.is_enum
 184         )),
 185         (DC.DEPLOYMENT_COMMUNICATION, dict(name = DC.DEPLOYMENT_COMMUNICATION,
 186                 help = "Instance communication mode",
 187                 type = Attribute.ENUM,
 188                 value = DC.ACCESS_LOCAL,
 189                 allowed = [
 190                     DC.ACCESS_LOCAL,
 191                     DC.ACCESS_SSH
 192                 ],
 193                 flags = Attribute.DesignOnly,
 194                 validation_function = validation.is_enum
 195         )),
 196         (DC.DEPLOYMENT_HOST, dict(name = DC.DEPLOYMENT_HOST,
 197                 help = "Host where the testbed will be executed",
 198                 type = Attribute.STRING,
 199                 value = "localhost",
 200                 flags = Attribute.DesignOnly,
 201                 validation_function = validation.is_string
 202         )),
 203         (DC.DEPLOYMENT_USER, dict(name = DC.DEPLOYMENT_USER,
 204                 help = "User on the Host to execute the testbed",
 205                 type = Attribute.STRING,
 206                 value = getpass.getuser(),
 207                 flags = Attribute.DesignOnly,
 208                 validation_function = validation.is_string
 209         )),
 210         (DC.DEPLOYMENT_KEY, dict(name = DC.DEPLOYMENT_KEY,
 211                 help = "Path to SSH key to use for connecting",
 212                 type = Attribute.STRING,
 213                 flags = Attribute.DesignOnly,
 214                 validation_function = validation.is_string
 215         )),
 216         (DC.DEPLOYMENT_PORT, dict(name = DC.DEPLOYMENT_PORT,
 217                 help = "Port on the Host",
 218                 type = Attribute.INTEGER,
 219                 value = 22,
 220                 flags = Attribute.DesignOnly,
 221                 validation_function = validation.is_integer
 222         )),
 223         (DC.ROOT_DIRECTORY, dict(name = DC.ROOT_DIRECTORY,
 224                 help = "Root directory for storing process files",
 225                 type = Attribute.STRING,
 226                 value = ".",
 227                 flags = Attribute.DesignOnly,
 228                 validation_function = validation.is_string # TODO: validation.is_path
 229         )),
 230         (DC.USE_AGENT, dict(name = DC.USE_AGENT,
 231                 help = "Use -A option for forwarding of the authentication agent, if ssh access is used",
 232                 type = Attribute.BOOL,
 233                 value = False,
 234                 flags = Attribute.DesignOnly,
 235                 validation_function = validation.is_bool
 236         )),
 237         (DC.LOG_LEVEL, dict(name = DC.LOG_LEVEL,
 238                 help = "Log level for instance",
 239                 type = Attribute.ENUM,
 240                 value = DC.ERROR_LEVEL,
 241                 allowed = [
 242                     DC.ERROR_LEVEL,
 243                     DC.DEBUG_LEVEL
 244                 ],
 245                 flags = Attribute.DesignOnly,
 246                 validation_function = validation.is_enum
 247         )),
 248         (DC.RECOVER, dict(name = DC.RECOVER,
 249                 help = "Do not intantiate testbeds, rather, reconnect to already-running instances. Used to recover from a dead controller.",
 250                 type = Attribute.BOOL,
 251                 value = False,
 252                 flags = Attribute.DesignOnly,
 253                 validation_function = validation.is_bool
 254         )),
 255     )
 256
 257     STANDARD_TESTBED_ATTRIBUTES += DEPLOYMENT_ATTRIBUTES
 258
 259     del DC
 260
 261
 262     STANDARD_ATTRIBUTE_BUNDLES = {
 263             "tun_proto" : dict({
 264                 "name": "tun_proto",
 265                 "help": "TUNneling protocol used",
 266                 "type": Attribute.STRING,
 267                 "flags": Attribute.Invisible,
 268                 "validation_function": validation.is_string,
 269             }),
 270             "tun_key" : dict({
 271                 "name": "tun_key",
 272                 "help": "Randomly selected TUNneling protocol cryptographic key. "
 273                         "Endpoints must agree to use the minimum (in lexicographic order) "
 274                         "of both the remote and local sides.",
 275                 "type": Attribute.STRING,
 276                 "flags": Attribute.Invisible,
 277                 "validation_function": validation.is_string,
 278             }),
 279             "tun_addr" : dict({
 280                 "name": "tun_addr",
 281                 "help": "Address (IP, unix socket, whatever) of the tunnel endpoint",
 282                 "type": Attribute.STRING,
 283                 "flags": Attribute.Invisible,
 284                 "validation_function": validation.is_string,
 285             }),
 286             "tun_port" : dict({
 287                 "name": "tun_port",
 288                 "help": "IP port of the tunnel endpoint",
 289                 "type": Attribute.INTEGER,
 290                 "flags": Attribute.Invisible,
 291                 "validation_function": validation.is_integer,
 292             }),
 293             ATTR_NEPI_TESTBED_ENVIRONMENT_SETUP : dict({
 294                 "name": ATTR_NEPI_TESTBED_ENVIRONMENT_SETUP,
 295                 "help": "Commands to set up the environment needed to run NEPI testbeds",
 296                 "type": Attribute.STRING,
 297                 "flags": Attribute.Invisible,
 298                 "validation_function": validation.is_string
 299             }),
 300     }
 301
 302
 303     def __init__(self, testbed_id, version):
 304         self._version = version
 305         self._testbed_id = testbed_id
 306         metadata_module = self._load_versioned_metadata_module()
 307         self._metadata = metadata_module.VersionedMetadataInfo()
 308
 309     @property
 310     def create_order(self):
 311         return self._metadata.create_order
 312
 313     @property
 314     def configure_order(self):
 315         return self._metadata.configure_order
 316
 317     @property
 318     def preconfigure_order(self):
 319         return self._metadata.preconfigure_order
 320
 321     @property
 322     def start_order(self):
 323         return self._metadata.start_order
 324
 325     def testbed_attributes(self):
 326         attributes = AttributesMap()
 327
 328         # standard attributes
 329         self._add_standard_attributes(attributes, None, True, False,
 330             self.STANDARD_TESTBED_ATTRIBUTES)
 331
 332         # custom attributes - they override standard ones
 333         for attr_info in self._metadata.testbed_attributes.values():
 334             name = attr_info["name"]
 335             help = attr_info["help"]
 336             type = attr_info["type"]
 337             value = attr_info["value"] if "value" in attr_info else None
 338             range = attr_info["range"] if "range" in attr_info else None
 339             allowed = attr_info["allowed"] if "allowed" in attr_info else None
 340             flags =  attr_info["flags"] if "flags" in attr_info \
 341                     else Attribute.NoFlags
 342             validation_function = attr_info["validation_function"]
 343             attributes.add_attribute(name, help, type, value,
 344                     range, allowed, flags, validation_function)
 345
 346         return attributes
 347
 348     def build_design_factories(self):
 349         from nepi.core.design import Factory
 350         factories = list()
 351         for factory_id, info in self._metadata.factories_info.iteritems():
 352             help = info["help"]
 353             category = info["category"]
 354             allow_addresses = info.get("allow_addresses", False)
 355             allow_routes = info.get("allow_routes", False)
 356             has_addresses = info.get("has_addresses", False)
 357             has_routes = info.get("has_routes", False)
 358             factory = Factory(factory_id,
 359                     allow_addresses, has_addresses,
 360                     allow_routes, has_routes,
 361                     help, category)
 362
 363             # standard attributes
 364             self._add_standard_attributes(factory, info, True, True,
 365                 self.STANDARD_BOX_ATTRIBUTES)
 366
 367             # custom attributes - they override standard ones
 368             self._add_attributes(factory, info, "factory_attributes")
 369             self._add_attributes(factory, info, "box_attributes", True)
 370
 371             self._add_design_traces(factory, info)
 372             self._add_design_connector_types(factory, info)
 373             factories.append(factory)
 374         return factories
 375
 376     def build_execute_factories(self):
 377         from nepi.core.execute import Factory
 378         factories = list()
 379         for factory_id, info in self._metadata.factories_info.iteritems():
 380             create_function = info.get("create_function")
 381             start_function = info.get("start_function")
 382             stop_function = info.get("stop_function")
 383             status_function = info.get("status_function")
 384             configure_function = info.get("configure_function")
 385             preconfigure_function = info.get("preconfigure_function")
 386             allow_addresses = info.get("allow_addresses", False)
 387             allow_routes = info.get("allow_routes", False)
 388             has_addresses = info.get("has_addresses", False)
 389             has_routes = info.get("has_routes", False)
 390             factory = Factory(factory_id, create_function, start_function,
 391                     stop_function, status_function,
 392                     configure_function, preconfigure_function,
 393                     allow_addresses, has_addresses,
 394                     allow_routes, has_routes)
 395
 396             # standard attributes
 397             self._add_standard_attributes(factory, info, False, True,
 398                 self.STANDARD_BOX_ATTRIBUTES)
 399
 400             # custom attributes - they override standard ones
 401             self._add_attributes(factory, info, "factory_attributes")
 402             self._add_attributes(factory, info, "box_attributes", True)
 403
 404             self._add_execute_traces(factory, info)
 405             self._add_execute_connector_types(factory, info)
 406             factories.append(factory)
 407         return factories
 408
 409     def _load_versioned_metadata_module(self):
 410         mod_name = "nepi.testbeds.%s.metadata_v%s" % (self._testbed_id.lower(),
 411                 self._version)
 412         if not mod_name in sys.modules:
 413             __import__(mod_name)
 414         return sys.modules[mod_name]
 415
 416     def _add_standard_attributes(self, factory, info, design, box, STANDARD_ATTRIBUTES):
 417         if design:
 418             attr_bundle = STANDARD_ATTRIBUTES
 419         else:
 420             # Only add non-DesignOnly attributes
 421             def nonDesign(attr_info):
 422                 return not (attr_info[1].get('flags',Attribute.NoFlags) & Attribute.DesignOnly)
 423             attr_bundle = filter(nonDesign, STANDARD_ATTRIBUTES)
 424         self._add_attributes(factory, info, None, box,
 425             attr_bundle = STANDARD_ATTRIBUTES)
 426
 427     def _add_attributes(self, factory, info, attr_key, box_attributes = False, attr_bundle = ()):
 428         if not attr_bundle and info and attr_key in info:
 429             definitions = self.STANDARD_ATTRIBUTE_BUNDLES.copy()
 430             definitions.update(self._metadata.attributes)
 431             attr_bundle = [ (attr_id, definitions[attr_id])
 432                             for attr_id in info[attr_key] ]
 433         for attr_id, attr_info in attr_bundle:
 434             name = attr_info["name"]
 435             help = attr_info["help"]
 436             type = attr_info["type"]
 437             value = attr_info["value"] if "value" in attr_info else None
 438             range = attr_info["range"] if "range" in attr_info else None
 439             allowed = attr_info["allowed"] if "allowed" in attr_info \
 440                     else None
 441             flags = attr_info["flags"] if "flags" in attr_info \
 442                     and attr_info["flags"] != None \
 443                     else Attribute.NoFlags
 444             validation_function = attr_info["validation_function"]
 445             if box_attributes:
 446                 factory.add_box_attribute(name, help, type, value, range,
 447                         allowed, flags, validation_function)
 448             else:
 449                 factory.add_attribute(name, help, type, value, range,
 450                         allowed, flags, validation_function)
 451
 452     def _add_design_traces(self, factory, info):
 453         if "traces" in info:
 454             for trace in info["traces"]:
 455                 trace_info = self._metadata.traces[trace]
 456                 trace_id = trace_info["name"]
 457                 help = trace_info["help"]
 458                 factory.add_trace(trace_id, help)
 459
 460     def _add_execute_traces(self, factory, info):
 461         if "traces" in info:
 462             for trace in info["traces"]:
 463                 trace_info = self._metadata.traces[trace]
 464                 trace_id = trace_info["name"]
 465                 factory.add_trace(trace_id)
 466
 467     def _add_design_connector_types(self, factory, info):
 468         from nepi.core.design import ConnectorType
 469         if "connector_types" in info:
 470             connections = dict()
 471             for connection in self._metadata.connections:
 472                 from_ = connection["from"]
 473                 to = connection["to"]
 474                 can_cross = connection["can_cross"]
 475                 if from_ not in connections:
 476                     connections[from_] = list()
 477                 if to not in connections:
 478                     connections[to] = list()
 479                 connections[from_].append((to, can_cross))
 480                 connections[to].append((from_, can_cross))
 481             for connector_id in info["connector_types"]:
 482                 connector_type_info = self._metadata.connector_types[
 483                         connector_id]
 484                 name = connector_type_info["name"]
 485                 help = connector_type_info["help"]
 486                 max = connector_type_info["max"]
 487                 min = connector_type_info["min"]
 488                 testbed_id = self._testbed_id
 489                 factory_id = factory.factory_id
 490                 connector_type = ConnectorType(testbed_id, factory_id, name,
 491                         help, max, min)
 492                 for (to, can_cross) in connections[(testbed_id, factory_id,
 493                         name)]:
 494                     (testbed_id_to, factory_id_to, name_to) = to
 495                     connector_type.add_allowed_connection(testbed_id_to,
 496                             factory_id_to, name_to, can_cross)
 497                 factory.add_connector_type(connector_type)
 498
 499     def _add_execute_connector_types(self, factory, info):
 500         from nepi.core.execute import ConnectorType
 501         if "connector_types" in info:
 502             from_connections = dict()
 503             to_connections = dict()
 504             for connection in self._metadata.connections:
 505                 from_ = connection["from"]
 506                 to = connection["to"]
 507                 can_cross = connection["can_cross"]
 508                 init_code = connection["init_code"] \
 509                         if "init_code" in connection else None
 510                 compl_code = connection["compl_code"] \
 511                         if "compl_code" in connection else None
 512                 if from_ not in from_connections:
 513                     from_connections[from_] = list()
 514                 if to not in to_connections:
 515                     to_connections[to] = list()
 516                 from_connections[from_].append((to, can_cross, init_code,
 517                     compl_code))
 518                 to_connections[to].append((from_, can_cross, init_code,
 519                     compl_code))
 520             for connector_id in info["connector_types"]:
 521                 connector_type_info = self._metadata.connector_types[
 522                         connector_id]
 523                 name = connector_type_info["name"]
 524                 max = connector_type_info["max"]
 525                 min = connector_type_info["min"]
 526                 testbed_id = self._testbed_id
 527                 factory_id = factory.factory_id
 528                 connector_type = ConnectorType(testbed_id, factory_id, name,
 529                         max, min)
 530                 connector_key = (testbed_id, factory_id, name)
 531                 if connector_key in to_connections:
 532                     for (from_, can_cross, init_code, compl_code) in \
 533                             to_connections[connector_key]:
 534                         (testbed_id_from, factory_id_from, name_from) = from_
 535                         connector_type.add_from_connection(testbed_id_from,
 536                                 factory_id_from, name_from, can_cross,
 537                                 init_code, compl_code)
 538                 if connector_key in from_connections:
 539                     for (to, can_cross, init_code, compl_code) in \
 540                             from_connections[(testbed_id, factory_id, name)]:
 541                         (testbed_id_to, factory_id_to, name_to) = to
 542                         connector_type.add_to_connection(testbed_id_to,
 543                                 factory_id_to, name_to, can_cross, init_code,
 544                                 compl_code)
 545                 factory.add_connector_type(connector_type)
 546