src/nepi/core/metadata.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from nepi.core.attributes import Attribute, AttributesMap
   5 import sys
   6 import getpass
   7 from nepi.util import validation
   8 from nepi.util.constants import ATTR_NEPI_TESTBED_ENVIRONMENT_SETUP, DeploymentConfiguration
   9
  10 class VersionedMetadataInfo(object):
  11     @property
  12     def connector_types(self):
  13         """ dictionary of dictionaries with allowed connection information.
  14             connector_id: dict({
  15                 "help": help text,
  16                 "name": connector type name,
  17                 "max": maximum number of connections allowed (-1 for no limit),
  18                 "min": minimum number of connections allowed
  19             }),
  20         """
  21         raise NotImplementedError
  22
  23     @property
  24     def connections(self):
  25         """ array of dictionaries with allowed connection information.
  26         dict({
  27             "from": (testbed_id1, factory_id1, connector_type_name1),
  28             "to": (testbed_id2, factory_id2, connector_type_name2),
  29             "init_code": connection function to invoke for connection initiation
  30             "compl_code": connection function to invoke for connection
  31                 completion
  32             "can_cross": whether the connection can be done across testbed
  33                             instances
  34          }),
  35         """
  36         raise NotImplementedError
  37
  38     @property
  39     def attributes(self):
  40         """ dictionary of dictionaries of all available attributes.
  41             attribute_id: dict({
  42                 "name": attribute name,
  43                 "help": help text,
  44                 "type": attribute type,
  45                 "value": default attribute value,
  46                 "range": (maximum, minimun) values else None if not defined,
  47                 "allowed": array of posible values,
  48                 "flags": attributes flags,
  49                 "validation_function": validation function for the attribute
  50             })
  51         """
  52         raise NotImplementedError
  53
  54     @property
  55     def traces(self):
  56         """ dictionary of dictionaries of all available traces.
  57             trace_id: dict({
  58                 "name": trace name,
  59                 "help": help text
  60             })
  61         """
  62         raise NotImplementedError
  63
  64     @property
  65     def create_order(self):
  66         """ list of factory ids that indicates the order in which the elements
  67         should be instantiated.
  68         """
  69         raise NotImplementedError
  70
  71     @property
  72     def configure_order(self):
  73         """ list of factory ids that indicates the order in which the elements
  74         should be configured.
  75         """
  76         raise NotImplementedError
  77
  78     @property
  79     def preconfigure_order(self):
  80         """ list of factory ids that indicates the order in which the elements
  81         should be preconfigured.
  82
  83         Default: same as configure_order
  84         """
  85         return self.configure_order
  86
  87     @property
  88     def factories_info(self):
  89         """ dictionary of dictionaries of factory specific information
  90             factory_id: dict({
  91                 "allow_addresses": whether the box allows adding IP addresses,
  92                 "allow_routes": wether the box allows adding routes,
  93                 "has_addresses": whether the box allows obtaining IP addresses,
  94                 "has_routes": wether the box allows obtaining routes,
  95                 "help": help text,
  96                 "category": category the element belongs to,
  97                 "create_function": function for element instantiation,
  98                 "start_function": function for element starting,
  99                 "stop_function": function for element stoping,
 100                 "status_function": function for retrieving element status,
 101                 "preconfigure_function": function for element preconfiguration,
 102                     (just after connections are made,
 103                     just before netrefs are resolved)
 104                 "configure_function": function for element configuration,
 105                 "factory_attributes": list of references to attribute_ids,
 106                 "box_attributes": list of regerences to attribute_ids,
 107                 "traces": list of references to trace_id
 108                 "connector_types": list of references to connector_types
 109            })
 110         """
 111         raise NotImplementedError
 112
 113     @property
 114     def testbed_attributes(self):
 115         """ dictionary of attributes for testbed instance configuration
 116             attributes_id = dict({
 117                 "name": attribute name,
 118                 "help": help text,
 119                 "type": attribute type,
 120                 "value": default attribute value,
 121                 "range": (maximum, minimun) values else None if not defined,
 122                 "allowed": array of posible values,
 123                 "flags": attributes flags,
 124                 "validation_function": validation function for the attribute
 125              })
 126             ]
 127         """
 128         raise NotImplementedError
 129
 130 class Metadata(object):
 131     STANDARD_BOX_ATTRIBUTES = (
 132         ("label", dict(
 133             name = "label",
 134             validation_function = validation.is_string,
 135             type = Attribute.STRING,
 136             flags = Attribute.DesignOnly,
 137             help = "A unique identifier for referring to this box",
 138         )),
 139     )
 140
 141     # Shorthand for DeploymentConfiguration
 142     # Syntactic sugar to shorten stuff
 143     DC = DeploymentConfiguration
 144
 145     STANDARD_TESTBED_ATTRIBUTES = (
 146         ("home_directory", dict(
 147             name = "homeDirectory",
 148             validation_function = validation.is_string,
 149             help = "Path to the directory where traces and other files will be stored",
 150             type = Attribute.STRING,
 151             value = "",
 152             flags = Attribute.DesignOnly,
 153         )),
 154     )
 155
 156     DEPLOYMENT_ATTRIBUTES = (
 157         # TESTBED DEPLOYMENT ATTRIBUTES
 158         (DC.DEPLOYMENT_ENVIRONMENT_SETUP, dict(
 159                 name = DC.DEPLOYMENT_ENVIRONMENT_SETUP,
 160                 validation_function = validation.is_string,
 161                 help = "Shell commands to run before spawning TestbedController processes",
 162                 type = Attribute.STRING,
 163                 flags = Attribute.DesignOnly,
 164         )),
 165         (DC.DEPLOYMENT_MODE, dict(name = DC.DEPLOYMENT_MODE,
 166                 help = "Instance execution mode",
 167                 type = Attribute.ENUM,
 168                 value = DC.MODE_SINGLE_PROCESS,
 169                 allowed = [
 170                     DC.MODE_DAEMON,
 171                     DC.MODE_SINGLE_PROCESS
 172                 ],
 173                 flags = Attribute.DesignOnly,
 174                 validation_function = validation.is_enum
 175         )),
 176         (DC.DEPLOYMENT_COMMUNICATION, dict(name = DC.DEPLOYMENT_COMMUNICATION,
 177                 help = "Instance communication mode",
 178                 type = Attribute.ENUM,
 179                 value = DC.ACCESS_LOCAL,
 180                 allowed = [
 181                     DC.ACCESS_LOCAL,
 182                     DC.ACCESS_SSH
 183                 ],
 184                 flags = Attribute.DesignOnly,
 185                 validation_function = validation.is_enum
 186         )),
 187         (DC.DEPLOYMENT_HOST, dict(name = DC.DEPLOYMENT_HOST,
 188                 help = "Host where the testbed will be executed",
 189                 type = Attribute.STRING,
 190                 value = "localhost",
 191                 flags = Attribute.DesignOnly,
 192                 validation_function = validation.is_string
 193         )),
 194         (DC.DEPLOYMENT_USER, dict(name = DC.DEPLOYMENT_USER,
 195                 help = "User on the Host to execute the testbed",
 196                 type = Attribute.STRING,
 197                 value = getpass.getuser(),
 198                 flags = Attribute.DesignOnly,
 199                 validation_function = validation.is_string
 200         )),
 201         (DC.DEPLOYMENT_KEY, dict(name = DC.DEPLOYMENT_KEY,
 202                 help = "Path to SSH key to use for connecting",
 203                 type = Attribute.STRING,
 204                 flags = Attribute.DesignOnly,
 205                 validation_function = validation.is_string
 206         )),
 207         (DC.DEPLOYMENT_PORT, dict(name = DC.DEPLOYMENT_PORT,
 208                 help = "Port on the Host",
 209                 type = Attribute.INTEGER,
 210                 value = 22,
 211                 flags = Attribute.DesignOnly,
 212                 validation_function = validation.is_integer
 213         )),
 214         (DC.ROOT_DIRECTORY, dict(name = DC.ROOT_DIRECTORY,
 215                 help = "Root directory for storing process files",
 216                 type = Attribute.STRING,
 217                 value = ".",
 218                 flags = Attribute.DesignOnly,
 219                 validation_function = validation.is_string # TODO: validation.is_path
 220         )),
 221         (DC.USE_AGENT, dict(name = DC.USE_AGENT,
 222                 help = "Use -A option for forwarding of the authentication agent, if ssh access is used",
 223                 type = Attribute.BOOL,
 224                 value = False,
 225                 flags = Attribute.DesignOnly,
 226                 validation_function = validation.is_bool
 227         )),
 228         (DC.LOG_LEVEL, dict(name = DC.LOG_LEVEL,
 229                 help = "Log level for instance",
 230                 type = Attribute.ENUM,
 231                 value = DC.ERROR_LEVEL,
 232                 allowed = [
 233                     DC.ERROR_LEVEL,
 234                     DC.DEBUG_LEVEL
 235                 ],
 236                 flags = Attribute.DesignOnly,
 237                 validation_function = validation.is_enum
 238         )),
 239         (DC.RECOVER, dict(name = DC.RECOVER,
 240                 help = "Do not intantiate testbeds, rather, reconnect to already-running instances. Used to recover from a dead controller.",
 241                 type = Attribute.BOOL,
 242                 value = False,
 243                 flags = Attribute.DesignOnly,
 244                 validation_function = validation.is_bool
 245         )),
 246     )
 247
 248     STANDARD_TESTBED_ATTRIBUTES += DEPLOYMENT_ATTRIBUTES
 249
 250     del DC
 251
 252
 253     STANDARD_ATTRIBUTE_BUNDLES = {
 254             "tun_proto" : dict({
 255                 "name": "tun_proto",
 256                 "help": "TUNneling protocol used",
 257                 "type": Attribute.STRING,
 258                 "flags": Attribute.Invisible | Attribute.ReadOnly,
 259                 "validation_function": validation.is_string,
 260             }),
 261             "tun_addr" : dict({
 262                 "name": "tun_addr",
 263                 "help": "IP address of the tunnel endpoint",
 264                 "type": Attribute.STRING,
 265                 "flags": Attribute.Invisible | Attribute.ReadOnly,
 266                 "validation_function": validation.is_ip_address,
 267             }),
 268             "tun_port" : dict({
 269                 "name": "tun_port",
 270                 "help": "IP port of the tunnel endpoint",
 271                 "type": Attribute.INTEGER,
 272                 "flags": Attribute.Invisible | Attribute.ReadOnly,
 273                 "validation_function": validation.is_integer,
 274             }),
 275     }
 276
 277
 278     def __init__(self, testbed_id, version):
 279         self._version = version
 280         self._testbed_id = testbed_id
 281         metadata_module = self._load_versioned_metadata_module()
 282         self._metadata = metadata_module.VersionedMetadataInfo()
 283
 284     @property
 285     def create_order(self):
 286         return self._metadata.create_order
 287
 288     @property
 289     def configure_order(self):
 290         return self._metadata.configure_order
 291
 292     @property
 293     def preconfigure_order(self):
 294         return self._metadata.preconfigure_order
 295
 296     def testbed_attributes(self):
 297         attributes = AttributesMap()
 298
 299         # standard attributes
 300         self._add_standard_attributes(attributes, None, True, False,
 301             self.STANDARD_TESTBED_ATTRIBUTES)
 302
 303         # custom attributes - they override standard ones
 304         for attr_info in self._metadata.testbed_attributes.values():
 305             name = attr_info["name"]
 306             help = attr_info["help"]
 307             type = attr_info["type"]
 308             value = attr_info["value"] if "value" in attr_info else None
 309             range = attr_info["range"] if "range" in attr_info else None
 310             allowed = attr_info["allowed"] if "allowed" in attr_info else None
 311             flags =  attr_info["flags"] if "flags" in attr_info \
 312                     else Attribute.NoFlags
 313             validation_function = attr_info["validation_function"]
 314             attributes.add_attribute(name, help, type, value,
 315                     range, allowed, flags, validation_function)
 316
 317         return attributes
 318
 319     def build_design_factories(self):
 320         from nepi.core.design import Factory
 321         factories = list()
 322         for factory_id, info in self._metadata.factories_info.iteritems():
 323             help = info["help"]
 324             category = info["category"]
 325             allow_addresses = info.get("allow_addresses", False)
 326             allow_routes = info.get("allow_routes", False)
 327             has_addresses = info.get("has_addresses", False)
 328             has_routes = info.get("has_routes", False)
 329             factory = Factory(factory_id,
 330                     allow_addresses, has_addresses,
 331                     allow_routes, has_routes,
 332                     help, category)
 333
 334             # standard attributes
 335             self._add_standard_attributes(factory, info, True, True,
 336                 self.STANDARD_BOX_ATTRIBUTES)
 337
 338             # custom attributes - they override standard ones
 339             self._add_attributes(factory, info, "factory_attributes")
 340             self._add_attributes(factory, info, "box_attributes", True)
 341
 342             self._add_design_traces(factory, info)
 343             self._add_design_connector_types(factory, info)
 344             factories.append(factory)
 345         return factories
 346
 347     def build_execute_factories(self):
 348         from nepi.core.execute import Factory
 349         factories = list()
 350         for factory_id, info in self._metadata.factories_info.iteritems():
 351             create_function = info.get("create_function")
 352             start_function = info.get("start_function")
 353             stop_function = info.get("stop_function")
 354             status_function = info.get("status_function")
 355             configure_function = info.get("configure_function")
 356             preconfigure_function = info.get("preconfigure_function")
 357             allow_addresses = info.get("allow_addresses", False)
 358             allow_routes = info.get("allow_routes", False)
 359             has_addresses = info.get("has_addresses", False)
 360             has_routes = info.get("has_routes", False)
 361             factory = Factory(factory_id, create_function, start_function,
 362                     stop_function, status_function,
 363                     configure_function, preconfigure_function,
 364                     allow_addresses, has_addresses,
 365                     allow_routes, has_routes)
 366
 367             # standard attributes
 368             self._add_standard_attributes(factory, info, False, True,
 369                 self.STANDARD_BOX_ATTRIBUTES)
 370
 371             # custom attributes - they override standard ones
 372             self._add_attributes(factory, info, "factory_attributes")
 373             self._add_attributes(factory, info, "box_attributes", True)
 374
 375             self._add_execute_traces(factory, info)
 376             self._add_execute_connector_types(factory, info)
 377             factories.append(factory)
 378         return factories
 379
 380     def _load_versioned_metadata_module(self):
 381         mod_name = "nepi.testbeds.%s.metadata_v%s" % (self._testbed_id.lower(),
 382                 self._version)
 383         if not mod_name in sys.modules:
 384             __import__(mod_name)
 385         return sys.modules[mod_name]
 386
 387     def _add_standard_attributes(self, factory, info, design, box, STANDARD_ATTRIBUTES):
 388         if design:
 389             attr_bundle = STANDARD_ATTRIBUTES
 390         else:
 391             # Only add non-DesignOnly attributes
 392             def nonDesign(attr_info):
 393                 return not (attr_info[1].get('flags',Attribute.NoFlags) & Attribute.DesignOnly)
 394             attr_bundle = filter(nonDesign, STANDARD_ATTRIBUTES)
 395         self._add_attributes(factory, info, None, box,
 396             attr_bundle = STANDARD_ATTRIBUTES)
 397
 398     def _add_attributes(self, factory, info, attr_key, box_attributes = False, attr_bundle = ()):
 399         if not attr_bundle and info and attr_key in info:
 400             definitions = self.STANDARD_ATTRIBUTE_BUNDLES.copy()
 401             definitions.update(self._metadata.attributes)
 402             attr_bundle = [ (attr_id, definitions[attr_id])
 403                             for attr_id in info[attr_key] ]
 404         for attr_id, attr_info in attr_bundle:
 405             name = attr_info["name"]
 406             help = attr_info["help"]
 407             type = attr_info["type"]
 408             value = attr_info["value"] if "value" in attr_info else None
 409             range = attr_info["range"] if "range" in attr_info else None
 410             allowed = attr_info["allowed"] if "allowed" in attr_info \
 411                     else None
 412             flags = attr_info["flags"] if "flags" in attr_info \
 413                     and attr_info["flags"] != None \
 414                     else Attribute.NoFlags
 415             validation_function = attr_info["validation_function"]
 416             if box_attributes:
 417                 factory.add_box_attribute(name, help, type, value, range,
 418                         allowed, flags, validation_function)
 419             else:
 420                 factory.add_attribute(name, help, type, value, range,
 421                         allowed, flags, validation_function)
 422
 423     def _add_design_traces(self, factory, info):
 424         if "traces" in info:
 425             for trace in info["traces"]:
 426                 trace_info = self._metadata.traces[trace]
 427                 trace_id = trace_info["name"]
 428                 help = trace_info["help"]
 429                 factory.add_trace(trace_id, help)
 430
 431     def _add_execute_traces(self, factory, info):
 432         if "traces" in info:
 433             for trace in info["traces"]:
 434                 trace_info = self._metadata.traces[trace]
 435                 trace_id = trace_info["name"]
 436                 factory.add_trace(trace_id)
 437
 438     def _add_design_connector_types(self, factory, info):
 439         from nepi.core.design import ConnectorType
 440         if "connector_types" in info:
 441             connections = dict()
 442             for connection in self._metadata.connections:
 443                 from_ = connection["from"]
 444                 to = connection["to"]
 445                 can_cross = connection["can_cross"]
 446                 if from_ not in connections:
 447                     connections[from_] = list()
 448                 if to not in connections:
 449                     connections[to] = list()
 450                 connections[from_].append((to, can_cross))
 451                 connections[to].append((from_, can_cross))
 452             for connector_id in info["connector_types"]:
 453                 connector_type_info = self._metadata.connector_types[
 454                         connector_id]
 455                 name = connector_type_info["name"]
 456                 help = connector_type_info["help"]
 457                 max = connector_type_info["max"]
 458                 min = connector_type_info["min"]
 459                 testbed_id = self._testbed_id
 460                 factory_id = factory.factory_id
 461                 connector_type = ConnectorType(testbed_id, factory_id, name,
 462                         help, max, min)
 463                 for (to, can_cross) in connections[(testbed_id, factory_id,
 464                         name)]:
 465                     (testbed_id_to, factory_id_to, name_to) = to
 466                     connector_type.add_allowed_connection(testbed_id_to,
 467                             factory_id_to, name_to, can_cross)
 468                 factory.add_connector_type(connector_type)
 469
 470     def _add_execute_connector_types(self, factory, info):
 471         from nepi.core.execute import ConnectorType
 472         if "connector_types" in info:
 473             from_connections = dict()
 474             to_connections = dict()
 475             for connection in self._metadata.connections:
 476                 from_ = connection["from"]
 477                 to = connection["to"]
 478                 can_cross = connection["can_cross"]
 479                 init_code = connection["init_code"] \
 480                         if "init_code" in connection else None
 481                 compl_code = connection["compl_code"] \
 482                         if "compl_code" in connection else None
 483                 if from_ not in from_connections:
 484                     from_connections[from_] = list()
 485                 if to not in to_connections:
 486                     to_connections[to] = list()
 487                 from_connections[from_].append((to, can_cross, init_code,
 488                     compl_code))
 489                 to_connections[to].append((from_, can_cross, init_code,
 490                     compl_code))
 491             for connector_id in info["connector_types"]:
 492                 connector_type_info = self._metadata.connector_types[
 493                         connector_id]
 494                 name = connector_type_info["name"]
 495                 max = connector_type_info["max"]
 496                 min = connector_type_info["min"]
 497                 testbed_id = self._testbed_id
 498                 factory_id = factory.factory_id
 499                 connector_type = ConnectorType(testbed_id, factory_id, name,
 500                         max, min)
 501                 connector_key = (testbed_id, factory_id, name)
 502                 if connector_key in to_connections:
 503                     for (from_, can_cross, init_code, compl_code) in \
 504                             to_connections[connector_key]:
 505                         (testbed_id_from, factory_id_from, name_from) = from_
 506                         connector_type.add_from_connection(testbed_id_from,
 507                                 factory_id_from, name_from, can_cross,
 508                                 init_code, compl_code)
 509                 if connector_key in from_connections:
 510                     for (to, can_cross, init_code, compl_code) in \
 511                             from_connections[(testbed_id, factory_id, name)]:
 512                         (testbed_id_to, factory_id_to, name_to) = to
 513                         connector_type.add_to_connection(testbed_id_to,
 514                                 factory_id_to, name_to, can_cross, init_code,
 515                                 compl_code)
 516                 factory.add_connector_type(connector_type)
 517