src/nepi/core/metadata.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from nepi.core.attributes import Attribute, AttributesMap
   5 from nepi.core.connector import ConnectorType
   6 from nepi.core.factory import Factory
   7 import sys
   8 import getpass
   9 from nepi.util import validation
  10 from nepi.util.constants import ATTR_NEPI_TESTBED_ENVIRONMENT_SETUP, \
  11         DeploymentConfiguration as DC, \
  12         AttributeCategories as AC
  13
  14 class VersionedMetadataInfo(object):
  15     @property
  16     def connector_types(self):
  17         """ dictionary of dictionaries with allowed connection information.
  18             connector_id: dict({
  19                 "help": help text,
  20                 "name": connector type name,
  21                 "max": maximum number of connections allowed (-1 for no limit),
  22                 "min": minimum number of connections allowed
  23             }),
  24         """
  25         raise NotImplementedError
  26
  27     @property
  28     def connections(self):
  29         """ array of dictionaries with allowed connection information.
  30         dict({
  31             "from": (testbed_id1, factory_id1, connector_type_name1),
  32             "to": (testbed_id2, factory_id2, connector_type_name2),
  33             "init_code": connection function to invoke for connection initiation
  34             "compl_code": connection function to invoke for connection
  35                 completion
  36             "can_cross": whether the connection can be done across testbed
  37                             instances
  38          }),
  39         """
  40         raise NotImplementedError
  41
  42     @property
  43     def attributes(self):
  44         """ dictionary of dictionaries of all available attributes.
  45             attribute_id: dict({
  46                 "name": attribute name,
  47                 "help": help text,
  48                 "type": attribute type,
  49                 "value": default attribute value,
  50                 "range": (maximum, minimun) values else None if not defined,
  51                 "allowed": array of posible values,
  52                 "flags": attributes flags,
  53                 "validation_function": validation function for the attribute
  54                 "category": category for the attribute
  55             })
  56         """
  57         raise NotImplementedError
  58
  59     @property
  60     def traces(self):
  61         """ dictionary of dictionaries of all available traces.
  62             trace_id: dict({
  63                 "name": trace name,
  64                 "help": help text
  65             })
  66         """
  67         raise NotImplementedError
  68
  69     @property
  70     def create_order(self):
  71         """ list of factory ids that indicates the order in which the elements
  72         should be instantiated.
  73         """
  74         raise NotImplementedError
  75
  76     @property
  77     def configure_order(self):
  78         """ list of factory ids that indicates the order in which the elements
  79         should be configured.
  80         """
  81         raise NotImplementedError
  82
  83     @property
  84     def preconfigure_order(self):
  85         """ list of factory ids that indicates the order in which the elements
  86         should be preconfigured.
  87
  88         Default: same as configure_order
  89         """
  90         return self.configure_order
  91
  92     @property
  93     def prestart_order(self):
  94         """ list of factory ids that indicates the order in which the elements
  95         should be prestart-configured.
  96
  97         Default: same as configure_order
  98         """
  99         return self.configure_order
 100
 101     @property
 102     def start_order(self):
 103         """ list of factory ids that indicates the order in which the elements
 104         should be started.
 105
 106         Default: same as configure_order
 107         """
 108         return self.configure_order
 109
 110     @property
 111     def factories_info(self):
 112         """ dictionary of dictionaries of factory specific information
 113             factory_id: dict({
 114                 "allow_addresses": whether the box allows adding IP addresses,
 115                 "allow_routes": wether the box allows adding routes,
 116                 "has_addresses": whether the box allows obtaining IP addresses,
 117                 "has_routes": wether the box allows obtaining routes,
 118                 "help": help text,
 119                 "category": category the element belongs to,
 120                 "create_function": function for element instantiation,
 121                 "start_function": function for element starting,
 122                 "stop_function": function for element stoping,
 123                 "status_function": function for retrieving element status,
 124                 "preconfigure_function": function for element preconfiguration,
 125                     (just after connections are made,
 126                     just before netrefs are resolved)
 127                 "configure_function": function for element configuration,
 128                 "prestart_function": function for pre-start
 129                     element configuration (just before starting applications),
 130                     useful for synchronization of background setup tasks or
 131                     lazy instantiation or configuration of attributes
 132                     that require connection/cross-connection state before
 133                     being created.
 134                     After this point, all applications should be able to run.
 135                 "factory_attributes": list of references to attribute_ids,
 136                 "box_attributes": list of regerences to attribute_ids,
 137                 "traces": list of references to trace_id
 138                 "tags": list of references to tag_id
 139                 "connector_types": list of references to connector_types
 140            })
 141         """
 142         raise NotImplementedError
 143
 144     @property
 145     def testbed_attributes(self):
 146         """ dictionary of attributes for testbed instance configuration
 147             attributes_id = dict({
 148                 "name": attribute name,
 149                 "help": help text,
 150                 "type": attribute type,
 151                 "value": default attribute value,
 152                 "range": (maximum, minimun) values else None if not defined,
 153                 "allowed": array of posible values,
 154                 "flags": attributes flags,
 155                 "validation_function": validation function for the attribute
 156                 "category": category for the attribute
 157              })
 158             ]
 159         """
 160         raise NotImplementedError
 161
 162 class Metadata(object):
 163     # These attributes should be added to all boxes
 164     STANDARD_BOX_ATTRIBUTES = dict({
 165         "label" : dict({
 166             "name" : "label",
 167             "validation_function" : validation.is_string,
 168             "type" : Attribute.STRING,
 169             "flags" : Attribute.DesignOnly,
 170             "help" : "A unique identifier for referring to this box",
 171         }),
 172      })
 173
 174     # These attributes should be added to all testbeds
 175     STANDARD_TESTBED_ATTRIBUTES = dict({
 176         "home_directory" : dict({
 177             "name" : "homeDirectory",
 178             "validation_function" : validation.is_string,
 179             "help" : "Path to the directory where traces and other files will be stored",
 180             "type" : Attribute.STRING,
 181             "value" : "",
 182             "flags" : Attribute.DesignOnly
 183             }),
 184         "label" : dict({
 185             "name" : "label",
 186             "validation_function" : validation.is_string,
 187             "type" : Attribute.STRING,
 188             "flags" : Attribute.DesignOnly,
 189             "help" : "A unique identifier for referring to this testbed",
 190             }),
 191         })
 192
 193     # These attributes should be added to all testbeds
 194     DEPLOYMENT_ATTRIBUTES = dict({
 195         # TESTBED DEPLOYMENT ATTRIBUTES
 196         DC.DEPLOYMENT_ENVIRONMENT_SETUP : dict({
 197             "name" : DC.DEPLOYMENT_ENVIRONMENT_SETUP,
 198             "validation_function" : validation.is_string,
 199             "help" : "Shell commands to run before spawning TestbedController processes",
 200             "type" : Attribute.STRING,
 201             "flags" : Attribute.DesignOnly,
 202             "category" : AC.CATEGORY_DEPLOYMENT,
 203         }),
 204         DC.DEPLOYMENT_MODE: dict({
 205             "name" : DC.DEPLOYMENT_MODE,
 206             "help" : "Instance execution mode",
 207             "type" : Attribute.ENUM,
 208             "value" : DC.MODE_SINGLE_PROCESS,
 209             "allowed" : [
 210                     DC.MODE_DAEMON,
 211                     DC.MODE_SINGLE_PROCESS
 212                 ],
 213             "flags" : Attribute.DesignOnly,
 214             "validation_function" : validation.is_enum,
 215             "category" : AC.CATEGORY_DEPLOYMENT,
 216             }),
 217         DC.DEPLOYMENT_COMMUNICATION : dict({
 218             "name" : DC.DEPLOYMENT_COMMUNICATION,
 219             "help" : "Instance communication mode",
 220             "type" : Attribute.ENUM,
 221             "value" : DC.ACCESS_LOCAL,
 222             "allowed" : [
 223                     DC.ACCESS_LOCAL,
 224                     DC.ACCESS_SSH
 225                 ],
 226             "flags" : Attribute.DesignOnly,
 227             "validation_function" : validation.is_enum,
 228             "category" : AC.CATEGORY_DEPLOYMENT,
 229             }),
 230         DC.DEPLOYMENT_HOST : dict({
 231             "name" : DC.DEPLOYMENT_HOST,
 232             "help" : "Host where the testbed will be executed",
 233             "type" : Attribute.STRING,
 234             "value" : "localhost",
 235             "flags" : Attribute.DesignOnly,
 236             "validation_function" : validation.is_string,
 237             "category" : AC.CATEGORY_DEPLOYMENT,
 238             }),
 239         DC.DEPLOYMENT_USER : dict({
 240             "name" : DC.DEPLOYMENT_USER,
 241             "help" : "User on the Host to execute the testbed",
 242             "type" : Attribute.STRING,
 243             "value" : getpass.getuser(),
 244             "flags" : Attribute.DesignOnly,
 245             "validation_function" : validation.is_string,
 246             "category" : AC.CATEGORY_DEPLOYMENT,
 247             }),
 248         DC.DEPLOYMENT_KEY : dict({
 249             "name" : DC.DEPLOYMENT_KEY,
 250             "help" : "Path to SSH key to use for connecting",
 251             "type" : Attribute.STRING,
 252             "flags" : Attribute.DesignOnly,
 253             "validation_function" : validation.is_string,
 254             "category" : AC.CATEGORY_DEPLOYMENT,
 255             }),
 256         DC.DEPLOYMENT_PORT : dict({
 257             "name" : DC.DEPLOYMENT_PORT,
 258             "help" : "Port on the Host",
 259             "type" : Attribute.INTEGER,
 260             "value" : 22,
 261             "flags" : Attribute.DesignOnly,
 262             "validation_function" : validation.is_integer,
 263             "category" : AC.CATEGORY_DEPLOYMENT,
 264             }),
 265         DC.ROOT_DIRECTORY : dict({
 266             "name" : DC.ROOT_DIRECTORY,
 267             "help" : "Root directory for storing process files",
 268             "type" : Attribute.STRING,
 269             "value" : ".",
 270             "flags" : Attribute.DesignOnly,
 271             "validation_function" : validation.is_string, # TODO: validation.is_path
 272             "category" : AC.CATEGORY_DEPLOYMENT,
 273             }),
 274         DC.USE_AGENT : dict({
 275             "name" : DC.USE_AGENT,
 276             "help" : "Use -A option for forwarding of the authentication agent, if ssh access is used",
 277             "type" : Attribute.BOOL,
 278             "value" : False,
 279             "flags" : Attribute.DesignOnly,
 280             "validation_function" : validation.is_bool,
 281             "category" : AC.CATEGORY_DEPLOYMENT,
 282             }),
 283         DC.LOG_LEVEL : dict({
 284             "name" : DC.LOG_LEVEL,
 285             "help" : "Log level for instance",
 286             "type" : Attribute.ENUM,
 287             "value" : DC.ERROR_LEVEL,
 288             "allowed" : [
 289                     DC.ERROR_LEVEL,
 290                     DC.DEBUG_LEVEL
 291                 ],
 292             "flags" : Attribute.DesignOnly,
 293             "validation_function" : validation.is_enum,
 294             "category" : AC.CATEGORY_DEPLOYMENT,
 295             }),
 296         DC.RECOVER : dict({
 297             "name" : DC.RECOVER,
 298             "help" : "Do not intantiate testbeds, rather, reconnect to already-running instances. Used to recover from a dead controller.",
 299             "type" : Attribute.BOOL,
 300             "value" : False,
 301             "flags" : Attribute.DesignOnly,
 302             "validation_function" : validation.is_bool,
 303             "category" : AC.CATEGORY_DEPLOYMENT,
 304             }),
 305         })
 306
 307     # These attributes could appear in the boxes attribute list
 308     STANDARD_BOX_ATTRIBUTE_DEFINITIONS = dict({
 309         "tun_proto" : dict({
 310             "name" : "tun_proto",
 311             "help" : "TUNneling protocol used",
 312             "type" : Attribute.STRING,
 313             "flags" : Attribute.Invisible,
 314             "validation_function" : validation.is_string,
 315             }),
 316         "tun_key" : dict({
 317             "name" : "tun_key",
 318             "help" : "Randomly selected TUNneling protocol cryptographic key. "
 319                      "Endpoints must agree to use the minimum (in lexicographic order) "
 320                      "of both the remote and local sides.",
 321             "type" : Attribute.STRING,
 322             "flags" :Attribute.Invisible,
 323             "validation_function" : validation.is_string,
 324             }),
 325         "tun_addr" : dict({
 326             "name": "tun_addr",
 327             "help" : "Address (IP, unix socket, whatever) of the tunnel endpoint",
 328             "type" : Attribute.STRING,
 329             "flags" : Attribute.Invisible,
 330             "validation_function" : validation.is_string,
 331             }),
 332         "tun_port" : dict({
 333             "name" : "tun_port",
 334             "help" : "IP port of the tunnel endpoint",
 335             "type" : Attribute.INTEGER,
 336             "flags" : Attribute.Invisible,
 337             "validation_function" : validation.is_integer,
 338             }),
 339         ATTR_NEPI_TESTBED_ENVIRONMENT_SETUP : dict({
 340             "name" : ATTR_NEPI_TESTBED_ENVIRONMENT_SETUP,
 341             "help" : "Commands to set up the environment needed to run NEPI testbeds",
 342             "type" : Attribute.STRING,
 343             "flags" : Attribute.Invisible,
 344             "validation_function" : validation.is_string
 345             }),
 346         })
 347
 348     STANDARD_TESTBED_ATTRIBUTES.update(DEPLOYMENT_ATTRIBUTES.copy())
 349
 350     def __init__(self, testbed_id, version):
 351         self._version = version
 352         self._testbed_id = testbed_id
 353         metadata_module = self._load_versioned_metadata_module()
 354         self._metadata = metadata_module.VersionedMetadataInfo()
 355
 356     @property
 357     def create_order(self):
 358         return self._metadata.create_order
 359
 360     @property
 361     def configure_order(self):
 362         return self._metadata.configure_order
 363
 364     @property
 365     def preconfigure_order(self):
 366         return self._metadata.preconfigure_order
 367
 368     @property
 369     def prestart_order(self):
 370         return self._metadata.prestart_order
 371
 372     @property
 373     def start_order(self):
 374         return self._metadata.start_order
 375
 376     def testbed_attributes(self):
 377         attributes = AttributesMap()
 378         testbed_attributes = self._testbed_attributes()
 379         self._add_attributes(attributes.add_attribute, testbed_attributes)
 380         return attributes
 381
 382     def build_factories(self):
 383         factories = list()
 384         for factory_id, info in self._metadata.factories_info.iteritems():
 385             create_function = info.get("create_function")
 386             start_function = info.get("start_function")
 387             stop_function = info.get("stop_function")
 388             status_function = info.get("status_function")
 389             configure_function = info.get("configure_function")
 390             preconfigure_function = info.get("preconfigure_function")
 391             prestart_function = info.get("prestart_function")
 392             allow_addresses = info.get("allow_addresses", False)
 393             allow_routes = info.get("allow_routes", False)
 394             has_addresses = info.get("has_addresses", False)
 395             has_routes = info.get("has_routes", False)
 396             help = info["help"]
 397             category = info["category"]
 398             factory = Factory(factory_id,
 399                     create_function,
 400                     start_function,
 401                     stop_function,
 402                     status_function,
 403                     configure_function,
 404                     preconfigure_function,
 405                     prestart_function,
 406                     help,
 407                     category,
 408                     allow_addresses,
 409                     has_addresses,
 410                     allow_routes,
 411                     has_routes)
 412
 413             factory_attributes = self._factory_attributes(factory_id, info)
 414             self._add_attributes(factory.add_attribute, factory_attributes)
 415             box_attributes = self._box_attributes(factory_id, info)
 416             self._add_attributes(factory.add_box_attribute, box_attributes)
 417
 418             self._add_traces(factory, info)
 419             self._add_tags(factory, info)
 420             self._add_connector_types(factory, info)
 421             factories.append(factory)
 422         return factories
 423
 424     def _load_versioned_metadata_module(self):
 425         mod_name = "nepi.testbeds.%s.metadata_v%s" % (self._testbed_id.lower(),
 426                 self._version)
 427         if not mod_name in sys.modules:
 428             __import__(mod_name)
 429         return sys.modules[mod_name]
 430
 431     def _testbed_attributes(self):
 432         # standar attributes
 433         attributes = self.STANDARD_TESTBED_ATTRIBUTES.copy()
 434         # custom attributes
 435         attributes.update(self._metadata.testbed_attributes.copy())
 436         return attributes
 437
 438     def _factory_attributes(self, factory_id, info):
 439         if "factory_attributes" not in info:
 440             return dict()
 441         definitions = self._metadata.attributes.copy()
 442         # filter attributes corresponding to the factory_id
 443         return self._filter_attributes(info["factory_attributes"],
 444                 definitions)
 445
 446     def _box_attributes(self, factory_id, info):
 447         if "box_attributes" in info:
 448             definitions = self.STANDARD_BOX_ATTRIBUTE_DEFINITIONS.copy()
 449             definitions.update(self._metadata.attributes)
 450             attributes = self._filter_attributes(info["box_attributes"],
 451                 definitions)
 452         else:
 453             attributes = dict()
 454         attributes.update(self.STANDARD_BOX_ATTRIBUTES.copy())
 455         return attributes
 456
 457     def _filter_attributes(self, attr_list, definitions):
 458         # filter attributes corresponding to the factory_id
 459         attributes = dict((attr_id, definitions[attr_id]) \
 460            for attr_id in attr_list)
 461         return attributes
 462
 463     def _add_attributes(self, add_attr_func, attributes):
 464         for attr_id, attr_info in attributes.iteritems():
 465             name = attr_info["name"]
 466             help = attr_info["help"]
 467             type = attr_info["type"]
 468             value = attr_info["value"] if "value" in attr_info else None
 469             range = attr_info["range"] if "range" in attr_info else None
 470             allowed = attr_info["allowed"] if "allowed" in attr_info \
 471                     else None
 472             flags = attr_info["flags"] if "flags" in attr_info \
 473                     and attr_info["flags"] != None \
 474                     else Attribute.NoFlags
 475             validation_function = attr_info["validation_function"]
 476             category = attr_info["category"] if "category" in attr_info else None
 477             add_attr_func(name, help, type, value, range, allowed, flags,
 478                     validation_function, category)
 479
 480     def _add_traces(self, factory, info):
 481         if "traces" in info:
 482             for trace_id in info["traces"]:
 483                 trace_info = self._metadata.traces[trace_id]
 484                 name = trace_info["name"]
 485                 help = trace_info["help"]
 486                 factory.add_trace(name, help)
 487
 488     def _add_tags(self, factory, info):
 489         if "tags" in info:
 490             for tag_id in info["tags"]:
 491                 factory.add_tag(tag_id)
 492
 493     def _add_connector_types(self, factory, info):
 494         if "connector_types" in info:
 495             from_connections = dict()
 496             to_connections = dict()
 497             for connection in self._metadata.connections:
 498                 from_ = connection["from"]
 499                 to = connection["to"]
 500                 can_cross = connection["can_cross"]
 501                 init_code = connection["init_code"] \
 502                         if "init_code" in connection else None
 503                 compl_code = connection["compl_code"] \
 504                         if "compl_code" in connection else None
 505                 if from_ not in from_connections:
 506                     from_connections[from_] = list()
 507                 if to not in to_connections:
 508                     to_connections[to] = list()
 509                 from_connections[from_].append((to, can_cross, init_code,
 510                     compl_code))
 511                 to_connections[to].append((from_, can_cross, init_code,
 512                     compl_code))
 513             for connector_id in info["connector_types"]:
 514                 connector_type_info = self._metadata.connector_types[
 515                         connector_id]
 516                 name = connector_type_info["name"]
 517                 help = connector_type_info["help"]
 518                 max = connector_type_info["max"]
 519                 min = connector_type_info["min"]
 520                 testbed_id = self._testbed_id
 521                 factory_id = factory.factory_id
 522                 connector_type = ConnectorType(testbed_id, factory_id, name,
 523                         help, max, min)
 524                 connector_key = (testbed_id, factory_id, name)
 525                 if connector_key in to_connections:
 526                     for (from_, can_cross, init_code, compl_code) in \
 527                             to_connections[connector_key]:
 528                         (testbed_id_from, factory_id_from, name_from) = from_
 529                         connector_type.add_from_connection(testbed_id_from,
 530                                 factory_id_from, name_from, can_cross,
 531                                 init_code, compl_code)
 532                 if connector_key in from_connections:
 533                     for (to, can_cross, init_code, compl_code) in \
 534                             from_connections[(testbed_id, factory_id, name)]:
 535                         (testbed_id_to, factory_id_to, name_to) = to
 536                         connector_type.add_to_connection(testbed_id_to,
 537                                 factory_id_to, name_to, can_cross, init_code,
 538                                 compl_code)
 539                 factory.add_connector_type(connector_type)
 540