src/nepi/core/metadata.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from nepi.core.attributes import Attribute, AttributesMap
   5 from nepi.core.connector import ConnectorType
   6 from nepi.core.factory import Factory
   7 import sys
   8 import getpass
   9 from nepi.util import tags, validation
  10 from nepi.util.constants import ATTR_NEPI_TESTBED_ENVIRONMENT_SETUP, \
  11         DeploymentConfiguration as DC, \
  12         AttributeCategories as AC
  13
  14 class MetadataInfo(object):
  15     @property
  16     def connector_types(self):
  17         """ dictionary of dictionaries with allowed connection information.
  18             connector_id: dict({
  19                 "help": help text,
  20                 "name": connector type name,
  21                 "max": maximum number of connections allowed (-1 for no limit),
  22                 "min": minimum number of connections allowed
  23             }),
  24         """
  25         raise NotImplementedError
  26
  27     @property
  28     def connections(self):
  29         """ array of dictionaries with allowed connection information.
  30         dict({
  31             "from": (testbed_id1, factory_id1, connector_type_name1),
  32             "to": (testbed_id2, factory_id2, connector_type_name2),
  33             "init_code": connection function to invoke for connection initiation
  34             "compl_code": connection function to invoke for connection
  35                 completion
  36             "can_cross": whether the connection can be done across testbed
  37                             instances
  38          }),
  39         """
  40         raise NotImplementedError
  41
  42     @property
  43     def attributes(self):
  44         """ dictionary of dictionaries of all available attributes.
  45             attribute_id: dict({
  46                 "name": attribute name,
  47                 "help": help text,
  48                 "type": attribute type,
  49                 "value": default attribute value,
  50                 "range": (maximum, minimun) values else None if not defined,
  51                 "allowed": array of posible values,
  52                 "flags": attributes flags,
  53                 "validation_function": validation function for the attribute
  54                 "category": category for the attribute
  55             })
  56         """
  57         raise NotImplementedError
  58
  59     @property
  60     def traces(self):
  61         """ dictionary of dictionaries of all available traces.
  62             trace_id: dict({
  63                 "name": trace name,
  64                 "help": help text
  65             })
  66         """
  67         raise NotImplementedError
  68
  69     @property
  70     def create_order(self):
  71         """ list of factory ids that indicates the order in which the elements
  72         should be instantiated.
  73         """
  74         raise NotImplementedError
  75
  76     @property
  77     def configure_order(self):
  78         """ list of factory ids that indicates the order in which the elements
  79         should be configured.
  80         """
  81         raise NotImplementedError
  82
  83     @property
  84     def preconfigure_order(self):
  85         """ list of factory ids that indicates the order in which the elements
  86         should be preconfigured.
  87
  88         Default: same as configure_order
  89         """
  90         return self.configure_order
  91
  92     @property
  93     def prestart_order(self):
  94         """ list of factory ids that indicates the order in which the elements
  95         should be prestart-configured.
  96
  97         Default: same as configure_order
  98         """
  99         return self.configure_order
 100
 101     @property
 102     def start_order(self):
 103         """ list of factory ids that indicates the order in which the elements
 104         should be started.
 105
 106         Default: same as configure_order
 107         """
 108         return self.configure_order
 109
 110     @property
 111     def factories_info(self):
 112         """ dictionary of dictionaries of factory specific information
 113             factory_id: dict({
 114                 "help": help text,
 115                 "category": category the element belongs to,
 116                 "create_function": function for element instantiation,
 117                 "start_function": function for element starting,
 118                 "stop_function": function for element stoping,
 119                 "status_function": function for retrieving element status,
 120                 "preconfigure_function": function for element preconfiguration,
 121                     (just after connections are made,
 122                     just before netrefs are resolved)
 123                 "configure_function": function for element configuration,
 124                 "prestart_function": function for pre-start
 125                     element configuration (just before starting applications),
 126                     useful for synchronization of background setup tasks or
 127                     lazy instantiation or configuration of attributes
 128                     that require connection/cross-connection state before
 129                     being created.
 130                     After this point, all applications should be able to run.
 131                 "factory_attributes": list of references to attribute_ids,
 132                 "box_attributes": list of regerences to attribute_ids,
 133                 "traces": list of references to trace_id
 134                 "tags": list of references to tag_id
 135                 "connector_types": list of references to connector_types
 136            })
 137         """
 138         raise NotImplementedError
 139
 140     @property
 141     def testbed_attributes(self):
 142         """ dictionary of attributes for testbed instance configuration
 143             attributes_id = dict({
 144                 "name": attribute name,
 145                 "help": help text,
 146                 "type": attribute type,
 147                 "value": default attribute value,
 148                 "range": (maximum, minimun) values else None if not defined,
 149                 "allowed": array of posible values,
 150                 "flags": attributes flags,
 151                 "validation_function": validation function for the attribute
 152                 "category": category for the attribute
 153              })
 154             ]
 155         """
 156         raise NotImplementedError
 157
 158     @property
 159     def testbed_id(self):
 160         """ ID for the testbed """
 161         raise NotImplementedError
 162
 163     @property
 164     def testbed_version(self):
 165         """ version for the testbed """
 166         raise NotImplementedError
 167
 168 class Metadata(object):
 169     # These attributes should be added to all boxes
 170     STANDARD_BOX_ATTRIBUTES = dict({
 171         "label" : dict({
 172             "name" : "label",
 173             "validation_function" : validation.is_string,
 174             "type" : Attribute.STRING,
 175             "flags" : Attribute.ExecReadOnly |\
 176                     Attribute.ExecImmutable |\
 177                     Attribute.Metadata,
 178             "help" : "A unique identifier for referring to this box",
 179         }),
 180      })
 181
 182     # These are the attribute definitions for tagged attributes
 183     STANDARD_TAGGED_ATTRIBUTES_DEFINITIONS = dict({
 184         "maxAddresses" : dict({
 185             "name" : "maxAddresses",
 186             "validation_function" : validation.is_integer,
 187             "type" : Attribute.INTEGER,
 188             "value" : 1,
 189             "flags" : Attribute.DesignReadOnly |\
 190                     Attribute.ExecInvisible |\
 191                     Attribute.Metadata,
 192             "help" : "The maximum allowed number of addresses",
 193             }),
 194         })
 195
 196     # Attributes to be added to all boxes with specific tags
 197     STANDARD_TAGGED_BOX_ATTRIBUTES = dict({
 198         tags.ALLOW_ADDRESSES : ["maxAddresses"],
 199         tags.HAS_ADDRESSES : ["maxAddresses"],
 200     })
 201
 202     # These attributes should be added to all testbeds
 203     STANDARD_TESTBED_ATTRIBUTES = dict({
 204         "home_directory" : dict({
 205             "name" : "homeDirectory",
 206             "validation_function" : validation.is_string,
 207             "help" : "Path to the directory where traces and other files will be stored",
 208             "type" : Attribute.STRING,
 209             "value" : "",
 210             "flags" : Attribute.ExecReadOnly |\
 211                     Attribute.ExecImmutable |\
 212                     Attribute.Metadata,
 213             }),
 214         "label" : dict({
 215             "name" : "label",
 216             "validation_function" : validation.is_string,
 217             "type" : Attribute.STRING,
 218             "flags" : Attribute.ExecReadOnly |\
 219                     Attribute.ExecImmutable |\
 220                     Attribute.Metadata,
 221             "help" : "A unique identifier for referring to this testbed",
 222             }),
 223         })
 224
 225     # These attributes should be added to all testbeds
 226     DEPLOYMENT_ATTRIBUTES = dict({
 227         # TESTBED DEPLOYMENT ATTRIBUTES
 228         DC.DEPLOYMENT_ENVIRONMENT_SETUP : dict({
 229             "name" : DC.DEPLOYMENT_ENVIRONMENT_SETUP,
 230             "validation_function" : validation.is_string,
 231             "help" : "Shell commands to run before spawning TestbedController processes",
 232             "type" : Attribute.STRING,
 233             "flags" : Attribute.ExecReadOnly |\
 234                     Attribute.ExecImmutable |\
 235                     Attribute.Metadata,
 236             "category" : AC.CATEGORY_DEPLOYMENT,
 237         }),
 238         DC.DEPLOYMENT_MODE: dict({
 239             "name" : DC.DEPLOYMENT_MODE,
 240             "help" : "Instance execution mode",
 241             "type" : Attribute.ENUM,
 242             "value" : DC.MODE_SINGLE_PROCESS,
 243             "allowed" : [
 244                     DC.MODE_DAEMON,
 245                     DC.MODE_SINGLE_PROCESS
 246                 ],
 247            "flags" : Attribute.ExecReadOnly |\
 248                     Attribute.ExecImmutable |\
 249                     Attribute.Metadata,
 250             "validation_function" : validation.is_enum,
 251             "category" : AC.CATEGORY_DEPLOYMENT,
 252             }),
 253         DC.DEPLOYMENT_COMMUNICATION : dict({
 254             "name" : DC.DEPLOYMENT_COMMUNICATION,
 255             "help" : "Instance communication mode",
 256             "type" : Attribute.ENUM,
 257             "value" : DC.ACCESS_LOCAL,
 258             "allowed" : [
 259                     DC.ACCESS_LOCAL,
 260                     DC.ACCESS_SSH
 261                 ],
 262             "flags" : Attribute.ExecReadOnly |\
 263                     Attribute.ExecImmutable |\
 264                     Attribute.Metadata,
 265             "validation_function" : validation.is_enum,
 266             "category" : AC.CATEGORY_DEPLOYMENT,
 267             }),
 268         DC.DEPLOYMENT_HOST : dict({
 269             "name" : DC.DEPLOYMENT_HOST,
 270             "help" : "Host where the testbed will be executed",
 271             "type" : Attribute.STRING,
 272             "value" : "localhost",
 273             "flags" : Attribute.ExecReadOnly |\
 274                     Attribute.ExecImmutable |\
 275                     Attribute.Metadata,
 276             "validation_function" : validation.is_string,
 277             "category" : AC.CATEGORY_DEPLOYMENT,
 278             }),
 279         DC.DEPLOYMENT_USER : dict({
 280             "name" : DC.DEPLOYMENT_USER,
 281             "help" : "User on the Host to execute the testbed",
 282             "type" : Attribute.STRING,
 283             "value" : getpass.getuser(),
 284             "flags" : Attribute.ExecReadOnly |\
 285                     Attribute.ExecImmutable |\
 286                     Attribute.Metadata,
 287             "validation_function" : validation.is_string,
 288             "category" : AC.CATEGORY_DEPLOYMENT,
 289             }),
 290         DC.DEPLOYMENT_KEY : dict({
 291             "name" : DC.DEPLOYMENT_KEY,
 292             "help" : "Path to SSH key to use for connecting",
 293             "type" : Attribute.STRING,
 294             "flags" : Attribute.ExecReadOnly |\
 295                     Attribute.ExecImmutable |\
 296                     Attribute.Metadata,
 297             "validation_function" : validation.is_string,
 298             "category" : AC.CATEGORY_DEPLOYMENT,
 299             }),
 300         DC.DEPLOYMENT_PORT : dict({
 301             "name" : DC.DEPLOYMENT_PORT,
 302             "help" : "Port on the Host",
 303             "type" : Attribute.INTEGER,
 304             "value" : 22,
 305             "flags" : Attribute.ExecReadOnly |\
 306                     Attribute.ExecImmutable |\
 307                     Attribute.Metadata,
 308             "validation_function" : validation.is_integer,
 309             "category" : AC.CATEGORY_DEPLOYMENT,
 310             }),
 311         DC.ROOT_DIRECTORY : dict({
 312             "name" : DC.ROOT_DIRECTORY,
 313             "help" : "Root directory for storing process files",
 314             "type" : Attribute.STRING,
 315             "value" : ".",
 316             "flags" : Attribute.ExecReadOnly |\
 317                     Attribute.ExecImmutable |\
 318                     Attribute.Metadata,
 319             "validation_function" : validation.is_string, # TODO: validation.is_path
 320             "category" : AC.CATEGORY_DEPLOYMENT,
 321             }),
 322         DC.USE_AGENT : dict({
 323             "name" : DC.USE_AGENT,
 324             "help" : "Use -A option for forwarding of the authentication agent, if ssh access is used",
 325             "type" : Attribute.BOOL,
 326             "value" : False,
 327             "flags" : Attribute.ExecReadOnly |\
 328                     Attribute.ExecImmutable |\
 329                     Attribute.Metadata,
 330             "validation_function" : validation.is_bool,
 331             "category" : AC.CATEGORY_DEPLOYMENT,
 332             }),
 333         DC.LOG_LEVEL : dict({
 334             "name" : DC.LOG_LEVEL,
 335             "help" : "Log level for instance",
 336             "type" : Attribute.ENUM,
 337             "value" : DC.ERROR_LEVEL,
 338             "allowed" : [
 339                     DC.ERROR_LEVEL,
 340                     DC.DEBUG_LEVEL
 341                 ],
 342             "flags" : Attribute.ExecReadOnly |\
 343                     Attribute.ExecImmutable |\
 344                     Attribute.Metadata,
 345             "validation_function" : validation.is_enum,
 346             "category" : AC.CATEGORY_DEPLOYMENT,
 347             }),
 348         DC.RECOVER : dict({
 349             "name" : DC.RECOVER,
 350             "help" : "Do not intantiate testbeds, rather, reconnect to already-running instances. Used to recover from a dead controller.",
 351             "type" : Attribute.BOOL,
 352             "value" : False,
 353             "flags" : Attribute.ExecReadOnly |\
 354                     Attribute.ExecImmutable |\
 355                     Attribute.Metadata,
 356             "validation_function" : validation.is_bool,
 357             "category" : AC.CATEGORY_DEPLOYMENT,
 358             }),
 359         })
 360
 361     # These attributes could appear in the boxes attribute list
 362     STANDARD_BOX_ATTRIBUTE_DEFINITIONS = dict({
 363         "tun_proto" : dict({
 364             "name" : "tun_proto",
 365             "help" : "TUNneling protocol used",
 366             "type" : Attribute.STRING,
 367             "flags" : Attribute.DesignInvisible | \
 368                     Attribute.ExecInvisible | \
 369                     Attribute.ExecImmutable | \
 370                     Attribute.Metadata,
 371             "validation_function" : validation.is_string,
 372             }),
 373         "tun_key" : dict({
 374             "name" : "tun_key",
 375             "help" : "Randomly selected TUNneling protocol cryptographic key. "
 376                      "Endpoints must agree to use the minimum (in lexicographic order) "
 377                      "of both the remote and local sides.",
 378             "type" : Attribute.STRING,
 379             "flags" : Attribute.DesignInvisible | \
 380                     Attribute.ExecInvisible | \
 381                     Attribute.ExecImmutable | \
 382                     Attribute.Metadata,
 383             "validation_function" : validation.is_string,
 384             }),
 385         "tun_addr" : dict({
 386             "name": "tun_addr",
 387             "help" : "Address (IP, unix socket, whatever) of the tunnel endpoint",
 388             "type" : Attribute.STRING,
 389             "flags" : Attribute.DesignInvisible | \
 390                     Attribute.ExecInvisible | \
 391                     Attribute.ExecImmutable | \
 392                     Attribute.Metadata,
 393             "validation_function" : validation.is_string,
 394             }),
 395         "tun_port" : dict({
 396             "name" : "tun_port",
 397             "help" : "IP port of the tunnel endpoint",
 398             "type" : Attribute.INTEGER,
 399             "flags" : Attribute.DesignInvisible | \
 400                     Attribute.ExecInvisible | \
 401                     Attribute.ExecImmutable | \
 402                     Attribute.Metadata,
 403             "validation_function" : validation.is_integer,
 404             }),
 405         ATTR_NEPI_TESTBED_ENVIRONMENT_SETUP : dict({
 406             "name" : ATTR_NEPI_TESTBED_ENVIRONMENT_SETUP,
 407             "help" : "Commands to set up the environment needed to run NEPI testbeds",
 408             "type" : Attribute.STRING,
 409             "flags" : Attribute.DesignInvisible | \
 410                     Attribute.ExecInvisible | \
 411                     Attribute.ExecImmutable | \
 412                     Attribute.Metadata,
 413             "validation_function" : validation.is_string
 414             }),
 415         })
 416
 417     STANDARD_TESTBED_ATTRIBUTES.update(DEPLOYMENT_ATTRIBUTES.copy())
 418
 419     def __init__(self, testbed_id):
 420         self._testbed_id = testbed_id
 421         metadata_module = self._load_metadata_module()
 422         self._metadata = metadata_module.MetadataInfo()
 423         if testbed_id != self._metadata.testbed_id:
 424             raise RuntimeError("Bad testbed id. Asked for %s, got %s" % \
 425                     (testbed_id, self._metadata.testbed_id ))
 426
 427     @property
 428     def create_order(self):
 429         return self._metadata.create_order
 430
 431     @property
 432     def configure_order(self):
 433         return self._metadata.configure_order
 434
 435     @property
 436     def preconfigure_order(self):
 437         return self._metadata.preconfigure_order
 438
 439     @property
 440     def prestart_order(self):
 441         return self._metadata.prestart_order
 442
 443     @property
 444     def start_order(self):
 445         return self._metadata.start_order
 446
 447     @property
 448     def testbed_version(self):
 449         return self._metadata.testbed_version
 450
 451     @property
 452     def testbed_id(self):
 453         return self._testbed_id
 454
 455     def testbed_attributes(self):
 456         attributes = AttributesMap()
 457         testbed_attributes = self._testbed_attributes()
 458         self._add_attributes(attributes.add_attribute, testbed_attributes)
 459         return attributes
 460
 461     def build_factories(self):
 462         factories = list()
 463         for factory_id, info in self._metadata.factories_info.iteritems():
 464             create_function = info.get("create_function")
 465             start_function = info.get("start_function")
 466             stop_function = info.get("stop_function")
 467             status_function = info.get("status_function")
 468             configure_function = info.get("configure_function")
 469             preconfigure_function = info.get("preconfigure_function")
 470             prestart_function = info.get("prestart_function")
 471             help = info["help"]
 472             category = info["category"]
 473             factory = Factory(factory_id,
 474                     create_function,
 475                     start_function,
 476                     stop_function,
 477                     status_function,
 478                     configure_function,
 479                     preconfigure_function,
 480                     prestart_function,
 481                     help,
 482                     category)
 483
 484             factory_attributes = self._factory_attributes(info)
 485             self._add_attributes(factory.add_attribute, factory_attributes)
 486             box_attributes = self._box_attributes(info)
 487             self._add_attributes(factory.add_box_attribute, box_attributes)
 488
 489             self._add_traces(factory, info)
 490             self._add_tags(factory, info)
 491             self._add_connector_types(factory, info)
 492             factories.append(factory)
 493         return factories
 494
 495     def _load_metadata_module(self):
 496         mod_name = "nepi.testbeds.%s.metadata" % (self._testbed_id.lower())
 497         if not mod_name in sys.modules:
 498             __import__(mod_name)
 499         return sys.modules[mod_name]
 500
 501     def _testbed_attributes(self):
 502         # standar attributes
 503         attributes = self.STANDARD_TESTBED_ATTRIBUTES.copy()
 504         # custom attributes
 505         attributes.update(self._metadata.testbed_attributes.copy())
 506         return attributes
 507
 508     def _factory_attributes(self, info):
 509         tagged_attributes = self._tagged_attributes(info)
 510         if "factory_attributes" in info:
 511             definitions = self._metadata.attributes.copy()
 512             # filter attributes corresponding to the factory_id
 513             factory_attributes = self._filter_attributes(info["factory_attributes"],
 514                 definitions)
 515         else:
 516             factory_attributes = dict()
 517         attributes = dict(tagged_attributes.items() + \
 518                 factory_attributes.items())
 519         return attributes
 520
 521     def _box_attributes(self, info):
 522         tagged_attributes = self._tagged_attributes(info)
 523         if "box_attributes" in info:
 524             definitions = self.STANDARD_BOX_ATTRIBUTE_DEFINITIONS.copy()
 525             definitions.update(self._metadata.attributes)
 526             box_attributes = self._filter_attributes(info["box_attributes"],
 527                 definitions)
 528         else:
 529             box_attributes = dict()
 530         attributes = dict(tagged_attributes.items() + \
 531                 box_attributes.items())
 532         attributes.update(self.STANDARD_BOX_ATTRIBUTES.copy())
 533         return attributes
 534
 535     def _tagged_attributes(self, info):
 536         tagged_attributes = dict()
 537         for tag_id in info.get("tags", []):
 538             if tag_id in self.STANDARD_TAGGED_BOX_ATTRIBUTES:
 539                 attr_list = self.STANDARD_TAGGED_BOX_ATTRIBUTES[tag_id]
 540                 attributes = self._filter_attributes(attr_list,
 541                     self.STANDARD_TAGGED_ATTRIBUTES_DEFINITIONS)
 542                 tagged_attributes.update(attributes)
 543         return tagged_attributes
 544
 545     def _filter_attributes(self, attr_list, definitions):
 546         # filter attributes not corresponding to the factory
 547         attributes = dict((attr_id, definitions[attr_id]) \
 548            for attr_id in attr_list)
 549         return attributes
 550
 551     def _add_attributes(self, add_attr_func, attributes):
 552         for attr_id, attr_info in attributes.iteritems():
 553             name = attr_info["name"]
 554             help = attr_info["help"]
 555             type = attr_info["type"]
 556             value = attr_info.get("value")
 557             range = attr_info.get("range")
 558             allowed = attr_info.get("allowed")
 559             flags = attr_info.get("flags")
 560             validation_function = attr_info["validation_function"]
 561             category = attr_info.get("category")
 562             add_attr_func(name, help, type, value, range, allowed, flags,
 563                     validation_function, category)
 564
 565     def _add_traces(self, factory, info):
 566         for trace_id in info.get("traces", []):
 567             trace_info = self._metadata.traces[trace_id]
 568             name = trace_info["name"]
 569             help = trace_info["help"]
 570             factory.add_trace(name, help)
 571
 572     def _add_tags(self, factory, info):
 573         for tag_id in info.get("tags", []):
 574             factory.add_tag(tag_id)
 575
 576     def _add_connector_types(self, factory, info):
 577         if "connector_types" in info:
 578             from_connections = dict()
 579             to_connections = dict()
 580             for connection in self._metadata.connections:
 581                 from_ = connection["from"]
 582                 to = connection["to"]
 583                 can_cross = connection["can_cross"]
 584                 init_code = connection.get("init_code")
 585                 compl_code = connection.get("compl_code")
 586                 if from_ not in from_connections:
 587                     from_connections[from_] = list()
 588                 if to not in to_connections:
 589                     to_connections[to] = list()
 590                 from_connections[from_].append((to, can_cross, init_code,
 591                     compl_code))
 592                 to_connections[to].append((from_, can_cross, init_code,
 593                     compl_code))
 594             for connector_id in info["connector_types"]:
 595                 connector_type_info = self._metadata.connector_types[
 596                         connector_id]
 597                 name = connector_type_info["name"]
 598                 help = connector_type_info["help"]
 599                 max = connector_type_info["max"]
 600                 min = connector_type_info["min"]
 601                 testbed_id = self._testbed_id
 602                 factory_id = factory.factory_id
 603                 connector_type = ConnectorType(testbed_id, factory_id, name,
 604                         help, max, min)
 605                 connector_key = (testbed_id, factory_id, name)
 606                 if connector_key in to_connections:
 607                     for (from_, can_cross, init_code, compl_code) in \
 608                             to_connections[connector_key]:
 609                         (testbed_id_from, factory_id_from, name_from) = from_
 610                         connector_type.add_from_connection(testbed_id_from,
 611                                 factory_id_from, name_from, can_cross,
 612                                 init_code, compl_code)
 613                 if connector_key in from_connections:
 614                     for (to, can_cross, init_code, compl_code) in \
 615                             from_connections[(testbed_id, factory_id, name)]:
 616                         (testbed_id_to, factory_id_to, name_to) = to
 617                         connector_type.add_to_connection(testbed_id_to,
 618                                 factory_id_to, name_to, can_cross, init_code,
 619                                 compl_code)
 620                 factory.add_connector_type(connector_type)
 621