src/nepi/core/metadata.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from nepi.core.attributes import Attribute, AttributesMap
   5 import sys
   6 import getpass
   7 from nepi.util import validation
   8 from nepi.util.constants import ATTR_NEPI_TESTBED_ENVIRONMENT_SETUP, DeploymentConfiguration
   9
  10 # Attribute categories
  11 CATEGORY_DEPLOYMENT = "Deployment"
  12
  13 class VersionedMetadataInfo(object):
  14     @property
  15     def connector_types(self):
  16         """ dictionary of dictionaries with allowed connection information.
  17             connector_id: dict({
  18                 "help": help text,
  19                 "name": connector type name,
  20                 "max": maximum number of connections allowed (-1 for no limit),
  21                 "min": minimum number of connections allowed
  22             }),
  23         """
  24         raise NotImplementedError
  25
  26     @property
  27     def connections(self):
  28         """ array of dictionaries with allowed connection information.
  29         dict({
  30             "from": (testbed_id1, factory_id1, connector_type_name1),
  31             "to": (testbed_id2, factory_id2, connector_type_name2),
  32             "init_code": connection function to invoke for connection initiation
  33             "compl_code": connection function to invoke for connection
  34                 completion
  35             "can_cross": whether the connection can be done across testbed
  36                             instances
  37          }),
  38         """
  39         raise NotImplementedError
  40
  41     @property
  42     def attributes(self):
  43         """ dictionary of dictionaries of all available attributes.
  44             attribute_id: dict({
  45                 "name": attribute name,
  46                 "help": help text,
  47                 "type": attribute type,
  48                 "value": default attribute value,
  49                 "range": (maximum, minimun) values else None if not defined,
  50                 "allowed": array of posible values,
  51                 "flags": attributes flags,
  52                 "validation_function": validation function for the attribute
  53                 "category": category for the attribute
  54             })
  55         """
  56         raise NotImplementedError
  57
  58     @property
  59     def traces(self):
  60         """ dictionary of dictionaries of all available traces.
  61             trace_id: dict({
  62                 "name": trace name,
  63                 "help": help text
  64             })
  65         """
  66         raise NotImplementedError
  67
  68     @property
  69     def create_order(self):
  70         """ list of factory ids that indicates the order in which the elements
  71         should be instantiated.
  72         """
  73         raise NotImplementedError
  74
  75     @property
  76     def configure_order(self):
  77         """ list of factory ids that indicates the order in which the elements
  78         should be configured.
  79         """
  80         raise NotImplementedError
  81
  82     @property
  83     def preconfigure_order(self):
  84         """ list of factory ids that indicates the order in which the elements
  85         should be preconfigured.
  86
  87         Default: same as configure_order
  88         """
  89         return self.configure_order
  90
  91     @property
  92     def prestart_order(self):
  93         """ list of factory ids that indicates the order in which the elements
  94         should be prestart-configured.
  95
  96         Default: same as configure_order
  97         """
  98         return self.configure_order
  99
 100     @property
 101     def start_order(self):
 102         """ list of factory ids that indicates the order in which the elements
 103         should be started.
 104
 105         Default: same as configure_order
 106         """
 107         return self.configure_order
 108
 109     @property
 110     def factories_info(self):
 111         """ dictionary of dictionaries of factory specific information
 112             factory_id: dict({
 113                 "allow_addresses": whether the box allows adding IP addresses,
 114                 "allow_routes": wether the box allows adding routes,
 115                 "has_addresses": whether the box allows obtaining IP addresses,
 116                 "has_routes": wether the box allows obtaining routes,
 117                 "help": help text,
 118                 "category": category the element belongs to,
 119                 "create_function": function for element instantiation,
 120                 "start_function": function for element starting,
 121                 "stop_function": function for element stoping,
 122                 "status_function": function for retrieving element status,
 123                 "preconfigure_function": function for element preconfiguration,
 124                     (just after connections are made,
 125                     just before netrefs are resolved)
 126                 "configure_function": function for element configuration,
 127                 "prestart_function": function for pre-start
 128                     element configuration (just before starting applications),
 129                     useful for synchronization of background setup tasks or
 130                     lazy instantiation or configuration of attributes
 131                     that require connection/cross-connection state before
 132                     being created.
 133                     After this point, all applications should be able to run.
 134                 "factory_attributes": list of references to attribute_ids,
 135                 "box_attributes": list of regerences to attribute_ids,
 136                 "traces": list of references to trace_id
 137                 "tags": list of references to tag_id
 138                 "connector_types": list of references to connector_types
 139            })
 140         """
 141         raise NotImplementedError
 142
 143     @property
 144     def testbed_attributes(self):
 145         """ dictionary of attributes for testbed instance configuration
 146             attributes_id = dict({
 147                 "name": attribute name,
 148                 "help": help text,
 149                 "type": attribute type,
 150                 "value": default attribute value,
 151                 "range": (maximum, minimun) values else None if not defined,
 152                 "allowed": array of posible values,
 153                 "flags": attributes flags,
 154                 "validation_function": validation function for the attribute
 155                 "category": category for the attribute
 156              })
 157             ]
 158         """
 159         raise NotImplementedError
 160
 161 class Metadata(object):
 162     STANDARD_BOX_ATTRIBUTES = (
 163         ("label", dict(
 164             name = "label",
 165             validation_function = validation.is_string,
 166             type = Attribute.STRING,
 167             flags = Attribute.DesignOnly,
 168             help = "A unique identifier for referring to this box",
 169         )),
 170     )
 171
 172     # Shorthand for DeploymentConfiguration
 173     # Syntactic sugar to shorten stuff
 174     DC = DeploymentConfiguration
 175
 176     STANDARD_TESTBED_ATTRIBUTES = (
 177         ("home_directory", dict(
 178             name = "homeDirectory",
 179             validation_function = validation.is_string,
 180             help = "Path to the directory where traces and other files will be stored",
 181             type = Attribute.STRING,
 182             value = "",
 183             flags = Attribute.DesignOnly
 184         )),
 185         ("label", dict(
 186             name = "label",
 187             validation_function = validation.is_string,
 188             type = Attribute.STRING,
 189             flags = Attribute.DesignOnly,
 190             help = "A unique identifier for referring to this testbed",
 191         )),
 192     )
 193
 194     DEPLOYMENT_ATTRIBUTES = (
 195         # TESTBED DEPLOYMENT ATTRIBUTES
 196         (DC.DEPLOYMENT_ENVIRONMENT_SETUP, dict(
 197                 name = DC.DEPLOYMENT_ENVIRONMENT_SETUP,
 198                 validation_function = validation.is_string,
 199                 help = "Shell commands to run before spawning TestbedController processes",
 200                 type = Attribute.STRING,
 201                 flags = Attribute.DesignOnly,
 202                 category = CATEGORY_DEPLOYMENT,
 203         )),
 204         (DC.DEPLOYMENT_MODE, dict(name = DC.DEPLOYMENT_MODE,
 205                 help = "Instance execution mode",
 206                 type = Attribute.ENUM,
 207                 value = DC.MODE_SINGLE_PROCESS,
 208                 allowed = [
 209                     DC.MODE_DAEMON,
 210                     DC.MODE_SINGLE_PROCESS
 211                 ],
 212                 flags = Attribute.DesignOnly,
 213                 validation_function = validation.is_enum,
 214                 category = CATEGORY_DEPLOYMENT,
 215         )),
 216         (DC.DEPLOYMENT_COMMUNICATION, dict(name = DC.DEPLOYMENT_COMMUNICATION,
 217                 help = "Instance communication mode",
 218                 type = Attribute.ENUM,
 219                 value = DC.ACCESS_LOCAL,
 220                 allowed = [
 221                     DC.ACCESS_LOCAL,
 222                     DC.ACCESS_SSH
 223                 ],
 224                 flags = Attribute.DesignOnly,
 225                 validation_function = validation.is_enum,
 226                 category = CATEGORY_DEPLOYMENT,
 227         )),
 228         (DC.DEPLOYMENT_HOST, dict(name = DC.DEPLOYMENT_HOST,
 229                 help = "Host where the testbed will be executed",
 230                 type = Attribute.STRING,
 231                 value = "localhost",
 232                 flags = Attribute.DesignOnly,
 233                 validation_function = validation.is_string,
 234                 category = CATEGORY_DEPLOYMENT,
 235         )),
 236         (DC.DEPLOYMENT_USER, dict(name = DC.DEPLOYMENT_USER,
 237                 help = "User on the Host to execute the testbed",
 238                 type = Attribute.STRING,
 239                 value = getpass.getuser(),
 240                 flags = Attribute.DesignOnly,
 241                 validation_function = validation.is_string,
 242                 category = CATEGORY_DEPLOYMENT,
 243         )),
 244         (DC.DEPLOYMENT_KEY, dict(name = DC.DEPLOYMENT_KEY,
 245                 help = "Path to SSH key to use for connecting",
 246                 type = Attribute.STRING,
 247                 flags = Attribute.DesignOnly,
 248                 validation_function = validation.is_string,
 249                 category = CATEGORY_DEPLOYMENT,
 250         )),
 251         (DC.DEPLOYMENT_PORT, dict(name = DC.DEPLOYMENT_PORT,
 252                 help = "Port on the Host",
 253                 type = Attribute.INTEGER,
 254                 value = 22,
 255                 flags = Attribute.DesignOnly,
 256                 validation_function = validation.is_integer,
 257                 category = CATEGORY_DEPLOYMENT,
 258         )),
 259         (DC.ROOT_DIRECTORY, dict(name = DC.ROOT_DIRECTORY,
 260                 help = "Root directory for storing process files",
 261                 type = Attribute.STRING,
 262                 value = ".",
 263                 flags = Attribute.DesignOnly,
 264                 validation_function = validation.is_string, # TODO: validation.is_path
 265                 category = CATEGORY_DEPLOYMENT,
 266         )),
 267         (DC.USE_AGENT, dict(name = DC.USE_AGENT,
 268                 help = "Use -A option for forwarding of the authentication agent, if ssh access is used",
 269                 type = Attribute.BOOL,
 270                 value = False,
 271                 flags = Attribute.DesignOnly,
 272                 validation_function = validation.is_bool,
 273                 category = CATEGORY_DEPLOYMENT,
 274         )),
 275         (DC.LOG_LEVEL, dict(name = DC.LOG_LEVEL,
 276                 help = "Log level for instance",
 277                 type = Attribute.ENUM,
 278                 value = DC.ERROR_LEVEL,
 279                 allowed = [
 280                     DC.ERROR_LEVEL,
 281                     DC.DEBUG_LEVEL
 282                 ],
 283                 flags = Attribute.DesignOnly,
 284                 validation_function = validation.is_enum,
 285                 category = CATEGORY_DEPLOYMENT,
 286         )),
 287         (DC.RECOVER, dict(name = DC.RECOVER,
 288                 help = "Do not intantiate testbeds, rather, reconnect to already-running instances. Used to recover from a dead controller.",
 289                 type = Attribute.BOOL,
 290                 value = False,
 291                 flags = Attribute.DesignOnly,
 292                 validation_function = validation.is_bool,
 293                 category = CATEGORY_DEPLOYMENT,
 294         )),
 295     )
 296
 297     STANDARD_TESTBED_ATTRIBUTES += DEPLOYMENT_ATTRIBUTES
 298
 299     del DC
 300
 301
 302     STANDARD_ATTRIBUTE_BUNDLES = {
 303             "tun_proto" : dict({
 304                 "name": "tun_proto",
 305                 "help": "TUNneling protocol used",
 306                 "type": Attribute.STRING,
 307                 "flags": Attribute.Invisible,
 308                 "validation_function": validation.is_string,
 309             }),
 310             "tun_key" : dict({
 311                 "name": "tun_key",
 312                 "help": "Randomly selected TUNneling protocol cryptographic key. "
 313                         "Endpoints must agree to use the minimum (in lexicographic order) "
 314                         "of both the remote and local sides.",
 315                 "type": Attribute.STRING,
 316                 "flags": Attribute.Invisible,
 317                 "validation_function": validation.is_string,
 318             }),
 319             "tun_addr" : dict({
 320                 "name": "tun_addr",
 321                 "help": "Address (IP, unix socket, whatever) of the tunnel endpoint",
 322                 "type": Attribute.STRING,
 323                 "flags": Attribute.Invisible,
 324                 "validation_function": validation.is_string,
 325             }),
 326             "tun_port" : dict({
 327                 "name": "tun_port",
 328                 "help": "IP port of the tunnel endpoint",
 329                 "type": Attribute.INTEGER,
 330                 "flags": Attribute.Invisible,
 331                 "validation_function": validation.is_integer,
 332             }),
 333             ATTR_NEPI_TESTBED_ENVIRONMENT_SETUP : dict({
 334                 "name": ATTR_NEPI_TESTBED_ENVIRONMENT_SETUP,
 335                 "help": "Commands to set up the environment needed to run NEPI testbeds",
 336                 "type": Attribute.STRING,
 337                 "flags": Attribute.Invisible,
 338                 "validation_function": validation.is_string
 339             }),
 340     }
 341
 342
 343     def __init__(self, testbed_id, version):
 344         self._version = version
 345         self._testbed_id = testbed_id
 346         metadata_module = self._load_versioned_metadata_module()
 347         self._metadata = metadata_module.VersionedMetadataInfo()
 348
 349     @property
 350     def create_order(self):
 351         return self._metadata.create_order
 352
 353     @property
 354     def configure_order(self):
 355         return self._metadata.configure_order
 356
 357     @property
 358     def preconfigure_order(self):
 359         return self._metadata.preconfigure_order
 360
 361     @property
 362     def prestart_order(self):
 363         return self._metadata.prestart_order
 364
 365     @property
 366     def start_order(self):
 367         return self._metadata.start_order
 368
 369     def testbed_attributes(self):
 370         attributes = AttributesMap()
 371
 372         # standard attributes
 373         self._add_standard_attributes(attributes, None, True, False,
 374             self.STANDARD_TESTBED_ATTRIBUTES)
 375
 376         # custom attributes - they override standard ones
 377         for attr_info in self._metadata.testbed_attributes.values():
 378             name = attr_info["name"]
 379             help = attr_info["help"]
 380             type = attr_info["type"]
 381             value = attr_info["value"] if "value" in attr_info else None
 382             range = attr_info["range"] if "range" in attr_info else None
 383             allowed = attr_info["allowed"] if "allowed" in attr_info else None
 384             flags =  attr_info["flags"] if "flags" in attr_info \
 385                     else Attribute.NoFlags
 386             validation_function = attr_info["validation_function"]
 387             category = attr_info["category"] if "category" in attr_info else None
 388             attributes.add_attribute(name, help, type, value,
 389                     range, allowed, flags, validation_function, category)
 390
 391         return attributes
 392
 393     def build_design_factories(self):
 394         from nepi.core.design import Factory
 395         factories = list()
 396         for factory_id, info in self._metadata.factories_info.iteritems():
 397             help = info["help"]
 398             category = info["category"]
 399             allow_addresses = info.get("allow_addresses", False)
 400             allow_routes = info.get("allow_routes", False)
 401             has_addresses = info.get("has_addresses", False)
 402             has_routes = info.get("has_routes", False)
 403             factory = Factory(factory_id,
 404                     allow_addresses, has_addresses,
 405                     allow_routes, has_routes,
 406                     help, category)
 407
 408             # standard attributes
 409             self._add_standard_attributes(factory, info, True, True,
 410                 self.STANDARD_BOX_ATTRIBUTES)
 411
 412             # custom attributes - they override standard ones
 413             self._add_attributes(factory, info, "factory_attributes")
 414             self._add_attributes(factory, info, "box_attributes", True)
 415
 416             self._add_design_traces(factory, info)
 417             self._add_tags(factory, info)
 418             self._add_design_connector_types(factory, info)
 419             factories.append(factory)
 420         return factories
 421
 422     def build_execute_factories(self):
 423         from nepi.core.execute import Factory
 424         factories = list()
 425         for factory_id, info in self._metadata.factories_info.iteritems():
 426             create_function = info.get("create_function")
 427             start_function = info.get("start_function")
 428             stop_function = info.get("stop_function")
 429             status_function = info.get("status_function")
 430             configure_function = info.get("configure_function")
 431             preconfigure_function = info.get("preconfigure_function")
 432             prestart_function = info.get("prestart_function")
 433             allow_addresses = info.get("allow_addresses", False)
 434             allow_routes = info.get("allow_routes", False)
 435             has_addresses = info.get("has_addresses", False)
 436             has_routes = info.get("has_routes", False)
 437             factory = Factory(factory_id, create_function, start_function,
 438                     stop_function, status_function,
 439                     configure_function, preconfigure_function,
 440                     prestart_function,
 441                     allow_addresses, has_addresses,
 442                     allow_routes, has_routes)
 443
 444             # standard attributes
 445             self._add_standard_attributes(factory, info, False, True,
 446                 self.STANDARD_BOX_ATTRIBUTES)
 447
 448             # custom attributes - they override standard ones
 449             self._add_attributes(factory, info, "factory_attributes")
 450             self._add_attributes(factory, info, "box_attributes", True)
 451
 452             self._add_execute_traces(factory, info)
 453             self._add_tags(factory, info)
 454             self._add_execute_connector_types(factory, info)
 455             factories.append(factory)
 456         return factories
 457
 458     def _load_versioned_metadata_module(self):
 459         mod_name = "nepi.testbeds.%s.metadata_v%s" % (self._testbed_id.lower(),
 460                 self._version)
 461         if not mod_name in sys.modules:
 462             __import__(mod_name)
 463         return sys.modules[mod_name]
 464
 465     def _add_standard_attributes(self, factory, info, design, box, STANDARD_ATTRIBUTES):
 466         if design:
 467             attr_bundle = STANDARD_ATTRIBUTES
 468         else:
 469             # Only add non-DesignOnly attributes
 470             def nonDesign(attr_info):
 471                 return not (attr_info[1].get('flags',Attribute.NoFlags) & Attribute.DesignOnly)
 472             attr_bundle = filter(nonDesign, STANDARD_ATTRIBUTES)
 473         self._add_attributes(factory, info, None, box,
 474             attr_bundle = STANDARD_ATTRIBUTES)
 475
 476     def _add_attributes(self, factory, info, attr_key, box_attributes = False, attr_bundle = ()):
 477         if not attr_bundle and info and attr_key in info:
 478             definitions = self.STANDARD_ATTRIBUTE_BUNDLES.copy()
 479             definitions.update(self._metadata.attributes)
 480             attr_bundle = [ (attr_id, definitions[attr_id])
 481                             for attr_id in info[attr_key] ]
 482         for attr_id, attr_info in attr_bundle:
 483             name = attr_info["name"]
 484             help = attr_info["help"]
 485             type = attr_info["type"]
 486             value = attr_info["value"] if "value" in attr_info else None
 487             range = attr_info["range"] if "range" in attr_info else None
 488             allowed = attr_info["allowed"] if "allowed" in attr_info \
 489                     else None
 490             flags = attr_info["flags"] if "flags" in attr_info \
 491                     and attr_info["flags"] != None \
 492                     else Attribute.NoFlags
 493             validation_function = attr_info["validation_function"]
 494             category = attr_info["category"] if "category" in attr_info else None
 495             if box_attributes:
 496                 factory.add_box_attribute(name, help, type, value, range,
 497                         allowed, flags, validation_function, category)
 498             else:
 499                 factory.add_attribute(name, help, type, value, range,
 500                         allowed, flags, validation_function, category)
 501
 502     def _add_design_traces(self, factory, info):
 503         if "traces" in info:
 504             for trace in info["traces"]:
 505                 trace_info = self._metadata.traces[trace]
 506                 trace_id = trace_info["name"]
 507                 help = trace_info["help"]
 508                 factory.add_trace(trace_id, help)
 509
 510     def _add_execute_traces(self, factory, info):
 511         if "traces" in info:
 512             for trace in info["traces"]:
 513                 trace_info = self._metadata.traces[trace]
 514                 trace_id = trace_info["name"]
 515                 factory.add_trace(trace_id)
 516
 517     def _add_tags(self, factory, info):
 518         if "tags" in info:
 519             for tag_id in info["tags"]:
 520                 factory.add_tag(tag_id)
 521
 522     def _add_design_connector_types(self, factory, info):
 523         from nepi.core.design import ConnectorType
 524         if "connector_types" in info:
 525             connections = dict()
 526             for connection in self._metadata.connections:
 527                 from_ = connection["from"]
 528                 to = connection["to"]
 529                 can_cross = connection["can_cross"]
 530                 if from_ not in connections:
 531                     connections[from_] = list()
 532                 if to not in connections:
 533                     connections[to] = list()
 534                 connections[from_].append((to, can_cross))
 535                 connections[to].append((from_, can_cross))
 536             for connector_id in info["connector_types"]:
 537                 connector_type_info = self._metadata.connector_types[
 538                         connector_id]
 539                 name = connector_type_info["name"]
 540                 help = connector_type_info["help"]
 541                 max = connector_type_info["max"]
 542                 min = connector_type_info["min"]
 543                 testbed_id = self._testbed_id
 544                 factory_id = factory.factory_id
 545                 connector_type = ConnectorType(testbed_id, factory_id, name,
 546                         help, max, min)
 547                 for (to, can_cross) in connections[(testbed_id, factory_id,
 548                         name)]:
 549                     (testbed_id_to, factory_id_to, name_to) = to
 550                     connector_type.add_allowed_connection(testbed_id_to,
 551                             factory_id_to, name_to, can_cross)
 552                 factory.add_connector_type(connector_type)
 553
 554     def _add_execute_connector_types(self, factory, info):
 555         from nepi.core.execute import ConnectorType
 556         if "connector_types" in info:
 557             from_connections = dict()
 558             to_connections = dict()
 559             for connection in self._metadata.connections:
 560                 from_ = connection["from"]
 561                 to = connection["to"]
 562                 can_cross = connection["can_cross"]
 563                 init_code = connection["init_code"] \
 564                         if "init_code" in connection else None
 565                 compl_code = connection["compl_code"] \
 566                         if "compl_code" in connection else None
 567                 if from_ not in from_connections:
 568                     from_connections[from_] = list()
 569                 if to not in to_connections:
 570                     to_connections[to] = list()
 571                 from_connections[from_].append((to, can_cross, init_code,
 572                     compl_code))
 573                 to_connections[to].append((from_, can_cross, init_code,
 574                     compl_code))
 575             for connector_id in info["connector_types"]:
 576                 connector_type_info = self._metadata.connector_types[
 577                         connector_id]
 578                 name = connector_type_info["name"]
 579                 max = connector_type_info["max"]
 580                 min = connector_type_info["min"]
 581                 testbed_id = self._testbed_id
 582                 factory_id = factory.factory_id
 583                 connector_type = ConnectorType(testbed_id, factory_id, name,
 584                         max, min)
 585                 connector_key = (testbed_id, factory_id, name)
 586                 if connector_key in to_connections:
 587                     for (from_, can_cross, init_code, compl_code) in \
 588                             to_connections[connector_key]:
 589                         (testbed_id_from, factory_id_from, name_from) = from_
 590                         connector_type.add_from_connection(testbed_id_from,
 591                                 factory_id_from, name_from, can_cross,
 592                                 init_code, compl_code)
 593                 if connector_key in from_connections:
 594                     for (to, can_cross, init_code, compl_code) in \
 595                             from_connections[(testbed_id, factory_id, name)]:
 596                         (testbed_id_to, factory_id_to, name_to) = to
 597                         connector_type.add_to_connection(testbed_id_to,
 598                                 factory_id_to, name_to, can_cross, init_code,
 599                                 compl_code)
 600                 factory.add_connector_type(connector_type)
 601