From 0161a78a7762fc9a0056dd485be49f425d52f700 Mon Sep 17 00:00:00 2001 From: Alina Quereilhac Date: Thu, 25 Apr 2013 18:07:58 +0200 Subject: [PATCH] Removing 'waiters' from Resource (TO DISCUSS!). Proposing alterbative way for deploying ordering using more States (See test_deploy_in_order test as example) --- src/neco/__init__.py | 7 +- src/neco/execution/ec.py | 22 ++- src/neco/execution/resource.py | 232 +++++++++++------------- src/neco/resources/linux/application.py | 23 ++- src/neco/resources/linux/node.py | 1 + src/neco/util/sshfuncs.py | 20 +- test/execution/resource.py | 159 +++++++++++++++- 7 files changed, 305 insertions(+), 159 deletions(-) diff --git a/src/neco/__init__.py b/src/neco/__init__.py index f926e6de..d6f1dc03 100644 --- a/src/neco/__init__.py +++ b/src/neco/__init__.py @@ -1,5 +1,8 @@ import logging -logging.basicConfig() +import os -LOGLEVEL = logging.DEBUG +LOGLEVEL = os.environ.get("NEPI_LOGLEVEL", "DEBUG").upper() +LOGLEVEL = getattr(logging, LOGLEVEL) +FORMAT = "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" +logging.basicConfig(format = FORMAT, level = LOGLEVEL) diff --git a/src/neco/execution/ec.py b/src/neco/execution/ec.py index a7a33d83..1e55cc04 100644 --- a/src/neco/execution/ec.py +++ b/src/neco/execution/ec.py @@ -14,7 +14,7 @@ from neco.util.parallel import ParallelRun # TODO: use multiprocessing instead of threading class ExperimentController(object): - def __init__(self, root_dir = "/tmp", loglevel = 'error'): + def __init__(self, root_dir = "/tmp"): super(ExperimentController, self).__init__() # root directory to store files self._root_dir = root_dir @@ -42,7 +42,11 @@ class ExperimentController(object): # Logging self._logger = logging.getLogger("neco.execution.ec") - self._logger.setLevel(getattr(logging, loglevel.upper())) + + @property + def logger(self): + return self._logger + def get_task(self, tid): return self._tasks.get(tid) @@ -66,15 +70,15 @@ class ExperimentController(object): return guid - def create_group(self, *args): - guid = self._guid_generator.next(guid) + def register_group(self, group): + guid = self._guid_generator.next() - grp = [arg for arg in args] + if not isinstance(group, list): + group = [group] - self._resources[guid] = grp + self._groups[guid] = group return guid - def get_attributes(self, guid): rm = self.get_resource(guid) @@ -205,6 +209,8 @@ class ExperimentController(object): :type guid: int """ + self.logger.debug(" ------- DEPLOY START ------ ") + def steps(rm): rm.deploy() rm.start_with_conditions() @@ -225,7 +231,7 @@ class ExperimentController(object): towait = list(group) towait.remove(guid) self.register_condition(guid, ResourceAction.START, - towait, ResourceState.DEPLOYED) + towait, ResourceState.READY) thread = threading.Thread(target = steps, args = (rm,)) threads.append(thread) diff --git a/src/neco/execution/resource.py b/src/neco/execution/resource.py index ad9a5c56..f7010b0b 100644 --- a/src/neco/execution/resource.py +++ b/src/neco/execution/resource.py @@ -1,26 +1,26 @@ - from neco.util.timefuncs import strfnow, strfdiff, strfvalid import copy import functools import logging import weakref -import time as TIME _reschedule_delay = "1s" class ResourceAction: - DEPLOYED = 0 + DEPLOY = 0 START = 1 STOP = 2 class ResourceState: NEW = 0 - DEPLOYED = 1 - STARTED = 2 - STOPPED = 3 - FAILED = 4 - RELEASED = 5 + DISCOVERED = 1 + PROVISIONED = 2 + READY = 3 + STARTED = 4 + STOPPED = 5 + FAILED = 6 + RELEASED = 7 def clsinit(cls): cls._clsinit() @@ -32,7 +32,6 @@ class ResourceManager(object): _rtype = "Resource" _filters = None _attributes = None - _waiters = [] @classmethod def _register_filter(cls, attr): @@ -86,10 +85,6 @@ class ResourceManager(object): def rtype(cls): return cls._rtype - @classmethod - def waiters(cls): - return cls._waiters - @classmethod def get_filters(cls): """ Returns a copy of the filters @@ -118,10 +113,13 @@ class ResourceManager(object): self._start_time = None self._stop_time = None + self._discover_time = None + self._provision_time = None + self._ready_time = None + self._release_time = None # Logging - self._logger = logging.getLogger("neco.execution.resource.Resource.%s" % - self.guid) + self._logger = logging.getLogger("neco.execution.resource.Resource %s.%d " % (self._rtype, self.guid)) @property def logger(self): @@ -145,37 +143,57 @@ class ResourceManager(object): @property def start_time(self): - """ timestamp with """ + """ Returns timestamp with the time the RM started """ return self._start_time @property def stop_time(self): + """ Returns timestamp with the time the RM stopped """ return self._stop_time @property - def deploy_time(self): - return self._deploy_time + def discover_time(self): + """ Returns timestamp with the time the RM passed to state discovered """ + return self._discover_time + + @property + def provision_time(self): + """ Returns timestamp with the time the RM passed to state provisioned """ + return self._provision_time + + @property + def ready_time(self): + """ Returns timestamp with the time the RM passed to state ready """ + return self._ready_time + + @property + def release_time(self): + """ Returns timestamp with the time the RM was released """ + return self._release_time @property def state(self): return self._state def connect(self, guid): - if (self._validate_connection(guid)): + if self.valid_connection(guid): self._connections.add(guid) def discover(self, filters = None): - pass + self._discover_time = strfnow() + self._state = ResourceState.DISCOVERED def provision(self, filters = None): - pass + self._provision_time = strfnow() + self._state = ResourceState.PROVISIONED def start(self): """ Start the Resource Manager """ - if not self._state in [ResourceState.DEPLOYED, ResourceState.STOPPED]: + if not self._state in [ResourceState.READY, ResourceState.STOPPED]: self.logger.error("Wrong state %s for start" % self.state) + return self._start_time = strfnow() self._state = ResourceState.STARTED @@ -186,6 +204,7 @@ class ResourceManager(object): """ if not self._state in [ResourceState.STARTED]: self.logger.error("Wrong state %s for stop" % self.state) + return self._stop_time = strfnow() self._state = ResourceState.STOPPED @@ -214,36 +233,41 @@ class ResourceManager(object): def register_condition(self, action, group, state, time = None): - """ Do the 'action' after 'time' on the current RM when 'group' - reach the state 'state' + """ Registers a condition on the resource manager to allow execution + of 'action' only after 'time' has elapsed from the moment all resources + in 'group' reached state 'state' - :param action: Action to do. Either 'START' or 'STOP' + :param action: Action to restrict to condition (either 'START' or 'STOP') :type action: str - :param group: group of RM - :type group: str - :param state: RM that are part of the condition - :type state: list - :param time: Time to wait after the state is reached (ex : '2s' ) + :param group: Group of RMs to wait for (list of guids) + :type group: int or list of int + :param state: State to wait for on all RM in group. (either 'STARTED' or 'STOPPED') + :type state: str + :param time: Time to wait after 'state' is reached on all RMs in group. (e.g. '2s') :type time: str """ - if action not in self.conditions: - self._conditions[action] = set() + conditions = self.conditions.get(action) + if not conditions: + conditions = list() + self._conditions[action] = conditions - # We need to use only sequence inside a set and not a list. - # As group is a list, we need to change it. - #print (tuple(group), state, time) - self.conditions.get(action).add((tuple(group), state, time)) + # For each condition to register a tuple of (group, state, time) is + # added to the 'action' list + if not isinstance(group, list): + group = [group] + + conditions.append((group, state, time)) def _needs_reschedule(self, group, state, time): """ Internal method that verify if 'time' has elapsed since all elements in 'group' have reached state 'state'. - :param group: RM that are part of the condition - :type group: list - :param state: State that group need to reach for the condtion + :param group: Group of RMs to wait for (list of guids) + :type group: int or list of int + :param state: State to wait for on all RM in group. (either 'STARTED' or 'STOPPED') :type state: str - :param time: time to wait after the state + :param time: Time to wait after 'state' is reached on all RMs in group. (e.g. '2s') :type time: str .. note : time should be written like "2s" or "3m" with s for seconds, m for minutes, h for hours, ... @@ -257,15 +281,21 @@ class ResourceManager(object): # check state and time elapsed on all RMs for guid in group: rm = self.ec.get_resource(guid) - # If the RMs is lower than the requested state we must - # reschedule (e.g. if RM is DEPLOYED but we required STARTED) + # If the RM state is lower than the requested state we must + # reschedule (e.g. if RM is READY but we required STARTED) if rm.state < state: reschedule = True break + # If there is a time restriction, we must verify the + # restriction is satisfied if time: - if state == ResourceState.DEPLOYED: - t = rm.deploy_time + if state == ResourceState.DISCOVERED: + t = rm.discover_time + if state == ResourceState.PROVISIONED: + t = rm.provision_time + elif state == ResourceState.READY: + t = rm.ready_time elif state == ResourceState.STARTED: t = rm.start_time elif state == ResourceState.STOPPED: @@ -275,7 +305,6 @@ class ResourceManager(object): break d = strfdiff(strfnow(), t) - #print "This is the value of d : " + str(d) + " // With the value of t : " + str(t) + " // With the value of time : " + str(time) wait = strfdiff(strfvalid(time),strfvalid(str(d)+"s")) if wait > 0.001: reschedule = True @@ -286,17 +315,17 @@ class ResourceManager(object): def set_with_conditions(self, name, value, group, state, time): """ Set value 'value' on attribute with name 'name' when 'time' has elapsed since all elements in 'group' have reached state - 'state'. + 'state' - :param name: Name of the attribute + :param name: Name of the attribute to set :type name: str - :param name: Value of the attribute + :param name: Value of the attribute to set :type name: str - :param group: RM that are part of the condition - :type group: list - :param state: State that group need to reach before set + :param group: Group of RMs to wait for (list of guids) + :type group: int or list of int + :param state: State to wait for on all RM in group. (either 'STARTED', 'STOPPED' or 'READY') :type state: str - :param time: Time to wait after the state is reached (ex : '2s' ) + :param time: Time to wait after 'state' is reached on all RMs in group. (e.g. '2s') :type time: str """ @@ -320,7 +349,8 @@ class ResourceManager(object): self.set(name, value) def start_with_conditions(self): - """ Starts when all the conditions are reached + """ Starts RM when all the conditions in self.conditions for + action 'START' are satisfied. """ reschedule = False @@ -328,30 +358,29 @@ class ResourceManager(object): ## evaluate if set conditions are met - # only can start when RM is either STOPPED or DEPLOYED - if self.state not in [ResourceState.STOPPED, ResourceState.DEPLOYED]: + # only can start when RM is either STOPPED or READY + if self.state not in [ResourceState.STOPPED, ResourceState.READY]: reschedule = True else: - print TIME.strftime("%H:%M:%S", TIME.localtime()) + " RM : " + self._rtype + " (Guid : "+ str(self.guid) +") ----- start condition : " + str(self.conditions.items()) - # Need to separate because it could have more that tuple of condition - # for the same action. - conditions_start = self.conditions.get(ResourceAction.START, []) - for (group, state, time) in conditions_start: + self.logger.debug("---- START CONDITIONS ---- %s" % + self.conditions.get(ResourceAction.START)) + + # Verify all start conditions are met + start_conditions = self.conditions.get(ResourceAction.START, []) + for (group, state, time) in start_conditions: reschedule, delay = self._needs_reschedule(group, state, time) if reschedule: break if reschedule: - callback = functools.partial(self.start_with_conditions) - self.ec.schedule(delay, callback) + self.ec.schedule(delay, self.start_with_conditions) else: - print TIME.strftime("%H:%M:%S", TIME.localtime()) + " RM : " + self._rtype + " (Guid : "+ str(self.guid) +") ----\ -------------------------------------------------------------------------------\ ----------------------------------------------------------------- STARTING -- " + self.logger.debug("----- STARTING ---- ") self.start() def stop_with_conditions(self): - """ Stop when all the conditions are reached + """ Stops RM when all the conditions in self.conditions for + action 'STOP' are satisfied. """ reschedule = False @@ -363,84 +392,41 @@ class ResourceManager(object): if self.state != ResourceState.STARTED: reschedule = True else: - print TIME.strftime("%H:%M:%S", TIME.localtime()) + " RM : " + self._rtype + "\ - (Guid : "+ str(self.guid) +") ---- stop condition : " + str(self.conditions.items()) - conditions_stop = self.conditions.get(ResourceAction.STOP, []) - for (group, state, time) in conditions_stop: + self.logger.debug(" ---- STOP CONDITIONS ---- %s" % + self.conditions.get(ResourceAction.STOP)) + + stop_conditions = self.conditions.get(ResourceAction.STOP, []) + for (group, state, time) in stop_conditions: reschedule, delay = self._needs_reschedule(group, state, time) if reschedule: break + if reschedule: callback = functools.partial(self.stop_with_conditions) self.ec.schedule(delay, callback) else: - print TIME.strftime("%H:%M:%S", TIME.localtime()) + " RM : " + self._rtype + " (Guid : "+ str(self.guid) +") ----\ -------------------------------------------------------------------------------\ ----------------------------------------------------------------- STOPPING -- " + self.logger.debug(" ----- STOPPING ---- ") self.stop() def deploy(self): - """Execute all the differents steps required to reach the state DEPLOYED + """ Execute all steps required for the RM to reach the state READY """ - self.deploy_restriction() - self.discover() - self.provision() - self.deploy_with_conditions() - - def deploy_restriction(self): - dep = set() - for guid in self.connections: - if self.ec.get_resource(guid).rtype() in self.__class__._waiters: - dep.add(guid) - self.register_condition(ResourceAction.DEPLOYED, dep, ResourceState.DEPLOYED) - - - def deploy_with_conditions(self): - """ Starts when all the conditions are reached - - """ - reschedule = False - delay = _reschedule_delay - - ## evaluate if set conditions are met - - # only can deploy when RM is NEW - if not self._state in [ResourceState.NEW]: - self.logger.error("Wrong state %s for stop" % self.state) + if self._state > ResourceState.READY: + self.logger.error("Wrong state %s for deploy" % self.state) return - else: - print TIME.strftime("%H:%M:%S", TIME.localtime()) + " RM : " + self._rtype + " (Guid : "+ str(self.guid) +") ----- deploy condition : " + str(self.conditions.items()) - # Need to separate because it could have more that tuple of condition - # for the same action. - conditions_deployed = self.conditions.get(ResourceAction.DEPLOYED, []) - for (group, state, time) in conditions_deployed: - reschedule, delay = self._needs_reschedule(group, state, time) - if reschedule: - break - - if reschedule: - callback = functools.partial(self.deploy_with_conditions) - self.ec.schedule(delay, callback) - else: - print TIME.strftime("%H:%M:%S", TIME.localtime()) + " RM : " + self._rtype + " (Guid : "+ str(self.guid) +") ----\ -------------------------------------------------------------------------------\ ----------------------------------------------------------------- DEPLOY -- " - self.deploy_action() - - def deploy_action(self): - - self._deploy_time = strfnow() - self._state = ResourceState.DEPLOYED + self._ready_time = strfnow() + self._state = ResourceState.READY def release(self): """Clean the resource at the end of the Experiment and change the status """ + self._release_time = strfnow() self._state = ResourceState.RELEASED - def _validate_connection(self, guid): + def valid_connection(self, guid): """Check if the connection is available. :param guid: Guid of the current Resource Manager diff --git a/src/neco/resources/linux/application.py b/src/neco/resources/linux/application.py index 0c2f0ec1..befb22d2 100644 --- a/src/neco/resources/linux/application.py +++ b/src/neco/resources/linux/application.py @@ -12,6 +12,8 @@ class LinuxApplication(ResourceManager): def _register_attributes(cls): command = Attribute("command", "Command to execute", flags = Flags.ReadOnly) + forward_x11 = Attribute("forwardX11", " Enables X11 forwarding for SSH connections", + flags = Flags.ReadOnly) env = Attribute("env", "Environment variables string for command execution", flags = Flags.ReadOnly) sudo = Attribute("sudo", "Run with root privileges", @@ -54,6 +56,7 @@ class LinuxApplication(ResourceManager): releasing the resource", flags = Flags.ReadOnly) cls._register_attribute(command) + cls._register_attribute(forward_x11) cls._register_attribute(env) cls._register_attribute(sudo) cls._register_attribute(depends) @@ -69,22 +72,18 @@ class LinuxApplication(ResourceManager): super(LinuxApplication, self).__init__(ec, guid) self._pid = None self._ppid = None - self._home = "${HOME}/app-%s" % self.box.guid + self._home = "app-%s" % self.box.guid self._node = None self._logger = logging.getLogger("neco.linux.Application.%d" % guid) - @property - def api(self): - return self.node.api - @property def node(self): self._node @property def home(self): - return self._home + return self._home # + node home @property def pid(self): @@ -95,10 +94,16 @@ class LinuxApplication(ResourceManager): return self._ppid def provision(self, filters = None): - # clean home - # upload + # verify home hash or clean home + # upload sources # build # Install stuff!! + # upload app command + pass + + def deploy(self): + # Wait until node is associated and deployed + self.provision() pass def start(self): @@ -151,5 +156,3 @@ class LinuxApplication(ResourceManager): self._node = resources[0] if len(resources) == 1 else None return self._node - - diff --git a/src/neco/resources/linux/node.py b/src/neco/resources/linux/node.py index 56eb6e2a..b482747b 100644 --- a/src/neco/resources/linux/node.py +++ b/src/neco/resources/linux/node.py @@ -354,6 +354,7 @@ class LinuxNode(ResourceManager): os.path.join(home, stdout)) return (out, err), proc + def is_alive(self): if self.localhost: return True diff --git a/src/neco/util/sshfuncs.py b/src/neco/util/sshfuncs.py index e81c558d..0589fb8b 100644 --- a/src/neco/util/sshfuncs.py +++ b/src/neco/util/sshfuncs.py @@ -1,18 +1,18 @@ import base64 import errno +import hashlib +import logging import os import os.path +import re import select import signal import socket import subprocess import time -import traceback -import re import tempfile -import hashlib -TRACE = os.environ.get("NEPI_TRACE", "false").lower() in ("true", "1", "on") +logger = logging.getLogger("neco.execution.utils.sshfuncs") if hasattr(os, "devnull"): DEV_NULL = os.devnull @@ -239,8 +239,8 @@ def rexec(command, host, user, try: out, err = _communicate(proc, stdin, timeout, err_on_timeout) - if TRACE: - print "COMMAND host %s, command %s, out %s, error %s" % (host, " ".join(args), out, err) + logger.debug("COMMAND host %s, command %s, out %s, error %s" % ( + host, " ".join(args), out, err)) if proc.poll(): if err.strip().startswith('ssh: ') or err.strip().startswith('mux_client_hello_exchange: '): @@ -251,9 +251,8 @@ def rexec(command, host, user, continue break except RuntimeError, e: - if TRACE: - print "EXCEPTION host %s, command %s, out %s, error %s, exception TIMEOUT -> %s" % ( - host, " ".join(args), out, err, e.args) + logger.debug("EXCEPTION host %s, command %s, out %s, error %s, exception TIMEOUT -> %s" % ( + host, " ".join(args), out, err, e.args)) if retry <= 0: raise @@ -285,8 +284,7 @@ def rcopy(source, dest, in which case it is advised that the destination be a folder. """ - if TRACE: - print "scp", source, dest + logger.debug("SCP %s %s" % (source, dest)) if isinstance(source, file) and source.tell() == 0: source = source.name diff --git a/test/execution/resource.py b/test/execution/resource.py index 87e03384..d90b1379 100755 --- a/test/execution/resource.py +++ b/test/execution/resource.py @@ -1,7 +1,9 @@ #!/usr/bin/env python -from neco.execution.resource import ResourceManager, ResourceFactory, clsinit from neco.execution.attribute import Attribute +from neco.execution.ec import ExperimentController +from neco.execution.resource import ResourceManager, ResourceState, clsinit +import time import unittest @clsinit @@ -23,11 +25,10 @@ class AnotherResource(ResourceManager): def __init__(self, ec, guid): super(AnotherResource, self).__init__(ec, guid) -class EC(object): - pass - class ResourceFactoryTestCase(unittest.TestCase): def test_add_resource_factory(self): + from neco.execution.resource import ResourceFactory + ResourceFactory.register_type(MyResource) ResourceFactory.register_type(AnotherResource) @@ -42,15 +43,163 @@ class ResourceFactoryTestCase(unittest.TestCase): self.assertEquals(len(ResourceFactory.resource_types()), 2) -# TODO:!!! +def get_connected(connections, rtype, ec): + connected = [] + for guid in connections: + rm = ec.get_resource(guid) + if rm.rtype() == rtype: + connected.append(rm) + return connected + +class Channel(ResourceManager): + _rtype = "Channel" + + def __init__(self, ec, guid): + super(Channel, self).__init__(ec, guid) + + def deploy(self): + time.sleep(1) + super(Channel, self).deploy() + self.logger.debug(" -------- DEPLOYED ------- ") + +class Interface(ResourceManager): + _rtype = "Interface" + + def __init__(self, ec, guid): + super(Interface, self).__init__(ec, guid) + + def deploy(self): + node = get_connected(self.connections, Node.rtype(), self.ec)[0] + chan = get_connected(self.connections, Channel.rtype(), self.ec)[0] + + if node.state < ResourceState.PROVISIONED: + self.ec.schedule("0.5s", self.deploy) + elif chan.state < ResourceState.READY: + self.ec.schedule("0.5s", self.deploy) + else: + time.sleep(2) + super(Interface, self).deploy() + self.logger.debug(" -------- DEPLOYED ------- ") + +class Node(ResourceManager): + _rtype = "Node" + + def __init__(self, ec, guid): + super(Node, self).__init__(ec, guid) + + def deploy(self): + if self.state == ResourceState.NEW: + self.discover() + self.provision() + self.logger.debug(" -------- PROVISIONED ------- ") + self.ec.schedule("3s", self.deploy) + elif self.state == ResourceState.PROVISIONED: + ifaces = get_connected(self.connections, Interface.rtype(), self.ec) + for rm in ifaces: + if rm.state < ResourceState.READY: + self.ec.schedule("0.5s", self.deploy) + return + + super(Node, self).deploy() + self.logger.debug(" -------- DEPLOYED ------- ") + +class Application(ResourceManager): + _rtype = "Application" + + def __init__(self, ec, guid): + super(Application, self).__init__(ec, guid) + + def deploy(self): + node = get_connected(self.connections, Node.rtype(), self.ec)[0] + if node.state < ResourceState.READY: + self.ec.schedule("0.5s", self.deploy) + else: + super(Application, self).deploy() + self.logger.debug(" -------- DEPLOYED ------- ") + class ResourceManagerTestCase(unittest.TestCase): + def test_deploy_in_order(self): + """ + Test scenario: 2 applications running one on 1 node each. + Nodes are connected to Interfaces which are connected + through a channel between them. + + - Application needs to wait until Node is ready to be ready + - Node needs to wait until Interface is ready to be ready + - Interface needs to wait until Node is provisioned to be ready + - Interface needs to wait until Channel is ready to be ready + - The channel doesn't wait for any other resource to be ready + + """ + from neco.execution.resource import ResourceFactory + + ResourceFactory.register_type(Application) + ResourceFactory.register_type(Node) + ResourceFactory.register_type(Interface) + ResourceFactory.register_type(Channel) + + ec = ExperimentController() + + app1 = ec.register_resource("Application") + app2 = ec.register_resource("Application") + node1 = ec.register_resource("Node") + node2 = ec.register_resource("Node") + iface1 = ec.register_resource("Interface") + iface2 = ec.register_resource("Interface") + chan = ec.register_resource("Channel") + + ec.register_connection(app1, node1) + ec.register_connection(app2, node2) + ec.register_connection(iface1, node1) + ec.register_connection(iface2, node2) + ec.register_connection(iface1, chan) + ec.register_connection(iface2, chan) + + try: + ec.deploy() + + while not all([ ec.state(guid) == ResourceState.STARTED \ + for guid in [app1, app2, node1, node2, iface1, iface2, chan]]): + time.sleep(0.5) + + finally: + ec.shutdown() + + rmapp1 = ec.get_resource(app1) + rmapp2 = ec.get_resource(app2) + rmnode1 = ec.get_resource(node1) + rmnode2 = ec.get_resource(node2) + rmiface1 = ec.get_resource(iface1) + rmiface2 = ec.get_resource(iface2) + rmchan = ec.get_resource(chan) + + ## Validate deploy order + # - Application needs to wait until Node is ready to be ready + self.assertTrue(rmnode1.ready_time < rmapp1.ready_time) + self.assertTrue(rmnode2.ready_time < rmapp2.ready_time) + + # - Node needs to wait until Interface is ready to be ready + self.assertTrue(rmnode1.ready_time > rmiface1.ready_time) + self.assertTrue(rmnode2.ready_time > rmiface2.ready_time) + + # - Interface needs to wait until Node is provisioned to be ready + self.assertTrue(rmnode1.provision_time < rmiface1.ready_time) + self.assertTrue(rmnode2.provision_time < rmiface2.ready_time) + + # - Interface needs to wait until Channel is ready to be ready + self.assertTrue(rmchan.ready_time < rmiface1.ready_time) + self.assertTrue(rmchan.ready_time < rmiface2.ready_time) + def test_start_with_condition(self): + # TODO!!! pass def test_stop_with_condition(self): + # TODO!!! pass def test_set_with_condition(self): + # TODO!!! pass -- 2.43.0