From 46da4690c5120b820385fd47b7fbf995c1bff2a0 Mon Sep 17 00:00:00 2001 From: Alina Quereilhac Date: Thu, 4 Jul 2013 19:11:31 -0700 Subject: [PATCH] Porting changes in LinuxApplication directory structure to CCNx RMs --- .../linux/ccn/ccncat_extended_ring_topo.py | 2 +- src/nepi/resources/linux/application.py | 17 +++-- .../resources/linux/ccn/ccnapplication.py | 31 +++++++--- src/nepi/resources/linux/ccn/ccncontent.py | 24 ++++++- src/nepi/resources/linux/ccn/ccnd.py | 62 +++++++++++++------ src/nepi/resources/linux/ccn/ccnr.py | 27 +++++--- src/nepi/resources/linux/ccn/fibentry.py | 34 ++++++++-- 7 files changed, 148 insertions(+), 49 deletions(-) diff --git a/examples/linux/ccn/ccncat_extended_ring_topo.py b/examples/linux/ccn/ccncat_extended_ring_topo.py index e91351a9..21815634 100755 --- a/examples/linux/ccn/ccncat_extended_ring_topo.py +++ b/examples/linux/ccn/ccncat_extended_ring_topo.py @@ -54,7 +54,7 @@ def add_node(ec, host, user, ssh_key = None): ec.set(node, "hostname", host) ec.set(node, "username", user) ec.set(node, "identity", ssh_key) - #ec.set(node, "cleanHome", True) + ec.set(node, "cleanHome", True) ec.set(node, "cleanProcesses", True) return node diff --git a/src/nepi/resources/linux/application.py b/src/nepi/resources/linux/application.py index d0a8c32b..1ecabb89 100644 --- a/src/nepi/resources/linux/application.py +++ b/src/nepi/resources/linux/application.py @@ -32,6 +32,8 @@ import subprocess # TODO: During provisioning, everything that is not scp could be # uploaded to a same script, http_sources download, etc... # and like that require performing less ssh connections!!! +# TODO: Make stdin be a symlink to the original file in ${SHARE} +# - later use md5sum to check wether the file needs to be re-upload @clsinit @@ -298,6 +300,13 @@ class LinuxApplication(ResourceManager): step() + self.upload_start_command() + + self.info("Provisioning finished") + + super(LinuxApplication, self).provision() + + def upload_start_command(self): # Upload command to remote bash script # - only if command can be executed in background and detached command = self.get("command") @@ -312,15 +321,11 @@ class LinuxApplication(ResourceManager): env = self.get("env") env = env and self.replace_paths(env) - shfile = os.path.join(self.app_home, "app.sh") + shfile = os.path.join(self.app_home, "start.sh") self.node.upload_command(command, shfile = shfile, env = env) - - self.info("Provisioning finished") - - super(LinuxApplication, self).provision() def upload_sources(self): sources = self.get("sources") @@ -542,7 +547,7 @@ class LinuxApplication(ResourceManager): # The command to run was previously uploaded to a bash script # during deployment, now we launch the remote script using 'run' # method from the node. - cmd = "bash %s" % os.path.join(self.app_home, "app.sh") + cmd = "bash %s" % os.path.join(self.app_home, "start.sh") (out, err), proc = self.node.run(cmd, self.run_home, stdin = stdin, stdout = stdout, diff --git a/src/nepi/resources/linux/ccn/ccnapplication.py b/src/nepi/resources/linux/ccn/ccnapplication.py index 39ac2a92..0b0dd6ff 100644 --- a/src/nepi/resources/linux/ccn/ccnapplication.py +++ b/src/nepi/resources/linux/ccn/ccnapplication.py @@ -17,10 +17,11 @@ # # Author: Alina Quereilhac -from nepi.execution.resource import clsinit_copy, ResourceState, \ - ResourceAction +from nepi.execution.resource import ResourceManager, clsinit_copy, ResourceState, \ + reschedule_delay from nepi.resources.linux.application import LinuxApplication from nepi.resources.linux.ccn.ccnd import LinuxCCND +from nepi.util.timefuncs import tnow, tdiffsec import os @@ -44,15 +45,31 @@ class LinuxCCNApplication(LinuxApplication): return None def deploy(self): - if not self.get("env"): - self.set("env", self._environment) + if not self.ccnd or self.ccnd.state < ResourceState.READY: + self.debug("---- RESCHEDULING DEPLOY ---- node state %s " % self.node.state ) + self.ec.schedule(reschedule_delay, self.deploy) + else: + try: + command = self.get("command") or "" - super(LinuxCCNApplication, self).deploy() + self.info("Deploying command '%s' " % command) + + if not self.get("env"): + self.set("env", self._environment) + + self.discover() + self.provision() + except: + self._state = ResourceState.FAILED + raise + + self.debug("----- READY ---- ") + self._ready_time = tnow() + self._state = ResourceState.READY @property def _environment(self): - env = "PATH=$PATH:${STORE}/ccnx/bin " - return env + return self.ccnd.path def execute_command(self, command, env): environ = self.node.format_environment(env, inline = True) diff --git a/src/nepi/resources/linux/ccn/ccncontent.py b/src/nepi/resources/linux/ccn/ccncontent.py index 8b799743..d4e9f39c 100644 --- a/src/nepi/resources/linux/ccn/ccncontent.py +++ b/src/nepi/resources/linux/ccn/ccncontent.py @@ -20,14 +20,14 @@ from nepi.execution.attribute import Attribute, Flags, Types from nepi.execution.resource import clsinit_copy, ResourceState, \ ResourceAction, reschedule_delay -from nepi.resources.linux.ccn.ccnapplication import LinuxCCNApplication +from nepi.resources.linux.application import LinuxApplication from nepi.resources.linux.ccn.ccnr import LinuxCCNR from nepi.util.timefuncs import tnow import os @clsinit_copy -class LinuxCCNContent(LinuxCCNApplication): +class LinuxCCNContent(LinuxApplication): _rtype = "LinuxCCNContent" @classmethod @@ -53,11 +53,17 @@ class LinuxCCNContent(LinuxCCNApplication): if ccnr: return ccnr[0] return None + @property + def ccnd(self): + if self.ccnr: return self.ccnr.ccnd + return None + @property def node(self): if self.ccnr: return self.ccnr.node return None + def deploy(self): if not self.ccnr or self.ccnr.state < ResourceState.READY: self.debug("---- RESCHEDULING DEPLOY ---- node state %s " % self.node.state ) @@ -77,7 +83,8 @@ class LinuxCCNContent(LinuxCCNApplication): self.info("Deploying command '%s' " % command) - self.node.mkdir(self.app_home) + # create run dir for application + self.node.mkdir(self.run_home) # upload content self.upload_stdin() @@ -121,6 +128,17 @@ class LinuxCCNContent(LinuxCCNApplication): return "ccnseqwriter -r %s < %s" % (self.get("contentName"), os.path.join(self.app_home, 'stdin')) + @property + def _environment(self): + return self.ccnd.path + + def execute_command(self, command, env): + environ = self.node.format_environment(env, inline = True) + command = environ + command + command = self.replace_paths(command) + + return self.node.execute(command) + def valid_connection(self, guid): # TODO: Validate! return True diff --git a/src/nepi/resources/linux/ccn/ccnd.py b/src/nepi/resources/linux/ccn/ccnd.py index 7c150eaf..cc30823d 100644 --- a/src/nepi/resources/linux/ccn/ccnd.py +++ b/src/nepi/resources/linux/ccn/ccnd.py @@ -126,8 +126,15 @@ class LinuxCCND(LinuxApplication): def __init__(self, ec, guid): super(LinuxCCND, self).__init__(ec, guid) self._home = "ccnd-%s" % self.guid - self._version = None - self._environment = None + self._version = "ccnx" + + @property + def version(self): + return self._version + + @property + def path(self): + return "PATH=$PATH:${BIN}/%s/" % self.version def deploy(self): if not self.node or self.node.state < ResourceState.READY: @@ -145,6 +152,14 @@ class LinuxCCND(LinuxApplication): if not self.get("sources"): self.set("sources", self._sources) + sources = self.get("sources") + source = sources.split(" ")[0] + basename = os.path.basename(source) + self._version = ( basename.strip().replace(".tar.gz", "") + .replace(".tar","") + .replace(".gz","") + .replace(".zip","") ) + if not self.get("build"): self.set("build", self._build) @@ -160,7 +175,7 @@ class LinuxCCND(LinuxApplication): self.info("Deploying command '%s' " % command) # create home dir for application - self.node.mkdir(self.app_home) + self.node.mkdir(self.run_home) # upload sources self.upload_sources() @@ -188,9 +203,11 @@ class LinuxCCND(LinuxApplication): env = self.replace_paths(env) command = self.replace_paths(command) - self.node.run_and_wait(command, self.app_home, + shfile = os.path.join(self.app_home, "start.sh") + self.node.run_and_wait(command, self.run_home, + shfile = shfile, + overwrite = False, env = env, - shfile = "app.sh", raise_on_error = True) self.debug("----- READY ---- ") @@ -225,8 +242,10 @@ class LinuxCCND(LinuxApplication): env = env and self.replace_paths(env) # Upload the command to a file, and execute asynchronously - self.node.run_and_wait(command, self.app_home, - shfile = "ccndstop.sh", + shfile = os.path.join(self.app_home, "stop.sh") + self.node.run_and_wait(command, self.run_home, + shfile = shfile, + overwrite = False, env = env, pidfile = "ccndstop_pidfile", ecodefile = "ccndstop_exitcode", @@ -296,32 +315,35 @@ class LinuxCCND(LinuxApplication): return ( # Evaluate if ccnx binaries are already installed " ( " - " test -f ${STORE}/ccnx/bin/ccnd && " - " echo 'sources found, nothing to do' " + " test -f ${BIN}/%(version)s/ccnd && " + " echo 'binaries found, nothing to do' " " ) || ( " # If not, untar and build " ( " - " mkdir -p ${STORE}/ccnx && " - " tar xf ${STORE}/%(sources)s --strip-components=1 -C ${STORE}/ccnx " + " mkdir -p ${SRC}/%(version)s && " + " tar xf ${SRC}/%(sources)s --strip-components=1 -C ${SRC}/%(version)s " " ) && " - "cd ${STORE}/ccnx && " + "cd ${SRC}/%(version)s && " # Just execute and silence warnings... " ( ./configure && make ) " - " )") % ({ 'sources': sources }) + " )") % ({ 'sources': sources, + 'version': self.version + }) @property def _install(self): return ( # Evaluate if ccnx binaries are already installed " ( " - " test -f ${SOURCES}/ccnx/bin/ccnd && " - " echo 'sources found, nothing to do' " + " test -f ${BIN}/%(version)s/ccnd && " + " echo 'binaries found, nothing to do' " " ) || ( " # If not, install - " mkdir -p ${SOURCES}/ccnx/bin && " - " cp -r ${}/ccnx ${STORE}" + " mkdir -p ${BIN}/%(version)s && " + " mv ${SRC}/%(version)s/bin/* ${BIN}/%(version)s/ " " )" - ) + ) % ({ 'version': self.version + }) @property def _environment(self): @@ -341,12 +363,12 @@ class LinuxCCND(LinuxApplication): "prefix" : "CCND_PREFIX", }) - env = "PATH=$PATH:${SOURCES}/ccnx/bin " + env = self.path env += " ".join(map(lambda k: "%s=%s" % (envs.get(k), str(self.get(k))) \ if self.get(k) else "", envs.keys())) return env - + def valid_connection(self, guid): # TODO: Validate! return True diff --git a/src/nepi/resources/linux/ccn/ccnr.py b/src/nepi/resources/linux/ccn/ccnr.py index 920b3f97..b03dbed4 100644 --- a/src/nepi/resources/linux/ccn/ccnr.py +++ b/src/nepi/resources/linux/ccn/ccnr.py @@ -21,14 +21,14 @@ from nepi.execution.attribute import Attribute, Flags, Types from nepi.execution.trace import Trace, TraceAttr from nepi.execution.resource import clsinit_copy, ResourceState, \ ResourceAction, reschedule_delay -from nepi.resources.linux.ccn.ccnapplication import LinuxCCNApplication +from nepi.resources.linux.application import LinuxApplication from nepi.resources.linux.ccn.ccnd import LinuxCCND from nepi.util.timefuncs import tnow import os @clsinit_copy -class LinuxCCNR(LinuxCCNApplication): +class LinuxCCNR(LinuxApplication): _rtype = "LinuxCCNR" @classmethod @@ -183,6 +183,17 @@ class LinuxCCNR(LinuxCCNApplication): super(LinuxCCNR, self).__init__(ec, guid) self._home = "ccnr-%s" % self.guid + @property + def ccnd(self): + ccnd = self.get_connected(LinuxCCND.rtype()) + if ccnd: return ccnd[0] + return None + + @property + def node(self): + if self.ccnd: return self.ccnd.node + return None + def deploy(self): if not self.ccnd or self.ccnd.state < ResourceState.READY: self.debug("---- RESCHEDULING DEPLOY ---- CCND state %s " % self.ccnd.state ) @@ -198,7 +209,7 @@ class LinuxCCNR(LinuxCCNApplication): self.info("Deploying command '%s' " % command) - self.node.mkdir(self.app_home) + self.node.mkdir(self.run_home) # upload sources self.upload_sources() @@ -211,11 +222,13 @@ class LinuxCCNR(LinuxCCNApplication): env = self.replace_paths(env) command = self.replace_paths(command) - self.node.run_and_wait(command, self.app_home, + shfile = os.path.join(self.app_home, "start.sh") + self.node.run_and_wait(command, self.run_home, + shfile = shfile, + overwrite = False, env = env, - shfile = "app.sh", raise_on_error = True) - + self.debug("----- READY ---- ") self._ready_time = tnow() self._state = ResourceState.READY @@ -268,7 +281,7 @@ class LinuxCCNR(LinuxCCNApplication): "ccnsSyncScope": "CCNS_SYNC_SCOPE", }) - env = "PATH=$PATH:${STORE}/ccnx/bin " + env = self.ccnd.path env += " ".join(map(lambda k: "%s=%s" % (envs.get(k), self.get(k)) \ if self.get(k) else "", envs.keys())) diff --git a/src/nepi/resources/linux/ccn/fibentry.py b/src/nepi/resources/linux/ccn/fibentry.py index 9f4d1d35..2af3b2cf 100644 --- a/src/nepi/resources/linux/ccn/fibentry.py +++ b/src/nepi/resources/linux/ccn/fibentry.py @@ -21,7 +21,8 @@ from nepi.execution.attribute import Attribute, Flags, Types from nepi.execution.trace import Trace, TraceAttr from nepi.execution.resource import clsinit_copy, ResourceState, \ ResourceAction, reschedule_delay -from nepi.resources.linux.ccn.ccnapplication import LinuxCCNApplication +from nepi.resources.linux.application import LinuxApplication +from nepi.resources.linux.ccn.ccnd import LinuxCCND from nepi.util.timefuncs import tnow import os @@ -31,7 +32,7 @@ import os # Implement ENTRY DELETE!! @clsinit_copy -class LinuxFIBEntry(LinuxCCNApplication): +class LinuxFIBEntry(LinuxApplication): _rtype = "LinuxFIBEntry" @classmethod @@ -67,6 +68,17 @@ class LinuxFIBEntry(LinuxCCNApplication): super(LinuxFIBEntry, self).__init__(ec, guid) self._home = "fib-%s" % self.guid + @property + def ccnd(self): + ccnd = self.get_connected(LinuxCCND.rtype()) + if ccnd: return ccnd[0] + return None + + @property + def node(self): + if self.ccnd: return self.ccnd.node + return None + def deploy(self): # Wait until associated ccnd is provisioned if not self.ccnd or self.ccnd.state < ResourceState.READY: @@ -80,8 +92,10 @@ class LinuxFIBEntry(LinuxCCNApplication): self.set("env", env) self.info("Deploying command '%s' " % command) - - self.node.mkdir(self.app_home) + + # create run dir for application + self.node.mkdir(self.run_home) + (out, err), proc = self.execute_command(command, env) if proc.poll(): @@ -90,7 +104,6 @@ class LinuxFIBEntry(LinuxCCNApplication): self.error(msg, out, err) raise RuntimeError, msg - self.debug("----- READY ---- ") self._ready_time = tnow() self._state = ResourceState.READY @@ -158,6 +171,17 @@ class LinuxFIBEntry(LinuxCCNApplication): "port": port }) + @property + def _environment(self): + return self.ccnd.path + + def execute_command(self, command, env): + environ = self.node.format_environment(env, inline = True) + command = environ + command + command = self.replace_paths(command) + + return self.node.execute(command) + def valid_connection(self, guid): # TODO: Validate! return True -- 2.43.0