From: Alina Quereilhac Date: Sat, 6 Jul 2013 08:17:16 +0000 (-0700) Subject: LinuxApplication: making single deploy.sh script out of http_sources.sh, build.sh... X-Git-Tag: nepi-3.0.0~85 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=9bc3d34df11fdaf2138df236ff7ea842cda06215;p=nepi.git LinuxApplication: making single deploy.sh script out of http_sources.sh, build.sh and install.sh --- diff --git a/src/nepi/resources/linux/application.py b/src/nepi/resources/linux/application.py index 1ecabb89..506285ab 100644 --- a/src/nepi/resources/linux/application.py +++ b/src/nepi/resources/linux/application.py @@ -271,7 +271,8 @@ class LinuxApplication(ResourceManager): def provision(self): # create run dir for application self.node.mkdir(self.run_home) - + + # List of all the provision methods to invoke steps = [ # upload sources self.upload_sources, @@ -292,14 +293,23 @@ class LinuxApplication(ResourceManager): # Install self.install] + command = [] + # Since provisioning takes a long time, before # each step we check that the EC is still for step in steps: if self.ec.finished: raise RuntimeError, "EC finished" + + ret = step() + if ret: + command.append(ret) - step() + # upload deploy script + deploy_command = ";".join(command) + self.execute_deploy_command(deploy_command) + # upload start script self.upload_start_command() self.info("Provisioning finished") @@ -327,8 +337,24 @@ class LinuxApplication(ResourceManager): shfile = shfile, env = env) + def execute_deploy_command(self, command): + if command: + # Upload the command to a bash script and run it + # in background ( but wait until the command has + # finished to continue ) + shfile = os.path.join(self.app_home, "deploy.sh") + self.node.run_and_wait(command, self.run_home, + shfile = shfile, + overwrite = False, + pidfile = "deploy_pidfile", + ecodefile = "deploy_exitcode", + stdout = "deploy_stdout", + stderr = "deploy_stderr") + def upload_sources(self): sources = self.get("sources") + + command = "" if sources: self.info("Uploading sources ") @@ -357,27 +383,17 @@ class LinuxApplication(ResourceManager): "source": source }) - if command: - command = " && ".join(command) - - # replace application specific paths in the command - command = self.replace_paths(command) - - # Upload the command to a bash script and run it - # in background ( but wait until the command has - # finished to continue ) - self.node.run_and_wait(command, self.run_home, - shfile = os.path.join(self.app_home, "http_sources.sh"), - overwrite = False, - pidfile = "http_sources_pidfile", - ecodefile = "http_sources_exitcode", - stdout = "http_sources_stdout", - stderr = "http_sources_stderr") + command = " && ".join(command) + # replace application specific paths in the command + command = self.replace_paths(command) + if sources: sources = ' '.join(sources) self.node.upload(sources, self.node.src_dir, overwrite = False) + return command + def upload_files(self): files = self.get("files") @@ -430,19 +446,8 @@ class LinuxApplication(ResourceManager): self.info("Building sources ") # replace application specific paths in the command - command = self.replace_paths(build) + return self.replace_paths(build) - # Upload the command to a bash script and run it - # in background ( but wait until the command has - # finished to continue ) - self.node.run_and_wait(command, self.run_home, - shfile = os.path.join(self.app_home, "build.sh"), - overwrite = False, - pidfile = "build_pidfile", - ecodefile = "build_exitcode", - stdout = "build_stdout", - stderr = "build_stderr") - def install(self): install = self.get("install") @@ -450,18 +455,7 @@ class LinuxApplication(ResourceManager): self.info("Installing sources ") # replace application specific paths in the command - command = self.replace_paths(install) - - # Upload the command to a bash script and run it - # in background ( but wait until the command has - # finished to continue ) - self.node.run_and_wait(command, self.run_home, - shfile = os.path.join(self.app_home, "install.sh"), - overwrite = False, - pidfile = "install_pidfile", - ecodefile = "install_exitcode", - stdout = "install_stdout", - stderr = "install_stderr") + return self.replace_paths(install) def deploy(self): # Wait until node is associated and deployed @@ -476,7 +470,7 @@ class LinuxApplication(ResourceManager): self.discover() self.provision() except: - self._state = ResourceState.FAILED + self.fail() raise super(LinuxApplication, self).deploy() @@ -528,7 +522,7 @@ class LinuxApplication(ResourceManager): blocking = False) if self._proc.poll(): - self._state = ResourceState.FAILED + self.fail() self.error(msg, out, err) raise RuntimeError, msg @@ -558,7 +552,7 @@ class LinuxApplication(ResourceManager): msg = " Failed to start command '%s' " % command if proc.poll(): - self._state = ResourceState.FAILED + self.fail() self.error(msg, out, err) raise RuntimeError, msg @@ -575,7 +569,7 @@ class LinuxApplication(ResourceManager): # Out is what was written in the stderr file if err: - self._state = ResourceState.FAILED + self.fail() msg = " Failed to start command '%s' " % command self.error(msg, out, err) raise RuntimeError, msg @@ -607,7 +601,7 @@ class LinuxApplication(ResourceManager): # check if execution errors occurred msg = " Failed to STOP command '%s' " % self.get("command") self.error(msg, out, err) - self._state = ResourceState.FAILED + self.fail() stopped = False if stopped: @@ -643,7 +637,7 @@ class LinuxApplication(ResourceManager): msg = " Failed to execute command '%s'" % self.get("command") err = self._proc.stderr.read() self.error(msg, out, err) - self._state = ResourceState.FAILED + self.fail() elif retcode == 0: self._state = ResourceState.FINISHED @@ -660,7 +654,7 @@ class LinuxApplication(ResourceManager): if err: msg = " Failed to execute command '%s'" % self.get("command") self.error(msg, out, err) - self._state = ResourceState.FAILED + self.fail() elif self.pid and self.ppid: # No execution errors occurred. Make sure the background diff --git a/src/nepi/resources/linux/ccn/ccnapplication.py b/src/nepi/resources/linux/ccn/ccnapplication.py index 0b0dd6ff..740d3669 100644 --- a/src/nepi/resources/linux/ccn/ccnapplication.py +++ b/src/nepi/resources/linux/ccn/ccnapplication.py @@ -60,7 +60,7 @@ class LinuxCCNApplication(LinuxApplication): self.discover() self.provision() except: - self._state = ResourceState.FAILED + self.fail() raise self.debug("----- READY ---- ") diff --git a/src/nepi/resources/linux/ccn/ccncontent.py b/src/nepi/resources/linux/ccn/ccncontent.py index d4e9f39c..5222d034 100644 --- a/src/nepi/resources/linux/ccn/ccncontent.py +++ b/src/nepi/resources/linux/ccn/ccncontent.py @@ -63,7 +63,6 @@ class LinuxCCNContent(LinuxApplication): if self.ccnr: return self.ccnr.node return None - def deploy(self): if not self.ccnr or self.ccnr.state < ResourceState.READY: self.debug("---- RESCHEDULING DEPLOY ---- node state %s " % self.node.state ) @@ -71,41 +70,52 @@ class LinuxCCNContent(LinuxApplication): # ccnr needs to wait until ccnd is deployed and running self.ec.schedule(reschedule_delay, self.deploy) else: - command = self._start_command - env = self._environment + try: + if not self.get("command"): + self.set("command", self._start_command) + + if not self.get("env"): + self.set("env", self._environment) - self.set("command", command) - self.set("env", env) + # set content to stdin, so the content will be + # uploaded during provision + self.set("stdin", self.get("content")) - # set content to stdin, so the content will be - # uploaded during provision - self.set("stdin", self.get("content")) + command = self.get("command") - self.info("Deploying command '%s' " % command) + self.info("Deploying command '%s' " % command) - # create run dir for application - self.node.mkdir(self.run_home) + self.discover() + self.provision() + except: + self.fail() + raise + + self.debug("----- READY ---- ") + self._ready_time = tnow() + self._state = ResourceState.READY - # upload content - self.upload_stdin() + def upload_start_command(self): + command = self.get("command") + env = self.get("env") + if command: # We want to make sure the content is published # before the experiment starts. # Run the command as a bash script in the background, # in the host ( but wait until the command has # finished to continue ) + env = self.replace_paths(env) + command = self.replace_paths(command) + (out, err), proc = self.execute_command(command, env) if proc.poll(): - self._state = ResourceState.FAILED + self.fail() msg = "Failed to execute command" self.error(msg, out, err) raise RuntimeError, msg - self.debug("----- READY ---- ") - self._ready_time = tnow() - self._state = ResourceState.READY - def start(self): if self._state == ResourceState.READY: command = self.get("command") diff --git a/src/nepi/resources/linux/ccn/ccnd.py b/src/nepi/resources/linux/ccn/ccnd.py index cc30823d..d2358a4e 100644 --- a/src/nepi/resources/linux/ccn/ccnd.py +++ b/src/nepi/resources/linux/ccn/ccnd.py @@ -143,59 +143,53 @@ class LinuxCCND(LinuxApplication): # ccnd needs to wait until node is deployed and running self.ec.schedule(reschedule_delay, self.deploy) else: - if not self.get("command"): - self.set("command", self._start_command) - - if not self.get("depends"): - self.set("depends", self._dependencies) - - if not self.get("sources"): - self.set("sources", self._sources) - - sources = self.get("sources") - source = sources.split(" ")[0] - basename = os.path.basename(source) - self._version = ( basename.strip().replace(".tar.gz", "") - .replace(".tar","") - .replace(".gz","") - .replace(".zip","") ) - - if not self.get("build"): - self.set("build", self._build) - - if not self.get("install"): - self.set("install", self._install) - - if not self.get("env"): - self.set("env", self._environment) - - command = self.get("command") - env = self.get("env") - - self.info("Deploying command '%s' " % command) - - # create home dir for application - self.node.mkdir(self.run_home) - - # upload sources - self.upload_sources() - - # upload code - self.upload_code() - - # upload stdin - self.upload_stdin() - - # install dependencies - self.install_dependencies() - - # build - self.build() + try: + if not self.get("command"): + self.set("command", self._start_command) + + if not self.get("depends"): + self.set("depends", self._dependencies) + + if not self.get("sources"): + self.set("sources", self._sources) + + sources = self.get("sources") + source = sources.split(" ")[0] + basename = os.path.basename(source) + self._version = ( basename.strip().replace(".tar.gz", "") + .replace(".tar","") + .replace(".gz","") + .replace(".zip","") ) + + if not self.get("build"): + self.set("build", self._build) + + if not self.get("install"): + self.set("install", self._install) + + if not self.get("env"): + self.set("env", self._environment) + + command = self.get("command") + + self.info("Deploying command '%s' " % command) + + self.discover() + self.provision() + except: + self.fail() + raise + + self.debug("----- READY ---- ") + self._ready_time = tnow() + self._state = ResourceState.READY - # Install - self.install() + def upload_start_command(self): + command = self.get("command") + env = self.get("env") - # We want to make sure the repository is running + if command: + # We want to make sure the ccnd is running # before the experiment starts. # Run the command as a bash script in background, # in the host ( but wait until the command has @@ -210,10 +204,6 @@ class LinuxCCND(LinuxApplication): env = env, raise_on_error = True) - self.debug("----- READY ---- ") - self._ready_time = tnow() - self._state = ResourceState.READY - def start(self): if self._state == ResourceState.READY: command = self.get("command") diff --git a/src/nepi/resources/linux/ccn/ccnr.py b/src/nepi/resources/linux/ccn/ccnr.py index b03dbed4..7ed260c2 100644 --- a/src/nepi/resources/linux/ccn/ccnr.py +++ b/src/nepi/resources/linux/ccn/ccnr.py @@ -201,19 +201,32 @@ class LinuxCCNR(LinuxApplication): # ccnr needs to wait until ccnd is deployed and running self.ec.schedule(reschedule_delay, self.deploy) else: - command = self._start_command - env = self._environment + try: + if not self.get("command"): + self.set("command", self._start_command) - self.set("command", command) - self.set("env", env) + if not self.get("env"): + self.set("env", self._environment) - self.info("Deploying command '%s' " % command) + command = self.get("command") - self.node.mkdir(self.run_home) + self.info("Deploying command '%s' " % command) - # upload sources - self.upload_sources() + self.discover() + self.provision() + except: + self.fail() + raise + + self.debug("----- READY ---- ") + self._ready_time = tnow() + self._state = ResourceState.READY + + def upload_start_command(self): + command = self.get("command") + env = self.get("env") + if command: # We want to make sure the repository is running # before the experiment starts. # Run the command as a bash script in background, @@ -229,10 +242,6 @@ class LinuxCCNR(LinuxApplication): env = env, raise_on_error = True) - self.debug("----- READY ---- ") - self._ready_time = tnow() - self._state = ResourceState.READY - def start(self): if self._state == ResourceState.READY: command = self.get("command") diff --git a/src/nepi/resources/linux/ccn/fibentry.py b/src/nepi/resources/linux/ccn/fibentry.py index 2af3b2cf..cf2c8fe6 100644 --- a/src/nepi/resources/linux/ccn/fibentry.py +++ b/src/nepi/resources/linux/ccn/fibentry.py @@ -85,17 +85,40 @@ class LinuxFIBEntry(LinuxApplication): # ccnr needs to wait until ccnd is deployed and running self.ec.schedule(reschedule_delay, self.deploy) else: - command = self._start_command - env = self._environment + try: + if not self.get("command"): + self.set("command", self._start_command) - self.set("command", command) - self.set("env", env) + if not self.get("env"): + self.set("env", self._environment) - self.info("Deploying command '%s' " % command) - - # create run dir for application - self.node.mkdir(self.run_home) + command = self.get("command") + + self.info("Deploying command '%s' " % command) + + self.discover() + self.provision() + except: + self.fail() + raise + self.debug("----- READY ---- ") + self._ready_time = tnow() + self._state = ResourceState.READY + + def upload_start_command(self): + command = self.get("command") + env = self.get("env") + + if command: + # We want to make sure the FIB entries are created + # before the experiment starts. + # Run the command as a bash script in the background, + # in the host ( but wait until the command has + # finished to continue ) + env = env and self.replace_paths(env) + command = self.replace_paths(command) + (out, err), proc = self.execute_command(command, env) if proc.poll(): @@ -104,10 +127,6 @@ class LinuxFIBEntry(LinuxApplication): self.error(msg, out, err) raise RuntimeError, msg - self.debug("----- READY ---- ") - self._ready_time = tnow() - self._state = ResourceState.READY - def start(self): if self._state in [ResourceState.READY, ResourceState.STARTED]: command = self.get("command") diff --git a/test/resources/linux/application.py b/test/resources/linux/application.py index eb278ffb..11d5758a 100755 --- a/test/resources/linux/application.py +++ b/test/resources/linux/application.py @@ -49,7 +49,7 @@ class LinuxApplicationTestCase(unittest.TestCase): ResourceFactory.register_type(LinuxNode) ResourceFactory.register_type(LinuxApplication) - ec = ExperimentController() + ec = ExperimentController(exp_id = "test-stdout") node = ec.register_resource("LinuxNode") ec.set(node, "hostname", host) @@ -81,7 +81,7 @@ class LinuxApplicationTestCase(unittest.TestCase): ResourceFactory.register_type(LinuxNode) ResourceFactory.register_type(LinuxApplication) - ec = ExperimentController() + ec = ExperimentController(exp_id = "test-ping") node = ec.register_resource("LinuxNode") ec.set(node, "hostname", host) @@ -123,7 +123,7 @@ class LinuxApplicationTestCase(unittest.TestCase): ResourceFactory.register_type(LinuxNode) ResourceFactory.register_type(LinuxApplication) - ec = ExperimentController() + ec = ExperimentController(exp_id = "tests-code") node = ec.register_resource("LinuxNode") ec.set(node, "hostname", host) @@ -166,7 +166,7 @@ main (void) ResourceFactory.register_type(LinuxNode) ResourceFactory.register_type(LinuxApplication) - ec = ExperimentController() + ec = ExperimentController(exp_id="test-concurrency") node = ec.register_resource("LinuxNode") ec.set(node, "hostname", host) @@ -214,7 +214,7 @@ main (void) ResourceFactory.register_type(LinuxNode) ResourceFactory.register_type(LinuxApplication) - ec = ExperimentController() + ec = ExperimentController(exp_id="test-condition") node = ec.register_resource("LinuxNode") ec.set(node, "hostname", host) @@ -257,7 +257,7 @@ main (void) ResourceFactory.register_type(LinuxNode) ResourceFactory.register_type(LinuxApplication) - ec = ExperimentController() + ec = ExperimentController(exp_id="test-http-sources") node = ec.register_resource("LinuxNode") ec.set(node, "hostname", host) @@ -280,10 +280,10 @@ main (void) self.assertTrue(ec.state(node) == ResourceState.STARTED) self.assertTrue(ec.state(app) == ResourceState.FINISHED) - exitcode = ec.trace(app, "http_sources_exitcode") + exitcode = ec.trace(app, "deploy_exitcode") self.assertTrue(exitcode.strip() == "0") - out = ec.trace(app, "http_sources_stdout") + out = ec.trace(app, "deploy_stdout") self.assertTrue(out.find("tip.tar.gz") > -1) self.assertTrue(out.find("connect.png") > -1) @@ -296,7 +296,7 @@ main (void) ResourceFactory.register_type(LinuxNode) ResourceFactory.register_type(LinuxApplication) - ec = ExperimentController() + ec = ExperimentController(exp_id="test-xterm") node = ec.register_resource("LinuxNode") ec.set(node, "hostname", host)