+ if self.check_bad_host(out, err):
+ self.node.blacklist()
+ raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
+
+
+ pid = ppid = None
+ delay = 1.0
+ for i in xrange(5):
+ pidtuple = rspawn.remote_check_pid(
+ os.path.join(self.home_path,'build-pid'),
+ host = self.node.hostname,
+ port = None,
+ user = self.node.slicename,
+ agent = None,
+ ident_key = self.node.ident_path,
+ server_key = self.node.server_key,
+ hostip = self.node.hostip
+ )
+
+ if pidtuple:
+ pid, ppid = pidtuple
+ self._build_pid, self._build_ppid = pidtuple
+ break
+ else:
+ time.sleep(delay)
+ delay = min(30,delay*1.2)
+ else:
+ raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
+
+ self._logger.info("Deploying %s at %s", self, self.node.hostname)
+
+ def _do_wait_build(self, trial=0):
+ pid = self._build_pid
+ ppid = self._build_ppid
+
+ if pid and ppid:
+ delay = 1.0
+ first = True
+ bustspin = 0
+ while True:
+ status = rspawn.remote_status(
+ pid, ppid,
+ host = self.node.hostname,
+ port = None,
+ user = self.node.slicename,
+ agent = None,
+ ident_key = self.node.ident_path,
+ server_key = self.node.server_key,
+ hostip = self.node.hostip
+ )
+
+ if status is rspawn.FINISHED:
+ self._build_pid = self._build_ppid = None
+ break
+ elif status is not rspawn.RUNNING:
+ self._logger.warn("Busted waiting for %s to finish building at %s %s", self, self.node.hostname,
+ "(build slave)" if self._master is not None else "(build master)")
+ bustspin += 1
+ time.sleep(delay*(5.5+random.random()))
+ if bustspin > 12:
+ self._build_pid = self._build_ppid = None
+ break
+ else:
+ if first:
+ self._logger.info("Waiting for %s to finish building at %s %s", self, self.node.hostname,
+ "(build slave)" if self._master is not None else "(build master)")
+
+ first = False
+ time.sleep(delay*(0.5+random.random()))
+ delay = min(30,delay*1.2)
+ bustspin = 0
+
+ # check build token
+ slave_token = ""
+ for i in xrange(3):
+ (out, err), proc = self._popen_ssh_command(
+ "cat %(token_path)s" % {
+ 'token_path' : os.path.join(self.home_path, 'build.token'),
+ },
+ timeout = 120,
+ noerrors = True)
+ if not proc.wait() and out:
+ slave_token = out.strip()
+
+ if slave_token:
+ break
+ else:
+ time.sleep(2)
+
+ if slave_token != self._master_token:
+ # Get buildlog for the error message
+
+ (buildlog, err), proc = self._popen_ssh_command(
+ "cat %(buildlog)s" % {
+ 'buildlog' : os.path.join(self.home_path, 'buildlog'),
+ 'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
+ },
+ timeout = 120,
+ noerrors = True)
+
+ proc.wait()
+
+ if self.check_bad_host(buildlog, err):
+ self.node.blacklist()
+ elif self._master and trial < 3 and 'BAD TOKEN' in buildlog or 'BAD TOKEN' in err:
+ # bad sync with master, may try again
+ # but first wait for master
+ self._master.async_setup_wait()
+ self._launch_build(trial+1)
+ return self._do_wait_build(trial+1)
+ elif trial < 3:
+ return self._do_wait_build(trial+1)
+ else:
+ # No longer need'em
+ self._master_prk = None
+ self._master_puk = None
+
+ raise RuntimeError, "Failed to set up application %s: "\
+ "build failed, got wrong token from pid %s/%s "\
+ "(expected %r, got %r), see buildlog at %s:\n%s" % (
+ self.home_path, pid, ppid, self._master_token, slave_token, self.node.hostname, buildlog)
+
+ # No longer need'em
+ self._master_prk = None
+ self._master_puk = None
+
+ self._logger.info("Built %s at %s", self, self.node.hostname)
+
+ def _do_kill_build(self):
+ pid = self._build_pid
+ ppid = self._build_ppid
+
+ if pid and ppid:
+ self._logger.info("Killing build of %s", self)
+ rspawn.remote_kill(
+ pid, ppid,
+ host = self.node.hostname,
+ port = None,
+ user = self.node.slicename,
+ agent = None,
+ ident_key = self.node.ident_path,
+ hostip = self.node.hostip
+ )
+
+
+ def _do_build_master(self):
+ if not self.sources and not self.build and not self.buildDepends:
+ return None
+
+ if self.sources:
+ sources = self.sources.split(' ')
+
+ # Copy all sources
+ try:
+ self._popen_scp(
+ sources,
+ "%s@%s:%s" % (self.node.slicename, self.node.hostname,
+ os.path.join(self.home_path,'.'),)
+ )
+ except RuntimeError, e:
+ raise RuntimeError, "Failed upload source file %r: %s %s" \
+ % (sources, e.args[0], e.args[1],)
+
+ buildscript = cStringIO.StringIO()
+
+ buildscript.write("(\n")
+
+ if self.buildDepends:
+ # Install build dependencies
+ buildscript.write(
+ "sudo -S yum -y install %(packages)s\n" % {
+ 'packages' : self.buildDepends
+ }
+ )
+
+
+ if self.build:
+ # Build sources
+ buildscript.write(
+ "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
+ 'command' : self._replace_paths(self.build),
+ 'home' : server.shell_escape(self.home_path),
+ }
+ )
+
+ # Make archive
+ buildscript.write("tar czf build.tar.gz build\n")
+
+ # Write token
+ buildscript.write("echo %(master_token)s > build.token ) ; echo %(master_token)s > build.token.retcode" % {
+ 'master_token' : server.shell_escape(self._master_token)
+ })
+
+ buildscript.seek(0)
+
+ return buildscript
+
+ def _do_install(self):
+ if self.install:
+ self._logger.info("Installing %s at %s", self, self.node.hostname)
+
+ # Install application
+ try:
+ self._popen_ssh_command(
+ "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/{install,build}log >&2 && false )" % \
+ {
+ 'command' : self._replace_paths(self.install),
+ 'home' : server.shell_escape(self.home_path),
+ },
+ )
+ except RuntimeError, e:
+ if self.check_bad_host(e.args[0], e.args[1]):
+ self.node.blacklist()
+ raise RuntimeError, "Failed install build sources on node %s: %s %s" % (
+ self.node.hostname, e.args[0], e.args[1],)
+
+ def set_master(self, master):
+ self._master = master
+
+ def install_keys(self, prk, puk, passphrase):
+ # Install keys
+ self._master_passphrase = passphrase
+ self._master_prk = prk
+ self._master_puk = puk
+ self._master_prk_name = os.path.basename(prk.name)
+ self._master_puk_name = os.path.basename(puk.name)
+
+ def _do_install_keys(self):
+ prk = self._master_prk
+ puk = self._master_puk
+
+ try:
+ self._popen_scp(
+ [ prk.name, puk.name ],
+ '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path )
+ )
+ except RuntimeError, e:
+ raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
+ % (e.args[0], e.args[1],)
+
+ try:
+ self._popen_scp(
+ cStringIO.StringIO('%s,%s %s\n' % (
+ self._master.node.hostname, self._master.node.hostip,
+ self._master.node.server_key)),
+ '%s@%s:%s' % (self.node.slicename, self.node.hostname,
+ os.path.join(self.home_path,"master_known_hosts") )
+ )
+ except RuntimeError, e:
+ raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
+ % (e.args[0], e.args[1],)
+
+
+ def cleanup(self):
+ # make sure there's no leftover build processes
+ self._do_kill_build()
+
+ # No longer need'em
+ self._master_prk = None
+ self._master_puk = None
+
+ @server.eintr_retry
+ def _popen_scp(self, src, dst, retry = 3):
+ while 1:
+ try:
+ (out,err),proc = server.popen_scp(
+ src,
+ dst,
+ port = None,
+ agent = None,
+ ident_key = self.node.ident_path,
+ server_key = self.node.server_key
+ )
+
+ if server.eintr_retry(proc.wait)():
+ raise RuntimeError, (out, err)
+ return (out, err), proc
+ except:
+ if retry <= 0:
+ raise
+ else:
+ retry -= 1
+
+
+ @server.eintr_retry
+ def _popen_ssh_command(self, command, retry = 0, noerrors=False, timeout=None):
+ (out,err),proc = server.popen_ssh_command(
+ command,
+ host = self.node.hostname,
+ port = None,
+ user = self.node.slicename,
+ agent = None,
+ ident_key = self.node.ident_path,
+ server_key = self.node.server_key,
+ timeout = timeout,
+ retry = retry
+ )
+
+ if server.eintr_retry(proc.wait)():
+ if not noerrors:
+ raise RuntimeError, (out, err)
+ return (out, err), proc
+
+class Application(Dependency):
+ """
+ An application also has dependencies, but also a command to be ran and monitored.
+
+ It adds the output of that command as traces.
+ """
+
+ TRACES = ('stdout','stderr','buildlog', 'output')
+
+ def __init__(self, api=None):
+ super(Application,self).__init__(api)
+
+ # Attributes
+ self.command = None
+ self.sudo = False
+
+ self.stdin = None
+ self.stdout = None
+ self.stderr = None
+ self.output = None
+
+ # Those are filled when the app is started
+ # Having both pid and ppid makes it harder
+ # for pid rollover to induce tracking mistakes
+ self._started = False
+ self._pid = None
+ self._ppid = None
+
+ # Do not add to the python path of nodes
+ self.add_to_path = False
+
+ def __str__(self):
+ return "%s<command:%s%s>" % (
+ self.__class__.__name__,
+ "sudo " if self.sudo else "",
+ self.command,
+ )
+
+ def start(self):
+ self._logger.info("Starting %s", self)
+
+ # Create shell script with the command
+ # This way, complex commands and scripts can be ran seamlessly
+ # sync files
+ command = cStringIO.StringIO()
+ command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
+ ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
+ ))
+ command.write('export PATH=$PATH:%s\n' % (
+ ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
+ ))
+ if self.node.env:
+ for envkey, envvals in self.node.env.iteritems():
+ for envval in envvals:
+ command.write('export %s=%s\n' % (envkey, envval))
+ command.write(self.command)
+ command.seek(0)
+
+ try:
+ self._popen_scp(
+ command,
+ '%s@%s:%s' % (self.node.slicename, self.node.hostname,
+ os.path.join(self.home_path, "app.sh"))
+ )
+ except RuntimeError, e:
+ raise RuntimeError, "Failed to set up application: %s %s" \
+ % (e.args[0], e.args[1],)