2 # -*- coding: utf-8 -*-
4 from constants import TESTBED_ID
10 import nepi.util.server as server
19 from nepi.util.constants import ApplicationStatus as AS
21 class Dependency(object):
23 A Dependency is in every respect like an application.
25 It depends on some packages, it may require building binaries, it must deploy
28 But it has no command. Dependencies aren't ever started, or stopped, and have
34 def __init__(self, api=None):
46 self.buildDepends = None
48 self.rpmFusion = False
56 self.add_to_path = True
58 # Those are filled when the app is configured
61 # Those are filled when an actual node is connected
64 # Those are filled when the app is started
65 # Having both pid and ppid makes it harder
66 # for pid rollover to induce tracking mistakes
73 # Spanning tree deployment
75 self._master_passphrase = None
76 self._master_prk = None
77 self._master_puk = None
78 self._master_token = ''.join(map(chr,[rng.randint(0,255)
79 for rng in (random.SystemRandom(),)
80 for i in xrange(8)] )).encode("hex")
81 self._build_pid = None
82 self._build_ppid = None
87 self.__class__.__name__,
88 ' '.join(filter(bool,(self.depends, self.sources)))
92 if self.home_path is None:
93 raise AssertionError, "Misconfigured application: missing home path"
94 if self.node.ident_path is None or not os.access(self.node.ident_path, os.R_OK):
95 raise AssertionError, "Misconfigured application: missing slice SSH key"
97 raise AssertionError, "Misconfigured application: unconnected node"
98 if self.node.hostname is None:
99 raise AssertionError, "Misconfigured application: misconfigured node"
100 if self.node.slicename is None:
101 raise AssertionError, "Misconfigured application: unspecified slice"
103 def remote_trace_path(self, whichtrace):
104 if whichtrace in self.TRACES:
105 tracefile = os.path.join(self.home_path, whichtrace)
111 def sync_trace(self, local_dir, whichtrace):
112 tracefile = self.remote_trace_path(whichtrace)
116 local_path = os.path.join(local_dir, tracefile)
118 # create parent local folders
119 proc = subprocess.Popen(
120 ["mkdir", "-p", os.path.dirname(local_path)],
121 stdout = open("/dev/null","w"),
122 stdin = open("/dev/null","r"))
125 raise RuntimeError, "Failed to synchronize trace"
130 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
134 except RuntimeError, e:
135 raise RuntimeError, "Failed to synchronize trace: %s %s" \
136 % (e.args[0], e.args[1],)
141 print >>sys.stderr, "Setting up", self
147 def async_setup(self):
148 if not self._setuper:
153 self._setuper._exc.append(sys.exc_info())
154 self._setuper = threading.Thread(
156 self._setuper._exc = []
157 self._setuper.start()
159 def async_setup_wait(self):
161 print >>sys.stderr, "Waiting for", self, "to be setup"
165 if self._setuper._exc:
166 exctyp,exval,exctrace = self._setuper._exc[0]
167 raise exctyp,exval,exctrace
169 raise RuntimeError, "Failed to setup application"
173 def _make_home(self):
174 # Make sure all the paths are created where
175 # they have to be created for deployment
178 self._popen_ssh_command(
179 "mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" \
180 % { 'home' : server.shell_escape(self.home_path) }
182 except RuntimeError, e:
183 raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, e.args[0], e.args[1],)
186 # Write program input
189 cStringIO.StringIO(self.stdin),
190 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
191 os.path.join(self.home_path, 'stdin') ),
193 except RuntimeError, e:
194 raise RuntimeError, "Failed to set up application %s: %s %s" \
195 % (self.home_path, e.args[0], e.args[1],)
197 def _replace_paths(self, command):
199 Replace all special path tags with shell-escaped actual paths.
201 # need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
202 root = '' if self.home_path.startswith('/') else "${HOME}/"
204 .replace("${SOURCES}", root+server.shell_escape(self.home_path))
205 .replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
207 def _launch_build(self):
208 if self._master is not None:
209 self._do_install_keys()
210 buildscript = self._do_build_slave()
212 buildscript = self._do_build_master()
214 if buildscript is not None:
215 print >>sys.stderr, "Building", self
217 # upload build script
221 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
222 os.path.join(self.home_path, 'nepi-build.sh') )
224 except RuntimeError, e:
225 raise RuntimeError, "Failed to set up application %s: %s %s" \
226 % (self.home_path, e.args[0], e.args[1],)
229 self._do_launch_build()
231 def _finish_build(self):
232 self._do_wait_build()
235 def _do_build_slave(self):
236 if not self.sources and not self.build:
239 # Create build script
243 sources = self.sources.split(' ')
245 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
246 os.path.join(self._master.home_path, os.path.basename(source)),)
247 for source in sources
252 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
253 os.path.join(self._master.home_path, 'build.tar.gz'),)
256 launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
257 " && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
258 " && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" % \
260 'prk' : server.shell_escape(self._master_prk_name),
261 'puk' : server.shell_escape(self._master_puk_name),
264 kill_agent = "kill $SSH_AGENT_PID"
266 waitmaster = "{ . ./.ssh-agent.sh ; while [[ $(ssh -q -o UserKnownHostsFile=%(hostkey)s %(master)s cat %(token_path)s) != %(token)s ]] ; do sleep 5 ; done ; }" % {
267 'hostkey' : 'master_known_hosts',
268 'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostname),
269 'token_path' : os.path.join(self._master.home_path, 'build.token'),
270 'token' : server.shell_escape(self._master._master_token),
273 syncfiles = "scp -p -o UserKnownHostsFile=%(hostkey)s %(files)s ." % {
274 'hostkey' : 'master_known_hosts',
275 'files' : ' '.join(files),
278 syncfiles += " && tar xzf build.tar.gz"
279 syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
280 syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
282 cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
283 'prk' : server.shell_escape(self._master_prk_name),
284 'puk' : server.shell_escape(self._master_puk_name),
287 slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s )" % {
288 'waitmaster' : waitmaster,
289 'syncfiles' : syncfiles,
291 'kill_agent' : kill_agent,
292 'launch_agent' : launch_agent,
293 'home' : server.shell_escape(self.home_path),
296 return cStringIO.StringIO(slavescript)
298 def _do_launch_build(self):
299 script = "bash ./nepi-build.sh"
300 if self._master_passphrase:
301 script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
302 server.shell_escape(self._master_passphrase),
305 (out,err),proc = rspawn.remote_spawn(
307 pidfile = 'build-pid',
308 home = self.home_path,
311 stderr = rspawn.STDOUT,
313 host = self.node.hostname,
315 user = self.node.slicename,
317 ident_key = self.node.ident_path,
318 server_key = self.node.server_key
322 raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
328 pidtuple = rspawn.remote_check_pid(
329 os.path.join(self.home_path,'build-pid'),
330 host = self.node.hostname,
332 user = self.node.slicename,
334 ident_key = self.node.ident_path,
335 server_key = self.node.server_key
340 self._build_pid, self._build_ppid = pidtuple
344 delay = min(30,delay*1.2)
346 raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
348 print >>sys.stderr, "Deploying", self
350 def _do_wait_build(self):
351 pid = self._build_pid
352 ppid = self._build_ppid
358 status = rspawn.remote_status(
360 host = self.node.hostname,
362 user = self.node.slicename,
364 ident_key = self.node.ident_path,
365 server_key = self.node.server_key
368 if status is not rspawn.RUNNING:
369 self._build_pid = self._build_ppid = None
373 print >>sys.stderr, "Waiting for", self, "to finish building",
374 if self._master is not None:
375 print >>sys.stderr, "(build slave)"
377 print >>sys.stderr, "(build master)"
380 time.sleep(delay*(0.5+random.random()))
381 delay = min(30,delay*1.2)
384 (out, err), proc = self._popen_ssh_command(
385 "cat %(token_path)s" % {
386 'token_path' : os.path.join(self.home_path, 'build.token'),
390 if not proc.wait() and out:
391 slave_token = out.strip()
393 if slave_token != self._master_token:
394 # Get buildlog for the error message
396 (buildlog, err), proc = self._popen_ssh_command(
397 "cat %(buildlog)s" % {
398 'buildlog' : os.path.join(self.home_path, 'buildlog'),
399 'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
405 raise RuntimeError, "Failed to set up application %s: "\
406 "build failed, got wrong token from pid %s/%s "\
407 "(expected %r, got %r), see buildlog: %s" % (
408 self.home_path, pid, ppid, self._master_token, slave_token, buildlog)
410 print >>sys.stderr, "Built", self
412 def _do_kill_build(self):
413 pid = self._build_pid
414 ppid = self._build_ppid
417 print >>sys.stderr, "Killing build of", self
420 host = self.node.hostname,
422 user = self.node.slicename,
424 ident_key = self.node.ident_path
428 def _do_build_master(self):
429 if not self.sources and not self.build and not self.buildDepends:
433 sources = self.sources.split(' ')
439 "%s@%s:%s" % (self.node.slicename, self.node.hostname,
440 os.path.join(self.home_path,'.'),)
442 except RuntimeError, e:
443 raise RuntimeError, "Failed upload source file %r: %s %s" \
444 % (sources, e.args[0], e.args[1],)
446 buildscript = cStringIO.StringIO()
448 if self.buildDepends:
449 # Install build dependencies
451 "sudo -S yum -y install %(packages)s\n" % {
452 'packages' : self.buildDepends
460 "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
461 'command' : self._replace_paths(self.build),
462 'home' : server.shell_escape(self.home_path),
467 buildscript.write("tar czf build.tar.gz build\n")
470 buildscript.write("echo %(master_token)s > build.token" % {
471 'master_token' : server.shell_escape(self._master_token)
478 def _do_install(self):
480 print >>sys.stderr, "Installing", self
482 # Install application
484 self._popen_ssh_command(
485 "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/installlog >&2 && false )" % \
487 'command' : self._replace_paths(self.install),
488 'home' : server.shell_escape(self.home_path),
491 except RuntimeError, e:
492 raise RuntimeError, "Failed install build sources: %s %s" % (e.args[0], e.args[1],)
494 def set_master(self, master):
495 self._master = master
497 def install_keys(self, prk, puk, passphrase):
499 self._master_passphrase = passphrase
500 self._master_prk = prk
501 self._master_puk = puk
502 self._master_prk_name = os.path.basename(prk.name)
503 self._master_puk_name = os.path.basename(puk.name)
505 def _do_install_keys(self):
506 prk = self._master_prk
507 puk = self._master_puk
511 [ prk.name, puk.name ],
512 '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path )
514 except RuntimeError, e:
515 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
516 % (e.args[0], e.args[1],)
520 cStringIO.StringIO('%s,%s %s\n' % (
521 self._master.node.hostname, socket.gethostbyname(self._master.node.hostname),
522 self._master.node.server_key)),
523 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
524 os.path.join(self.home_path,"master_known_hosts") )
526 except RuntimeError, e:
527 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
528 % (e.args[0], e.args[1],)
531 self._master_prk = None
532 self._master_puk = None
535 # make sure there's no leftover build processes
536 self._do_kill_build()
539 def _popen_scp(self, src, dst, retry = True):
540 (out,err),proc = server.popen_scp(
545 ident_key = self.node.ident_path,
546 server_key = self.node.server_key
549 if server.eintr_retry(proc.wait)():
550 raise RuntimeError, (out, err)
551 return (out, err), proc
555 def _popen_ssh_command(self, command, retry = True, noerrors=False):
556 (out,err),proc = server.popen_ssh_command(
558 host = self.node.hostname,
560 user = self.node.slicename,
562 ident_key = self.node.ident_path,
563 server_key = self.node.server_key
566 if server.eintr_retry(proc.wait)():
568 raise RuntimeError, (out, err)
569 return (out, err), proc
571 class Application(Dependency):
573 An application also has dependencies, but also a command to be ran and monitored.
575 It adds the output of that command as traces.
578 TRACES = ('stdout','stderr','buildlog')
580 def __init__(self, api=None):
581 super(Application,self).__init__(api)
591 # Those are filled when the app is started
592 # Having both pid and ppid makes it harder
593 # for pid rollover to induce tracking mistakes
594 self._started = False
598 # Do not add to the python path of nodes
599 self.add_to_path = False
602 return "%s<command:%s%s>" % (
603 self.__class__.__name__,
604 "sudo " if self.sudo else "",
609 print >>sys.stderr, "Starting", self
611 # Create shell script with the command
612 # This way, complex commands and scripts can be ran seamlessly
614 command = cStringIO.StringIO()
615 command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
616 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
618 command.write('export PATH=$PATH:%s\n' % (
619 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
622 for envkey, envvals in self.node.env.iteritems():
623 for envval in envvals:
624 command.write('export %s=%s\n' % (envkey, envval))
625 command.write(self.command)
631 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
632 os.path.join(self.home_path, "app.sh"))
634 except RuntimeError, e:
635 raise RuntimeError, "Failed to set up application: %s %s" \
636 % (e.args[0], e.args[1],)
638 # Start process in a "daemonized" way, using nohup and heavy
639 # stdin/out redirection to avoid connection issues
640 (out,err),proc = rspawn.remote_spawn(
641 self._replace_paths("bash ./app.sh"),
644 home = self.home_path,
645 stdin = 'stdin' if self.stdin is not None else '/dev/null',
646 stdout = 'stdout' if self.stdout else '/dev/null',
647 stderr = 'stderr' if self.stderr else '/dev/null',
650 host = self.node.hostname,
652 user = self.node.slicename,
654 ident_key = self.node.ident_path,
655 server_key = self.node.server_key
659 raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
665 # NOTE: wait a bit for the pidfile to be created
666 if self._started and not self._pid or not self._ppid:
667 pidtuple = rspawn.remote_check_pid(
668 os.path.join(self.home_path,'pid'),
669 host = self.node.hostname,
671 user = self.node.slicename,
673 ident_key = self.node.ident_path,
674 server_key = self.node.server_key
678 self._pid, self._ppid = pidtuple
682 if not self._started:
683 return AS.STATUS_NOT_STARTED
684 elif not self._pid or not self._ppid:
685 return AS.STATUS_NOT_STARTED
687 status = rspawn.remote_status(
688 self._pid, self._ppid,
689 host = self.node.hostname,
691 user = self.node.slicename,
693 ident_key = self.node.ident_path,
694 server_key = self.node.server_key
697 if status is rspawn.NOT_STARTED:
698 return AS.STATUS_NOT_STARTED
699 elif status is rspawn.RUNNING:
700 return AS.STATUS_RUNNING
701 elif status is rspawn.FINISHED:
702 return AS.STATUS_FINISHED
705 return AS.STATUS_NOT_STARTED
708 status = self.status()
709 if status == AS.STATUS_RUNNING:
710 # kill by ppid+pid - SIGTERM first, then try SIGKILL
712 self._pid, self._ppid,
713 host = self.node.hostname,
715 user = self.node.slicename,
717 ident_key = self.node.ident_path,
718 server_key = self.node.server_key
720 print >>sys.stderr, "Killed", self
723 class NepiDependency(Dependency):
725 This dependency adds nepi itself to the python path,
726 so that you may run testbeds within PL nodes.
729 # Class attribute holding a *weak* reference to the shared NEPI tar file
730 # so that they may share it. Don't operate on the file itself, it would
731 # be a mess, just use its path.
732 _shared_nepi_tar = None
734 def __init__(self, api = None):
735 super(NepiDependency, self).__init__(api)
739 self.depends = 'python python-ipaddr python-setuptools'
741 # our sources are in our ad-hoc tarball
742 self.sources = self.tarball.name
744 tarname = os.path.basename(self.tarball.name)
746 # it's already built - just move the tarball into place
747 self.build = "mv -f ${SOURCES}/%s ." % (tarname,)
749 # unpack it into sources, and we're done
750 self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,)
754 if self._tarball is None:
755 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
756 if shared_tar is not None:
757 self._tarball = shared_tar
759 # Build an ad-hoc tarball
764 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
766 proc = subprocess.Popen(
767 ["tar", "czf", shared_tar.name,
768 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
770 stdout = open("/dev/null","w"),
771 stdin = open("/dev/null","r"))
774 raise RuntimeError, "Failed to create nepi tarball"
776 self._tarball = self._shared_nepi_tar = shared_tar
780 class NS3Dependency(Dependency):
782 This dependency adds NS3 libraries to the library paths,
783 so that you may run the NS3 testbed within PL nodes.
785 You'll also need the NepiDependency.
788 def __init__(self, api = None):
789 super(NS3Dependency, self).__init__(api)
791 self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip'
793 # We have to download the sources, untar, build...
794 pybindgen_source_url = "http://pybindgen.googlecode.com/files/pybindgen-0.15.0.zip"
795 pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
796 ns3_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/nepi-ns-3.9/archive/tip.tar.gz"
797 passfd_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/python-passfd/archive/tip.tar.gz"
801 " python -c 'import pygccxml, pybindgen, passfd' && "
802 " test -f lib/_ns3.so && "
803 " test -f lib/libns3.so "
805 # Not working, rebuild
806 "wget -q -c -O pybindgen-src.zip %(pybindgen_source_url)s && " # continue, to exploit the case when it has already been dl'ed
807 "wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
808 "wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
809 "wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
810 "unzip -n pybindgen-src.zip && " # Do not overwrite files, to exploit the case when it has already been built
811 "unzip -n pygccxml-1.0.0.zip && "
812 "mkdir -p ns3-src && "
813 "mkdir -p passfd-src && "
814 "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
815 "tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
816 "rm -rf target && " # mv doesn't like unclean targets
817 "mkdir -p target && "
818 "cd pygccxml-1.0.0 && "
819 "rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
820 "python setup.py build && "
821 "python setup.py install --install-lib ${BUILD}/target && "
822 "python setup.py clean && "
823 "cd ../pybindgen-0.15.0 && "
824 "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
825 "./waf configure --prefix=${BUILD}/target -d release && "
829 "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
830 "rm -rf ${BUILD}/target/lib && "
831 "cd ../passfd-src && "
832 "python setup.py build && "
833 "python setup.py install --install-lib ${BUILD}/target && "
834 "python setup.py clean && "
836 "./waf configure --prefix=${BUILD}/target -d release --disable-examples --high-precision-as-double && "
842 pybindgen_source_url = server.shell_escape(pybindgen_source_url),
843 pygccxml_source_url = server.shell_escape(pygccxml_source_url),
844 ns3_source_url = server.shell_escape(ns3_source_url),
845 passfd_source_url = server.shell_escape(passfd_source_url),
848 # Just move ${BUILD}/target
852 " python -c 'import pygccxml, pybindgen, passfd' && "
853 " test -f lib/_ns3.so && "
854 " test -f lib/libns3.so "
856 # Not working, reinstall
857 "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
858 "mv -f ${BUILD}/target/* ${SOURCES}"
862 # Set extra environment paths
863 self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
864 self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib/libns3.so"
868 if self._tarball is None:
869 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
870 if shared_tar is not None:
871 self._tarball = shared_tar
873 # Build an ad-hoc tarball
878 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
880 proc = subprocess.Popen(
881 ["tar", "czf", shared_tar.name,
882 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
884 stdout = open("/dev/null","w"),
885 stdin = open("/dev/null","r"))
888 raise RuntimeError, "Failed to create nepi tarball"
890 self._tarball = self._shared_nepi_tar = shared_tar