2 # -*- coding: utf-8 -*-
4 from constants import TESTBED_ID
10 import nepi.util.server as server
19 from nepi.util.constants import STATUS_NOT_STARTED, STATUS_RUNNING, \
22 class Dependency(object):
24 A Dependency is in every respect like an application.
26 It depends on some packages, it may require building binaries, it must deploy
29 But it has no command. Dependencies aren't ever started, or stopped, and have
35 def __init__(self, api=None):
47 self.buildDepends = None
56 self.add_to_path = True
58 # Those are filled when the app is configured
61 # Those are filled when an actual node is connected
64 # Those are filled when the app is started
65 # Having both pid and ppid makes it harder
66 # for pid rollover to induce tracking mistakes
73 # Spanning tree deployment
75 self._master_passphrase = None
76 self._master_prk = None
77 self._master_puk = None
78 self._master_token = ''.join(map(chr,[rng.randint(0,255)
79 for rng in (random.SystemRandom(),)
80 for i in xrange(8)] )).encode("hex")
81 self._build_pid = None
82 self._build_ppid = None
87 self.__class__.__name__,
88 ' '.join(filter(bool,(self.depends, self.sources)))
92 if self.home_path is None:
93 raise AssertionError, "Misconfigured application: missing home path"
94 if self.node.ident_path is None or not os.access(self.node.ident_path, os.R_OK):
95 raise AssertionError, "Misconfigured application: missing slice SSH key"
97 raise AssertionError, "Misconfigured application: unconnected node"
98 if self.node.hostname is None:
99 raise AssertionError, "Misconfigured application: misconfigured node"
100 if self.node.slicename is None:
101 raise AssertionError, "Misconfigured application: unspecified slice"
103 def remote_trace_path(self, whichtrace):
104 if whichtrace in self.TRACES:
105 tracefile = os.path.join(self.home_path, whichtrace)
111 def sync_trace(self, local_dir, whichtrace):
112 tracefile = self.remote_trace_path(whichtrace)
116 local_path = os.path.join(local_dir, tracefile)
118 # create parent local folders
119 proc = subprocess.Popen(
120 ["mkdir", "-p", os.path.dirname(local_path)],
121 stdout = open("/dev/null","w"),
122 stdin = open("/dev/null","r"))
125 raise RuntimeError, "Failed to synchronize trace"
128 (out,err),proc = server.popen_scp(
129 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
134 ident_key = self.node.ident_path,
135 server_key = self.node.server_key
139 raise RuntimeError, "Failed to synchronize trace: %s %s" % (out,err,)
150 def async_setup(self):
151 if not self._setuper:
156 self._setuper._exc.append(sys.exc_info())
157 self._setuper = threading.Thread(
159 self._setuper._exc = []
160 self._setuper.start()
162 def async_setup_wait(self):
167 if self._setuper._exc:
168 exctyp,exval,exctrace = self._setuper._exc[0]
169 raise exctyp,exval,exctrace
171 raise RuntimeError, "Failed to setup application"
175 def _make_home(self):
176 # Make sure all the paths are created where
177 # they have to be created for deployment
178 (out,err),proc = server.popen_ssh_command(
179 "mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" % { 'home' : server.shell_escape(self.home_path) },
180 host = self.node.hostname,
182 user = self.node.slicename,
184 ident_key = self.node.ident_path,
185 server_key = self.node.server_key
189 raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, out,err,)
192 # Write program input
193 (out,err),proc = server.popen_scp(
194 cStringIO.StringIO(self.stdin),
195 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
196 os.path.join(self.home_path, 'stdin') ),
199 ident_key = self.node.ident_path,
200 server_key = self.node.server_key
204 raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, out,err,)
206 def _replace_paths(self, command):
208 Replace all special path tags with shell-escaped actual paths.
210 # need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
211 root = '' if self.home_path.startswith('/') else "${HOME}/"
213 .replace("${SOURCES}", root+server.shell_escape(self.home_path))
214 .replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
216 def _launch_build(self):
217 if self._master is not None:
218 self._do_install_keys()
219 buildscript = self._do_build_slave()
221 buildscript = self._do_build_master()
223 if buildscript is not None:
224 # upload build script
225 (out,err),proc = server.popen_scp(
227 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
228 os.path.join(self.home_path, 'nepi-build.sh') ),
231 ident_key = self.node.ident_path,
232 server_key = self.node.server_key
236 raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, out,err,)
239 self._do_launch_build()
241 def _finish_build(self):
242 self._do_wait_build()
245 def _do_build_slave(self):
246 if not self.sources and not self.build:
249 # Create build script
253 sources = self.sources.split(' ')
255 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
256 os.path.join(self._master.home_path, os.path.basename(source)),)
257 for source in sources
262 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
263 os.path.join(self._master.home_path, 'build.tar.gz'),)
266 launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
267 " && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
268 " && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" % \
270 'prk' : server.shell_escape(self._master_prk_name),
271 'puk' : server.shell_escape(self._master_puk_name),
274 kill_agent = "kill $SSH_AGENT_PID"
276 waitmaster = "{ . ./.ssh-agent.sh ; while [[ $(ssh -q -o UserKnownHostsFile=%(hostkey)s %(master)s cat %(token_path)s) != %(token)s ]] ; do sleep 5 ; done ; }" % {
277 'hostkey' : 'master_known_hosts',
278 'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostname),
279 'token_path' : os.path.join(self._master.home_path, 'build.token'),
280 'token' : server.shell_escape(self._master._master_token),
283 syncfiles = "scp -p -o UserKnownHostsFile=%(hostkey)s %(files)s ." % {
284 'hostkey' : 'master_known_hosts',
285 'files' : ' '.join(files),
288 syncfiles += " && tar xzf build.tar.gz"
289 syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
290 syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
292 cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
293 'prk' : server.shell_escape(self._master_prk_name),
294 'puk' : server.shell_escape(self._master_puk_name),
297 slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s )" % {
298 'waitmaster' : waitmaster,
299 'syncfiles' : syncfiles,
301 'kill_agent' : kill_agent,
302 'launch_agent' : launch_agent,
303 'home' : server.shell_escape(self.home_path),
306 return cStringIO.StringIO(slavescript)
308 def _do_launch_build(self):
309 script = "bash ./nepi-build.sh"
310 if self._master_passphrase:
311 script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
312 server.shell_escape(self._master_passphrase),
315 (out,err),proc = rspawn.remote_spawn(
318 pidfile = 'build-pid',
319 home = self.home_path,
322 stderr = rspawn.STDOUT,
324 host = self.node.hostname,
326 user = self.node.slicename,
328 ident_key = self.node.ident_path,
329 server_key = self.node.server_key
333 raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
339 pidtuple = rspawn.remote_check_pid(
340 os.path.join(self.home_path,'build-pid'),
341 host = self.node.hostname,
343 user = self.node.slicename,
345 ident_key = self.node.ident_path,
346 server_key = self.node.server_key
351 self._build_pid, self._build_ppid = pidtuple
355 delay = min(30,delay*1.2)
357 raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
359 def _do_wait_build(self):
360 pid = self._build_pid
361 ppid = self._build_ppid
366 status = rspawn.remote_status(
368 host = self.node.hostname,
370 user = self.node.slicename,
372 ident_key = self.node.ident_path,
373 server_key = self.node.server_key
376 if status is not rspawn.RUNNING:
377 self._build_pid = self._build_ppid = None
380 time.sleep(delay*(0.5+random.random()))
381 delay = min(30,delay*1.2)
385 (out,err),proc = server.popen_ssh_command(
386 "cat %(token_path)s" % {
387 'token_path' : os.path.join(self.home_path, 'build.token'),
389 host = self.node.hostname,
391 user = self.node.slicename,
393 ident_key = self.node.ident_path,
394 server_key = self.node.server_key
398 if not proc.wait() and out:
399 slave_token = out.strip()
401 if slave_token != self._master_token:
402 # Get buildlog for the error message
404 (buildlog,err),proc = server.popen_ssh_command(
405 "cat %(buildlog)s" % {
406 'buildlog' : os.path.join(self.home_path, 'buildlog'),
407 'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
409 host = self.node.hostname,
411 user = self.node.slicename,
413 ident_key = self.node.ident_path,
414 server_key = self.node.server_key
419 raise RuntimeError, "Failed to set up application %s: "\
420 "build failed, got wrong token from pid %s/%s "\
421 "(expected %r, got %r), see buildlog: %s" % (
422 self.home_path, pid, ppid, self._master_token, slave_token, buildlog)
424 def _do_kill_build(self):
425 pid = self._build_pid
426 ppid = self._build_ppid
431 host = self.node.hostname,
433 user = self.node.slicename,
435 ident_key = self.node.ident_path
439 def _do_build_master(self):
440 if not self.sources and not self.build and not self.buildDepends:
444 sources = self.sources.split(' ')
447 (out,err),proc = server.popen_scp(
449 "%s@%s:%s" % (self.node.slicename, self.node.hostname,
450 os.path.join(self.home_path,'.'),),
451 ident_key = self.node.ident_path,
452 server_key = self.node.server_key
456 raise RuntimeError, "Failed upload source file %r: %s %s" % (source, out,err,)
458 buildscript = cStringIO.StringIO()
460 if self.buildDepends:
461 # Install build dependencies
463 "sudo -S yum -y install %(packages)s\n" % {
464 'packages' : self.buildDepends
472 "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
473 'command' : self._replace_paths(self.build),
474 'home' : server.shell_escape(self.home_path),
480 "tar czf build.tar.gz build && ( echo %(master_token)s > build.token )\n" % {
481 'master_token' : server.shell_escape(self._master_token)
490 def _do_install(self):
492 # Install application
493 (out,err),proc = server.popen_ssh_command(
494 "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/installlog >&2 && false )" % {
495 'command' : self._replace_paths(self.install),
496 'home' : server.shell_escape(self.home_path),
498 host = self.node.hostname,
500 user = self.node.slicename,
502 ident_key = self.node.ident_path,
503 server_key = self.node.server_key
507 raise RuntimeError, "Failed instal build sources: %s %s" % (out,err,)
509 def set_master(self, master):
510 self._master = master
512 def install_keys(self, prk, puk, passphrase):
514 self._master_passphrase = passphrase
515 self._master_prk = prk
516 self._master_puk = puk
517 self._master_prk_name = os.path.basename(prk.name)
518 self._master_puk_name = os.path.basename(puk.name)
520 def _do_install_keys(self):
521 prk = self._master_prk
522 puk = self._master_puk
524 (out,err),proc = server.popen_scp(
525 [ prk.name, puk.name ],
526 '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path ),
529 ident_key = self.node.ident_path,
530 server_key = self.node.server_key
534 raise RuntimeError, "Failed to set up application deployment keys: %s %s" % (out,err,)
536 (out,err),proc = server.popen_scp(
537 cStringIO.StringIO('%s,%s %s\n' % (
538 self._master.node.hostname, socket.gethostbyname(self._master.node.hostname),
539 self._master.node.server_key)),
540 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
541 os.path.join(self.home_path,"master_known_hosts") ),
544 ident_key = self.node.ident_path,
545 server_key = self.node.server_key
549 raise RuntimeError, "Failed to set up application deployment keys: %s %s" % (out,err,)
552 self._master_prk = None
553 self._master_puk = None
556 # make sure there's no leftover build processes
557 self._do_kill_build()
560 class Application(Dependency):
562 An application also has dependencies, but also a command to be ran and monitored.
564 It adds the output of that command as traces.
567 TRACES = ('stdout','stderr','buildlog')
569 def __init__(self, api=None):
570 super(Application,self).__init__(api)
580 # Those are filled when the app is started
581 # Having both pid and ppid makes it harder
582 # for pid rollover to induce tracking mistakes
583 self._started = False
587 # Do not add to the python path of nodes
588 self.add_to_path = False
591 return "%s<command:%s%s>" % (
592 self.__class__.__name__,
593 "sudo " if self.sudo else "",
598 # Create shell script with the command
599 # This way, complex commands and scripts can be ran seamlessly
601 command = cStringIO.StringIO()
602 command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
603 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
605 command.write('export PATH=$PATH:%s\n' % (
606 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
609 for envkey, envvals in self.node.env.iteritems():
610 for envval in envvals:
611 command.write('export %s=%s\n' % (envkey, envval))
612 command.write(self.command)
615 (out,err),proc = server.popen_scp(
617 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
618 os.path.join(self.home_path, "app.sh")),
621 ident_key = self.node.ident_path,
622 server_key = self.node.server_key
626 raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
628 # Start process in a "daemonized" way, using nohup and heavy
629 # stdin/out redirection to avoid connection issues
630 (out,err),proc = rspawn.remote_spawn(
631 self._replace_paths("bash ./app.sh"),
634 home = self.home_path,
635 stdin = 'stdin' if self.stdin is not None else '/dev/null',
636 stdout = 'stdout' if self.stdout else '/dev/null',
637 stderr = 'stderr' if self.stderr else '/dev/null',
640 host = self.node.hostname,
642 user = self.node.slicename,
644 ident_key = self.node.ident_path,
645 server_key = self.node.server_key
649 raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
655 # NOTE: wait a bit for the pidfile to be created
656 if self._started and not self._pid or not self._ppid:
657 pidtuple = rspawn.remote_check_pid(
658 os.path.join(self.home_path,'pid'),
659 host = self.node.hostname,
661 user = self.node.slicename,
663 ident_key = self.node.ident_path,
664 server_key = self.node.server_key
668 self._pid, self._ppid = pidtuple
672 if not self._started:
673 return STATUS_NOT_STARTED
674 elif not self._pid or not self._ppid:
675 return STATUS_NOT_STARTED
677 status = rspawn.remote_status(
678 self._pid, self._ppid,
679 host = self.node.hostname,
681 user = self.node.slicename,
683 ident_key = self.node.ident_path,
684 server_key = self.node.server_key
687 if status is rspawn.NOT_STARTED:
688 return STATUS_NOT_STARTED
689 elif status is rspawn.RUNNING:
690 return STATUS_RUNNING
691 elif status is rspawn.FINISHED:
692 return STATUS_FINISHED
695 return STATUS_NOT_STARTED
698 status = self.status()
699 if status == STATUS_RUNNING:
700 # kill by ppid+pid - SIGTERM first, then try SIGKILL
702 self._pid, self._ppid,
703 host = self.node.hostname,
705 user = self.node.slicename,
707 ident_key = self.node.ident_path,
708 server_key = self.node.server_key
711 class NepiDependency(Dependency):
713 This dependency adds nepi itself to the python path,
714 so that you may run testbeds within PL nodes.
717 # Class attribute holding a *weak* reference to the shared NEPI tar file
718 # so that they may share it. Don't operate on the file itself, it would
719 # be a mess, just use its path.
720 _shared_nepi_tar = None
722 def __init__(self, api = None):
723 super(NepiDependency, self).__init__(api)
727 self.depends = 'python python-ipaddr python-setuptools'
729 # our sources are in our ad-hoc tarball
730 self.sources = self.tarball.name
732 tarname = os.path.basename(self.tarball.name)
734 # it's already built - just move the tarball into place
735 self.build = "mv -f ${SOURCES}/%s ." % (tarname,)
737 # unpack it into sources, and we're done
738 self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,)
742 if self._tarball is None:
743 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
744 if shared_tar is not None:
745 self._tarball = shared_tar
747 # Build an ad-hoc tarball
752 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
754 proc = subprocess.Popen(
755 ["tar", "czf", shared_tar.name,
756 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
758 stdout = open("/dev/null","w"),
759 stdin = open("/dev/null","r"))
762 raise RuntimeError, "Failed to create nepi tarball"
764 self._tarball = self._shared_nepi_tar = shared_tar
768 class NS3Dependency(Dependency):
770 This dependency adds NS3 libraries to the library paths,
771 so that you may run the NS3 testbed within PL nodes.
773 You'll also need the NepiDependency.
776 def __init__(self, api = None):
777 super(NS3Dependency, self).__init__(api)
779 self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip'
781 # We have to download the sources, untar, build...
782 pybindgen_source_url = "http://pybindgen.googlecode.com/files/pybindgen-0.15.0.zip"
783 pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
784 ns3_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/nepi-ns-3.9/archive/tip.tar.gz"
785 passfd_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/python-passfd/archive/tip.tar.gz"
789 " python -c 'import pygccxml, pybindgen, passfd' && "
790 " test -f lib/_ns3.so && "
791 " test -f lib/libns3.so "
793 # Not working, rebuild
794 "wget -q -c -O pybindgen-src.zip %(pybindgen_source_url)s && " # continue, to exploit the case when it has already been dl'ed
795 "wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
796 "wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
797 "wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
798 "unzip -n pybindgen-src.zip && " # Do not overwrite files, to exploit the case when it has already been built
799 "unzip -n pygccxml-1.0.0.zip && "
800 "mkdir -p ns3-src && "
801 "mkdir -p passfd-src && "
802 "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
803 "tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
804 "rm -rf target && " # mv doesn't like unclean targets
805 "mkdir -p target && "
806 "cd pygccxml-1.0.0 && "
807 "rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
808 "python setup.py build && "
809 "python setup.py install --install-lib ${BUILD}/target && "
810 "python setup.py clean && "
811 "cd ../pybindgen-0.15.0 && "
812 "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
813 "./waf configure --prefix=${BUILD}/target -d release && "
817 "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
818 "rm -rf ${BUILD}/target/lib && "
819 "cd ../passfd-src && "
820 "python setup.py build && "
821 "python setup.py install --install-lib ${BUILD}/target && "
822 "python setup.py clean && "
824 "./waf configure --prefix=${BUILD}/target -d release --disable-examples --high-precision-as-double && "
830 pybindgen_source_url = server.shell_escape(pybindgen_source_url),
831 pygccxml_source_url = server.shell_escape(pygccxml_source_url),
832 ns3_source_url = server.shell_escape(ns3_source_url),
833 passfd_source_url = server.shell_escape(passfd_source_url),
836 # Just move ${BUILD}/target
840 " python -c 'import pygccxml, pybindgen, passfd' && "
841 " test -f lib/_ns3.so && "
842 " test -f lib/libns3.so "
844 # Not working, reinstall
845 "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
846 "mv -f ${BUILD}/target/* ${SOURCES}"
850 # Set extra environment paths
851 self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
852 self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib/libns3.so"
856 if self._tarball is None:
857 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
858 if shared_tar is not None:
859 self._tarball = shared_tar
861 # Build an ad-hoc tarball
866 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
868 proc = subprocess.Popen(
869 ["tar", "czf", shared_tar.name,
870 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
872 stdout = open("/dev/null","w"),
873 stdin = open("/dev/null","r"))
876 raise RuntimeError, "Failed to create nepi tarball"
878 self._tarball = self._shared_nepi_tar = shared_tar