2 # -*- coding: utf-8 -*-
4 from constants import TESTBED_ID
10 import nepi.util.server as server
19 from nepi.util.constants import STATUS_NOT_STARTED, STATUS_RUNNING, \
22 class Dependency(object):
24 A Dependency is in every respect like an application.
26 It depends on some packages, it may require building binaries, it must deploy
29 But it has no command. Dependencies aren't ever started, or stopped, and have
35 def __init__(self, api=None):
47 self.buildDepends = None
49 self.rpmFusion = False
57 self.add_to_path = True
59 # Those are filled when the app is configured
62 # Those are filled when an actual node is connected
65 # Those are filled when the app is started
66 # Having both pid and ppid makes it harder
67 # for pid rollover to induce tracking mistakes
74 # Spanning tree deployment
76 self._master_passphrase = None
77 self._master_prk = None
78 self._master_puk = None
79 self._master_token = ''.join(map(chr,[rng.randint(0,255)
80 for rng in (random.SystemRandom(),)
81 for i in xrange(8)] )).encode("hex")
82 self._build_pid = None
83 self._build_ppid = None
88 self.__class__.__name__,
89 ' '.join(filter(bool,(self.depends, self.sources)))
93 if self.home_path is None:
94 raise AssertionError, "Misconfigured application: missing home path"
95 if self.node.ident_path is None or not os.access(self.node.ident_path, os.R_OK):
96 raise AssertionError, "Misconfigured application: missing slice SSH key"
98 raise AssertionError, "Misconfigured application: unconnected node"
99 if self.node.hostname is None:
100 raise AssertionError, "Misconfigured application: misconfigured node"
101 if self.node.slicename is None:
102 raise AssertionError, "Misconfigured application: unspecified slice"
104 def remote_trace_path(self, whichtrace):
105 if whichtrace in self.TRACES:
106 tracefile = os.path.join(self.home_path, whichtrace)
112 def sync_trace(self, local_dir, whichtrace):
113 tracefile = self.remote_trace_path(whichtrace)
117 local_path = os.path.join(local_dir, tracefile)
119 # create parent local folders
120 proc = subprocess.Popen(
121 ["mkdir", "-p", os.path.dirname(local_path)],
122 stdout = open("/dev/null","w"),
123 stdin = open("/dev/null","r"))
126 raise RuntimeError, "Failed to synchronize trace"
131 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
135 except RuntimeError, e:
136 raise RuntimeError, "Failed to synchronize trace: %s %s" \
137 % (e.args[0], e.args[1],)
142 print >>sys.stderr, "Setting up", self
148 def async_setup(self):
149 if not self._setuper:
154 self._setuper._exc.append(sys.exc_info())
155 self._setuper = threading.Thread(
157 self._setuper._exc = []
158 self._setuper.start()
160 def async_setup_wait(self):
162 print >>sys.stderr, "Waiting for", self, "to be setup"
166 if self._setuper._exc:
167 exctyp,exval,exctrace = self._setuper._exc[0]
168 raise exctyp,exval,exctrace
170 raise RuntimeError, "Failed to setup application"
174 def _make_home(self):
175 # Make sure all the paths are created where
176 # they have to be created for deployment
179 self._popen_ssh_command(
180 "mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" \
181 % { 'home' : server.shell_escape(self.home_path) }
183 except RuntimeError, e:
184 raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, e.args[0], e.args[1],)
187 # Write program input
190 cStringIO.StringIO(self.stdin),
191 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
192 os.path.join(self.home_path, 'stdin') ),
194 except RuntimeError, e:
195 raise RuntimeError, "Failed to set up application %s: %s %s" \
196 % (self.home_path, e.args[0], e.args[1],)
198 def _replace_paths(self, command):
200 Replace all special path tags with shell-escaped actual paths.
202 # need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
203 root = '' if self.home_path.startswith('/') else "${HOME}/"
205 .replace("${SOURCES}", root+server.shell_escape(self.home_path))
206 .replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
208 def _launch_build(self):
209 if self._master is not None:
210 self._do_install_keys()
211 buildscript = self._do_build_slave()
213 buildscript = self._do_build_master()
215 if buildscript is not None:
216 print "Building", self
218 # upload build script
222 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
223 os.path.join(self.home_path, 'nepi-build.sh') )
225 except RuntimeError, e:
226 raise RuntimeError, "Failed to set up application %s: %s %s" \
227 % (self.home_path, e.args[0], e.args[1],)
230 self._do_launch_build()
232 def _finish_build(self):
233 self._do_wait_build()
236 def _do_build_slave(self):
237 if not self.sources and not self.build:
240 # Create build script
244 sources = self.sources.split(' ')
246 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
247 os.path.join(self._master.home_path, os.path.basename(source)),)
248 for source in sources
253 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
254 os.path.join(self._master.home_path, 'build.tar.gz'),)
257 launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
258 " && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
259 " && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" % \
261 'prk' : server.shell_escape(self._master_prk_name),
262 'puk' : server.shell_escape(self._master_puk_name),
265 kill_agent = "kill $SSH_AGENT_PID"
267 waitmaster = "{ . ./.ssh-agent.sh ; while [[ $(ssh -q -o UserKnownHostsFile=%(hostkey)s %(master)s cat %(token_path)s) != %(token)s ]] ; do sleep 5 ; done ; }" % {
268 'hostkey' : 'master_known_hosts',
269 'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostname),
270 'token_path' : os.path.join(self._master.home_path, 'build.token'),
271 'token' : server.shell_escape(self._master._master_token),
274 syncfiles = "scp -p -o UserKnownHostsFile=%(hostkey)s %(files)s ." % {
275 'hostkey' : 'master_known_hosts',
276 'files' : ' '.join(files),
279 syncfiles += " && tar xzf build.tar.gz"
280 syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
281 syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
283 cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
284 'prk' : server.shell_escape(self._master_prk_name),
285 'puk' : server.shell_escape(self._master_puk_name),
288 slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s )" % {
289 'waitmaster' : waitmaster,
290 'syncfiles' : syncfiles,
292 'kill_agent' : kill_agent,
293 'launch_agent' : launch_agent,
294 'home' : server.shell_escape(self.home_path),
297 return cStringIO.StringIO(slavescript)
299 def _do_launch_build(self):
300 script = "bash ./nepi-build.sh"
301 if self._master_passphrase:
302 script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
303 server.shell_escape(self._master_passphrase),
306 (out,err),proc = rspawn.remote_spawn(
308 pidfile = 'build-pid',
309 home = self.home_path,
312 stderr = rspawn.STDOUT,
314 host = self.node.hostname,
316 user = self.node.slicename,
318 ident_key = self.node.ident_path,
319 server_key = self.node.server_key
323 raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
329 pidtuple = rspawn.remote_check_pid(
330 os.path.join(self.home_path,'build-pid'),
331 host = self.node.hostname,
333 user = self.node.slicename,
335 ident_key = self.node.ident_path,
336 server_key = self.node.server_key
341 self._build_pid, self._build_ppid = pidtuple
345 delay = min(30,delay*1.2)
347 raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
349 print >>sys.stderr, "Deploying", self
351 def _do_wait_build(self):
352 pid = self._build_pid
353 ppid = self._build_ppid
359 status = rspawn.remote_status(
361 host = self.node.hostname,
363 user = self.node.slicename,
365 ident_key = self.node.ident_path,
366 server_key = self.node.server_key
369 if status is not rspawn.RUNNING:
370 self._build_pid = self._build_ppid = None
374 print >>sys.stderr, "Waiting for", self, "to finish building",
375 if self._master is not None:
376 print >>sys.stderr, "(build master)"
378 print >>sys.stderr, "(build slave)"
381 time.sleep(delay*(0.5+random.random()))
382 delay = min(30,delay*1.2)
385 (out, err), proc = self._popen_ssh_command(
386 "cat %(token_path)s" % {
387 'token_path' : os.path.join(self.home_path, 'build.token'),
391 if not proc.wait() and out:
392 slave_token = out.strip()
394 if slave_token != self._master_token:
395 # Get buildlog for the error message
397 (buildlog, err), proc = self._popen_ssh_command(
398 "cat %(buildlog)s" % {
399 'buildlog' : os.path.join(self.home_path, 'buildlog'),
400 'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
406 raise RuntimeError, "Failed to set up application %s: "\
407 "build failed, got wrong token from pid %s/%s "\
408 "(expected %r, got %r), see buildlog: %s" % (
409 self.home_path, pid, ppid, self._master_token, slave_token, buildlog)
411 print >>sys.stderr, "Built", self
413 def _do_kill_build(self):
414 pid = self._build_pid
415 ppid = self._build_ppid
418 print >>sys.stderr, "Killing build of", self
421 host = self.node.hostname,
423 user = self.node.slicename,
425 ident_key = self.node.ident_path
429 def _do_build_master(self):
430 if not self.sources and not self.build and not self.buildDepends:
434 sources = self.sources.split(' ')
440 "%s@%s:%s" % (self.node.slicename, self.node.hostname,
441 os.path.join(self.home_path,'.'),)
443 except RuntimeError, e:
444 raise RuntimeError, "Failed upload source file %r: %s %s" \
445 % (sources, e.args[0], e.args[1],)
447 buildscript = cStringIO.StringIO()
449 if self.buildDepends:
450 # Install build dependencies
452 "sudo -S yum -y install %(packages)s\n" % {
453 'packages' : self.buildDepends
461 "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
462 'command' : self._replace_paths(self.build),
463 'home' : server.shell_escape(self.home_path),
468 buildscript.write("tar czf build.tar.gz build\n")
471 buildscript.write("echo %(master_token)s > build.token" % {
472 'master_token' : server.shell_escape(self._master_token)
479 def _do_install(self):
481 print >>sys.stderr, "Installing", self
483 # Install application
485 self._popen_ssh_command(
486 "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/installlog >&2 && false )" % \
488 'command' : self._replace_paths(self.install),
489 'home' : server.shell_escape(self.home_path),
492 except RuntimeError, e:
493 raise RuntimeError, "Failed install build sources: %s %s" % (e.args[0], e.args[1],)
495 def set_master(self, master):
496 self._master = master
498 def install_keys(self, prk, puk, passphrase):
500 self._master_passphrase = passphrase
501 self._master_prk = prk
502 self._master_puk = puk
503 self._master_prk_name = os.path.basename(prk.name)
504 self._master_puk_name = os.path.basename(puk.name)
506 def _do_install_keys(self):
507 prk = self._master_prk
508 puk = self._master_puk
512 [ prk.name, puk.name ],
513 '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path )
515 except RuntimeError, e:
516 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
517 % (e.args[0], e.args[1],)
521 cStringIO.StringIO('%s,%s %s\n' % (
522 self._master.node.hostname, socket.gethostbyname(self._master.node.hostname),
523 self._master.node.server_key)),
524 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
525 os.path.join(self.home_path,"master_known_hosts") )
527 except RuntimeError, e:
528 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
529 % (e.args[0], e.args[1],)
532 self._master_prk = None
533 self._master_puk = None
536 # make sure there's no leftover build processes
537 self._do_kill_build()
540 def _popen_scp(self, src, dst, retry = True):
541 (out,err),proc = server.popen_scp(
546 ident_key = self.node.ident_path,
547 server_key = self.node.server_key
550 if server.eintr_retry(proc.wait)():
551 raise RuntimeError, (out, err)
552 return (out, err), proc
556 def _popen_ssh_command(self, command, retry = True, noerrors=False):
557 (out,err),proc = server.popen_ssh_command(
559 host = self.node.hostname,
561 user = self.node.slicename,
563 ident_key = self.node.ident_path,
564 server_key = self.node.server_key
567 if server.eintr_retry(proc.wait)():
569 raise RuntimeError, (out, err)
570 return (out, err), proc
572 class Application(Dependency):
574 An application also has dependencies, but also a command to be ran and monitored.
576 It adds the output of that command as traces.
579 TRACES = ('stdout','stderr','buildlog')
581 def __init__(self, api=None):
582 super(Application,self).__init__(api)
592 # Those are filled when the app is started
593 # Having both pid and ppid makes it harder
594 # for pid rollover to induce tracking mistakes
595 self._started = False
599 # Do not add to the python path of nodes
600 self.add_to_path = False
603 return "%s<command:%s%s>" % (
604 self.__class__.__name__,
605 "sudo " if self.sudo else "",
610 print >>sys.stderr, "Starting", self
612 # Create shell script with the command
613 # This way, complex commands and scripts can be ran seamlessly
615 command = cStringIO.StringIO()
616 command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
617 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
619 command.write('export PATH=$PATH:%s\n' % (
620 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
623 for envkey, envvals in self.node.env.iteritems():
624 for envval in envvals:
625 command.write('export %s=%s\n' % (envkey, envval))
626 command.write(self.command)
632 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
633 os.path.join(self.home_path, "app.sh"))
635 except RuntimeError, e:
636 raise RuntimeError, "Failed to set up application: %s %s" \
637 % (e.args[0], e.args[1],)
639 # Start process in a "daemonized" way, using nohup and heavy
640 # stdin/out redirection to avoid connection issues
641 (out,err),proc = rspawn.remote_spawn(
642 self._replace_paths("bash ./app.sh"),
645 home = self.home_path,
646 stdin = 'stdin' if self.stdin is not None else '/dev/null',
647 stdout = 'stdout' if self.stdout else '/dev/null',
648 stderr = 'stderr' if self.stderr else '/dev/null',
651 host = self.node.hostname,
653 user = self.node.slicename,
655 ident_key = self.node.ident_path,
656 server_key = self.node.server_key
660 raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
666 # NOTE: wait a bit for the pidfile to be created
667 if self._started and not self._pid or not self._ppid:
668 pidtuple = rspawn.remote_check_pid(
669 os.path.join(self.home_path,'pid'),
670 host = self.node.hostname,
672 user = self.node.slicename,
674 ident_key = self.node.ident_path,
675 server_key = self.node.server_key
679 self._pid, self._ppid = pidtuple
683 if not self._started:
684 return STATUS_NOT_STARTED
685 elif not self._pid or not self._ppid:
686 return STATUS_NOT_STARTED
688 status = rspawn.remote_status(
689 self._pid, self._ppid,
690 host = self.node.hostname,
692 user = self.node.slicename,
694 ident_key = self.node.ident_path,
695 server_key = self.node.server_key
698 if status is rspawn.NOT_STARTED:
699 return STATUS_NOT_STARTED
700 elif status is rspawn.RUNNING:
701 return STATUS_RUNNING
702 elif status is rspawn.FINISHED:
703 return STATUS_FINISHED
706 return STATUS_NOT_STARTED
709 status = self.status()
710 if status == STATUS_RUNNING:
711 # kill by ppid+pid - SIGTERM first, then try SIGKILL
713 self._pid, self._ppid,
714 host = self.node.hostname,
716 user = self.node.slicename,
718 ident_key = self.node.ident_path,
719 server_key = self.node.server_key
721 print >>sys.stderr, "Killed", self
724 class NepiDependency(Dependency):
726 This dependency adds nepi itself to the python path,
727 so that you may run testbeds within PL nodes.
730 # Class attribute holding a *weak* reference to the shared NEPI tar file
731 # so that they may share it. Don't operate on the file itself, it would
732 # be a mess, just use its path.
733 _shared_nepi_tar = None
735 def __init__(self, api = None):
736 super(NepiDependency, self).__init__(api)
740 self.depends = 'python python-ipaddr python-setuptools'
742 # our sources are in our ad-hoc tarball
743 self.sources = self.tarball.name
745 tarname = os.path.basename(self.tarball.name)
747 # it's already built - just move the tarball into place
748 self.build = "mv -f ${SOURCES}/%s ." % (tarname,)
750 # unpack it into sources, and we're done
751 self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,)
755 if self._tarball is None:
756 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
757 if shared_tar is not None:
758 self._tarball = shared_tar
760 # Build an ad-hoc tarball
765 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
767 proc = subprocess.Popen(
768 ["tar", "czf", shared_tar.name,
769 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
771 stdout = open("/dev/null","w"),
772 stdin = open("/dev/null","r"))
775 raise RuntimeError, "Failed to create nepi tarball"
777 self._tarball = self._shared_nepi_tar = shared_tar
781 class NS3Dependency(Dependency):
783 This dependency adds NS3 libraries to the library paths,
784 so that you may run the NS3 testbed within PL nodes.
786 You'll also need the NepiDependency.
789 def __init__(self, api = None):
790 super(NS3Dependency, self).__init__(api)
792 self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip'
794 # We have to download the sources, untar, build...
795 pybindgen_source_url = "http://pybindgen.googlecode.com/files/pybindgen-0.15.0.zip"
796 pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
797 ns3_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/nepi-ns-3.9/archive/tip.tar.gz"
798 passfd_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/python-passfd/archive/tip.tar.gz"
802 " python -c 'import pygccxml, pybindgen, passfd' && "
803 " test -f lib/_ns3.so && "
804 " test -f lib/libns3.so "
806 # Not working, rebuild
807 "wget -q -c -O pybindgen-src.zip %(pybindgen_source_url)s && " # continue, to exploit the case when it has already been dl'ed
808 "wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
809 "wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
810 "wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
811 "unzip -n pybindgen-src.zip && " # Do not overwrite files, to exploit the case when it has already been built
812 "unzip -n pygccxml-1.0.0.zip && "
813 "mkdir -p ns3-src && "
814 "mkdir -p passfd-src && "
815 "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
816 "tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
817 "rm -rf target && " # mv doesn't like unclean targets
818 "mkdir -p target && "
819 "cd pygccxml-1.0.0 && "
820 "rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
821 "python setup.py build && "
822 "python setup.py install --install-lib ${BUILD}/target && "
823 "python setup.py clean && "
824 "cd ../pybindgen-0.15.0 && "
825 "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
826 "./waf configure --prefix=${BUILD}/target -d release && "
830 "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
831 "rm -rf ${BUILD}/target/lib && "
832 "cd ../passfd-src && "
833 "python setup.py build && "
834 "python setup.py install --install-lib ${BUILD}/target && "
835 "python setup.py clean && "
837 "./waf configure --prefix=${BUILD}/target -d release --disable-examples --high-precision-as-double && "
843 pybindgen_source_url = server.shell_escape(pybindgen_source_url),
844 pygccxml_source_url = server.shell_escape(pygccxml_source_url),
845 ns3_source_url = server.shell_escape(ns3_source_url),
846 passfd_source_url = server.shell_escape(passfd_source_url),
849 # Just move ${BUILD}/target
853 " python -c 'import pygccxml, pybindgen, passfd' && "
854 " test -f lib/_ns3.so && "
855 " test -f lib/libns3.so "
857 # Not working, reinstall
858 "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
859 "mv -f ${BUILD}/target/* ${SOURCES}"
863 # Set extra environment paths
864 self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
865 self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib/libns3.so"
869 if self._tarball is None:
870 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
871 if shared_tar is not None:
872 self._tarball = shared_tar
874 # Build an ad-hoc tarball
879 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
881 proc = subprocess.Popen(
882 ["tar", "czf", shared_tar.name,
883 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
885 stdout = open("/dev/null","w"),
886 stdin = open("/dev/null","r"))
889 raise RuntimeError, "Failed to create nepi tarball"
891 self._tarball = self._shared_nepi_tar = shared_tar