2 # -*- coding: utf-8 -*-
4 from constants import TESTBED_ID
10 import nepi.util.server as server
21 from nepi.util.constants import ApplicationStatus as AS
23 class Dependency(object):
25 A Dependency is in every respect like an application.
27 It depends on some packages, it may require building binaries, it must deploy
30 But it has no command. Dependencies aren't ever started, or stopped, and have
36 def __init__(self, api=None):
48 self.buildDepends = None
50 self.rpmFusion = False
58 self.add_to_path = True
60 # Those are filled when the app is configured
63 # Those are filled when an actual node is connected
66 # Those are filled when the app is started
67 # Having both pid and ppid makes it harder
68 # for pid rollover to induce tracking mistakes
75 # Spanning tree deployment
77 self._master_passphrase = None
78 self._master_prk = None
79 self._master_puk = None
80 self._master_token = os.urandom(8).encode("hex")
81 self._build_pid = None
82 self._build_ppid = None
85 self._logger = logging.getLogger('nepi.testbeds.planetlab')
90 self.__class__.__name__,
91 ' '.join(filter(bool,(self.depends, self.sources)))
95 if self.home_path is None:
96 raise AssertionError, "Misconfigured application: missing home path"
97 if self.node.ident_path is None or not os.access(self.node.ident_path, os.R_OK):
98 raise AssertionError, "Misconfigured application: missing slice SSH key"
100 raise AssertionError, "Misconfigured application: unconnected node"
101 if self.node.hostname is None:
102 raise AssertionError, "Misconfigured application: misconfigured node"
103 if self.node.slicename is None:
104 raise AssertionError, "Misconfigured application: unspecified slice"
106 def check_bad_host(self, out, err):
108 Called whenever an operation fails, it's given the output to be checked for
109 telltale signs of unhealthy hosts.
113 def remote_trace_path(self, whichtrace):
114 if whichtrace in self.TRACES:
115 tracefile = os.path.join(self.home_path, whichtrace)
121 def remote_trace_name(self, whichtrace):
122 if whichtrace in self.TRACES:
126 def sync_trace(self, local_dir, whichtrace):
127 tracefile = self.remote_trace_path(whichtrace)
131 local_path = os.path.join(local_dir, tracefile)
133 # create parent local folders
134 proc = subprocess.Popen(
135 ["mkdir", "-p", os.path.dirname(local_path)],
136 stdout = open("/dev/null","w"),
137 stdin = open("/dev/null","r"))
140 raise RuntimeError, "Failed to synchronize trace"
145 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
149 except RuntimeError, e:
150 raise RuntimeError, "Failed to synchronize trace: %s %s" \
151 % (e.args[0], e.args[1],)
156 # We assume a correct deployment, so recovery only
157 # means we mark this dependency as deployed
161 self._logger.info("Setting up %s", self)
167 def async_setup(self):
168 if not self._setuper:
173 self._setuper._exc.append(sys.exc_info())
174 self._setuper = threading.Thread(
176 self._setuper._exc = []
177 self._setuper.start()
179 def async_setup_wait(self):
181 self._logger.info("Waiting for %s to be setup", self)
185 if self._setuper._exc:
186 exctyp,exval,exctrace = self._setuper._exc[0]
187 raise exctyp,exval,exctrace
189 raise RuntimeError, "Failed to setup application"
191 self._logger.info("Setup ready: %s at %s", self, self.node.hostname)
195 def _make_home(self):
196 # Make sure all the paths are created where
197 # they have to be created for deployment
200 self._popen_ssh_command(
201 "mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" \
202 % { 'home' : server.shell_escape(self.home_path) },
206 except RuntimeError, e:
207 raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, e.args[0], e.args[1],)
210 # Write program input
213 cStringIO.StringIO(self.stdin),
214 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
215 os.path.join(self.home_path, 'stdin') ),
217 except RuntimeError, e:
218 raise RuntimeError, "Failed to set up application %s: %s %s" \
219 % (self.home_path, e.args[0], e.args[1],)
221 def _replace_paths(self, command):
223 Replace all special path tags with shell-escaped actual paths.
225 # need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
226 root = '' if self.home_path.startswith('/') else "${HOME}/"
228 .replace("${SOURCES}", root+server.shell_escape(self.home_path))
229 .replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
231 def _launch_build(self):
232 if self._master is not None:
233 self._do_install_keys()
234 buildscript = self._do_build_slave()
236 buildscript = self._do_build_master()
238 if buildscript is not None:
239 self._logger.info("Building %s at %s", self, self.node.hostname)
241 # upload build script
245 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
246 os.path.join(self.home_path, 'nepi-build.sh') )
248 except RuntimeError, e:
249 raise RuntimeError, "Failed to set up application %s: %s %s" \
250 % (self.home_path, e.args[0], e.args[1],)
253 self._do_launch_build()
255 def _finish_build(self):
256 self._do_wait_build()
259 def _do_build_slave(self):
260 if not self.sources and not self.build:
263 # Create build script
267 sources = self.sources.split(' ')
269 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
270 os.path.join(self._master.home_path, os.path.basename(source)),)
271 for source in sources
276 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
277 os.path.join(self._master.home_path, 'build.tar.gz'),)
280 sshopts = "-o ConnectTimeout=30 -o ConnectionAttempts=3 -o ServerAliveInterval=30 -o TCPKeepAlive=yes"
282 launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
283 " && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
284 " && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" % \
286 'prk' : server.shell_escape(self._master_prk_name),
287 'puk' : server.shell_escape(self._master_puk_name),
290 kill_agent = "kill $SSH_AGENT_PID"
294 "echo 'Checking master reachability' ; "
295 "if ping -c 3 %(master_host)s ; then "
296 "echo 'Master node reachable' ; "
298 "echo 'MASTER NODE UNREACHABLE' && "
301 ". ./.ssh-agent.sh ; "
302 "while [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s.retcode || /bin/true) != %(token)s ]] ; do sleep 5 ; done ; "
303 "if [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s || /bin/true) != %(token)s ]] ; then echo BAD TOKEN ; exit 1 ; fi ; "
306 'hostkey' : 'master_known_hosts',
307 'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostname),
308 'master_host' : self._master.node.hostname,
309 'token_path' : os.path.join(self._master.home_path, 'build.token'),
310 'token' : server.shell_escape(self._master._master_token),
314 syncfiles = ". ./.ssh-agent.sh && scp -p -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(files)s ." % {
315 'hostkey' : 'master_known_hosts',
316 'files' : ' '.join(files),
320 syncfiles += " && tar xzf build.tar.gz"
321 syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
322 syncfiles += " && ( echo %s > build.token.retcode )" % (server.shell_escape(self._master_token),)
323 syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
325 cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
326 'prk' : server.shell_escape(self._master_prk_name),
327 'puk' : server.shell_escape(self._master_puk_name),
330 slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s ) ; echo %(token)s > build.token.retcode" % {
331 'waitmaster' : waitmaster,
332 'syncfiles' : syncfiles,
334 'kill_agent' : kill_agent,
335 'launch_agent' : launch_agent,
336 'home' : server.shell_escape(self.home_path),
337 'token' : server.shell_escape(self._master_token),
340 return cStringIO.StringIO(slavescript)
342 def _do_launch_build(self):
343 script = "bash ./nepi-build.sh"
344 if self._master_passphrase:
345 script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
346 server.shell_escape(self._master_passphrase),
349 (out,err),proc = rspawn.remote_spawn(
351 pidfile = 'build-pid',
352 home = self.home_path,
355 stderr = rspawn.STDOUT,
357 host = self.node.hostname,
359 user = self.node.slicename,
361 ident_key = self.node.ident_path,
362 server_key = self.node.server_key
366 if self.check_bad_host(out, err):
367 self.node.blacklist()
368 raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
374 pidtuple = rspawn.remote_check_pid(
375 os.path.join(self.home_path,'build-pid'),
376 host = self.node.hostname,
378 user = self.node.slicename,
380 ident_key = self.node.ident_path,
381 server_key = self.node.server_key
386 self._build_pid, self._build_ppid = pidtuple
390 delay = min(30,delay*1.2)
392 raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
394 self._logger.info("Deploying %s at %s", self, self.node.hostname)
396 def _do_wait_build(self):
397 pid = self._build_pid
398 ppid = self._build_ppid
405 status = rspawn.remote_status(
407 host = self.node.hostname,
409 user = self.node.slicename,
411 ident_key = self.node.ident_path,
412 server_key = self.node.server_key
415 if status is rspawn.FINISHED:
416 self._build_pid = self._build_ppid = None
418 elif status is not rspawn.RUNNING:
420 time.sleep(delay*(5.5+random.random()))
422 self._build_pid = self._build_ppid = None
426 self._logger.info("Waiting for %s to finish building at %s %s", self, self.node.hostname,
427 "(build slave)" if self._master is not None else "(build master)")
430 time.sleep(delay*(0.5+random.random()))
431 delay = min(30,delay*1.2)
437 (out, err), proc = self._popen_ssh_command(
438 "cat %(token_path)s" % {
439 'token_path' : os.path.join(self.home_path, 'build.token'),
443 if not proc.wait() and out:
444 slave_token = out.strip()
451 if slave_token != self._master_token:
452 # Get buildlog for the error message
454 (buildlog, err), proc = self._popen_ssh_command(
455 "cat %(buildlog)s" % {
456 'buildlog' : os.path.join(self.home_path, 'buildlog'),
457 'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
464 if self.check_bad_host(buildlog, err):
465 self.node.blacklist()
467 raise RuntimeError, "Failed to set up application %s: "\
468 "build failed, got wrong token from pid %s/%s "\
469 "(expected %r, got %r), see buildlog at %s:\n%s" % (
470 self.home_path, pid, ppid, self._master_token, slave_token, self.node.hostname, buildlog)
472 self._logger.info("Built %s at %s", self, self.node.hostname)
474 def _do_kill_build(self):
475 pid = self._build_pid
476 ppid = self._build_ppid
479 self._logger.info("Killing build of %s", self)
482 host = self.node.hostname,
484 user = self.node.slicename,
486 ident_key = self.node.ident_path
490 def _do_build_master(self):
491 if not self.sources and not self.build and not self.buildDepends:
495 sources = self.sources.split(' ')
501 "%s@%s:%s" % (self.node.slicename, self.node.hostname,
502 os.path.join(self.home_path,'.'),)
504 except RuntimeError, e:
505 raise RuntimeError, "Failed upload source file %r: %s %s" \
506 % (sources, e.args[0], e.args[1],)
508 buildscript = cStringIO.StringIO()
510 buildscript.write("(\n")
512 if self.buildDepends:
513 # Install build dependencies
515 "sudo -S yum -y install %(packages)s\n" % {
516 'packages' : self.buildDepends
524 "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
525 'command' : self._replace_paths(self.build),
526 'home' : server.shell_escape(self.home_path),
531 buildscript.write("tar czf build.tar.gz build\n")
534 buildscript.write("echo %(master_token)s > build.token ) ; echo %(master_token)s > build.token.retcode" % {
535 'master_token' : server.shell_escape(self._master_token)
542 def _do_install(self):
544 self._logger.info("Installing %s at %s", self, self.node.hostname)
546 # Install application
548 self._popen_ssh_command(
549 "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/{install,build}log >&2 && false )" % \
551 'command' : self._replace_paths(self.install),
552 'home' : server.shell_escape(self.home_path),
555 except RuntimeError, e:
556 if self.check_bad_host(e.args[0], e.args[1]):
557 self.node.blacklist()
558 raise RuntimeError, "Failed install build sources: %s %s" % (e.args[0], e.args[1],)
560 def set_master(self, master):
561 self._master = master
563 def install_keys(self, prk, puk, passphrase):
565 self._master_passphrase = passphrase
566 self._master_prk = prk
567 self._master_puk = puk
568 self._master_prk_name = os.path.basename(prk.name)
569 self._master_puk_name = os.path.basename(puk.name)
571 def _do_install_keys(self):
572 prk = self._master_prk
573 puk = self._master_puk
577 [ prk.name, puk.name ],
578 '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path )
580 except RuntimeError, e:
581 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
582 % (e.args[0], e.args[1],)
586 cStringIO.StringIO('%s,%s %s\n' % (
587 self._master.node.hostname, socket.gethostbyname(self._master.node.hostname),
588 self._master.node.server_key)),
589 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
590 os.path.join(self.home_path,"master_known_hosts") )
592 except RuntimeError, e:
593 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
594 % (e.args[0], e.args[1],)
597 self._master_prk = None
598 self._master_puk = None
601 # make sure there's no leftover build processes
602 self._do_kill_build()
605 def _popen_scp(self, src, dst, retry = 3):
608 (out,err),proc = server.popen_scp(
613 ident_key = self.node.ident_path,
614 server_key = self.node.server_key
617 if server.eintr_retry(proc.wait)():
618 raise RuntimeError, (out, err)
619 return (out, err), proc
628 def _popen_ssh_command(self, command, retry = 0, noerrors=False, timeout=None):
629 (out,err),proc = server.popen_ssh_command(
631 host = self.node.hostname,
633 user = self.node.slicename,
635 ident_key = self.node.ident_path,
636 server_key = self.node.server_key,
641 if server.eintr_retry(proc.wait)():
643 raise RuntimeError, (out, err)
644 return (out, err), proc
646 class Application(Dependency):
648 An application also has dependencies, but also a command to be ran and monitored.
650 It adds the output of that command as traces.
653 TRACES = ('stdout','stderr','buildlog', 'output')
655 def __init__(self, api=None):
656 super(Application,self).__init__(api)
667 # Those are filled when the app is started
668 # Having both pid and ppid makes it harder
669 # for pid rollover to induce tracking mistakes
670 self._started = False
674 # Do not add to the python path of nodes
675 self.add_to_path = False
678 return "%s<command:%s%s>" % (
679 self.__class__.__name__,
680 "sudo " if self.sudo else "",
685 self._logger.info("Starting %s", self)
687 # Create shell script with the command
688 # This way, complex commands and scripts can be ran seamlessly
690 command = cStringIO.StringIO()
691 command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
692 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
694 command.write('export PATH=$PATH:%s\n' % (
695 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
698 for envkey, envvals in self.node.env.iteritems():
699 for envval in envvals:
700 command.write('export %s=%s\n' % (envkey, envval))
701 command.write(self.command)
707 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
708 os.path.join(self.home_path, "app.sh"))
710 except RuntimeError, e:
711 raise RuntimeError, "Failed to set up application: %s %s" \
712 % (e.args[0], e.args[1],)
714 # Start process in a "daemonized" way, using nohup and heavy
715 # stdin/out redirection to avoid connection issues
716 (out,err),proc = rspawn.remote_spawn(
717 self._replace_paths("bash ./app.sh"),
720 home = self.home_path,
721 stdin = 'stdin' if self.stdin is not None else '/dev/null',
722 stdout = 'stdout' if self.stdout else '/dev/null',
723 stderr = 'stderr' if self.stderr else '/dev/null',
726 host = self.node.hostname,
728 user = self.node.slicename,
730 ident_key = self.node.ident_path,
731 server_key = self.node.server_key
735 if self.check_bad_host(out, err):
736 self.node.blacklist()
737 raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
742 # Assuming the application is running on PlanetLab,
743 # proper pidfiles should be present at the app's home path.
744 # So we mark this application as started, and check the pidfiles
750 # NOTE: wait a bit for the pidfile to be created
751 if self._started and not self._pid or not self._ppid:
752 pidtuple = rspawn.remote_check_pid(
753 os.path.join(self.home_path,'pid'),
754 host = self.node.hostname,
756 user = self.node.slicename,
758 ident_key = self.node.ident_path,
759 server_key = self.node.server_key
763 self._pid, self._ppid = pidtuple
767 if not self._started:
768 return AS.STATUS_NOT_STARTED
769 elif not self._pid or not self._ppid:
770 return AS.STATUS_NOT_STARTED
772 status = rspawn.remote_status(
773 self._pid, self._ppid,
774 host = self.node.hostname,
776 user = self.node.slicename,
778 ident_key = self.node.ident_path,
779 server_key = self.node.server_key
782 if status is rspawn.NOT_STARTED:
783 return AS.STATUS_NOT_STARTED
784 elif status is rspawn.RUNNING:
785 return AS.STATUS_RUNNING
786 elif status is rspawn.FINISHED:
787 return AS.STATUS_FINISHED
790 return AS.STATUS_NOT_STARTED
793 status = self.status()
794 if status == AS.STATUS_RUNNING:
795 # kill by ppid+pid - SIGTERM first, then try SIGKILL
797 self._pid, self._ppid,
798 host = self.node.hostname,
800 user = self.node.slicename,
802 ident_key = self.node.ident_path,
803 server_key = self.node.server_key,
806 self._logger.info("Killed %s", self)
809 class NepiDependency(Dependency):
811 This dependency adds nepi itself to the python path,
812 so that you may run testbeds within PL nodes.
815 # Class attribute holding a *weak* reference to the shared NEPI tar file
816 # so that they may share it. Don't operate on the file itself, it would
817 # be a mess, just use its path.
818 _shared_nepi_tar = None
820 def __init__(self, api = None):
821 super(NepiDependency, self).__init__(api)
825 self.depends = 'python python-ipaddr python-setuptools'
827 # our sources are in our ad-hoc tarball
828 self.sources = self.tarball.name
830 tarname = os.path.basename(self.tarball.name)
832 # it's already built - just move the tarball into place
833 self.build = "mv -f ${SOURCES}/%s ." % (tarname,)
835 # unpack it into sources, and we're done
836 self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,)
840 if self._tarball is None:
841 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
842 if shared_tar is not None:
843 self._tarball = shared_tar
845 # Build an ad-hoc tarball
850 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
852 proc = subprocess.Popen(
853 ["tar", "czf", shared_tar.name,
854 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
856 stdout = open("/dev/null","w"),
857 stdin = open("/dev/null","r"))
860 raise RuntimeError, "Failed to create nepi tarball"
862 self._tarball = self._shared_nepi_tar = shared_tar
866 class NS3Dependency(Dependency):
868 This dependency adds NS3 libraries to the library paths,
869 so that you may run the NS3 testbed within PL nodes.
871 You'll also need the NepiDependency.
874 def __init__(self, api = None):
875 super(NS3Dependency, self).__init__(api)
877 self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip'
879 # We have to download the sources, untar, build...
880 pybindgen_source_url = "http://yans.pl.sophia.inria.fr/trac/nepi/raw-attachment/wiki/WikiStart/pybindgen-r794.tar.gz"
881 pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
882 ns3_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/ns-3.11-nepi/archive/tip.tar.gz"
883 passfd_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/python-passfd/archive/tip.tar.gz"
887 " python -c 'import pygccxml, pybindgen, passfd' && "
888 " test -f lib/ns/_core.so && "
889 " test -f lib/ns/__init__.py && "
890 " test -f lib/ns/core.py && "
891 " test -f lib/libns3-core.so && "
892 " LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
894 # Not working, rebuild
895 # Archive SHA1 sums to check
896 "echo '7158877faff2254e6c094bf18e6b4283cac19137 pygccxml-1.0.0.zip' > archive_sums.txt && "
897 "echo 'a18c2ccffd0df517bc37e2f3a2475092517c43f2 pybindgen-src.tar.gz' >> archive_sums.txt && "
898 " ( " # check existing files
899 " sha1sum -c archive_sums.txt && "
900 " test -f passfd-src.tar.gz && "
901 " test -f ns3-src.tar.gz "
902 " ) || ( " # nope? re-download
903 " rm -f pybindgen-src.zip pygccxml-1.0.0.zip passfd-src.tar.gz ns3-src.tar.gz && "
904 " wget -q -c -O pybindgen-src.tar.gz %(pybindgen_source_url)s && " # continue, to exploit the case when it has already been dl'ed
905 " wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
906 " wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
907 " wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
908 " sha1sum -c archive_sums.txt " # Check SHA1 sums when applicable
910 "unzip -n pygccxml-1.0.0.zip && "
911 "mkdir -p pybindgen-src && "
912 "mkdir -p ns3-src && "
913 "mkdir -p passfd-src && "
914 "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
915 "tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
916 "tar xzf pybindgen-src.tar.gz --strip-components=1 -C pybindgen-src && "
917 "rm -rf target && " # mv doesn't like unclean targets
918 "mkdir -p target && "
919 "cd pygccxml-1.0.0 && "
920 "rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
921 "python setup.py build && "
922 "python setup.py install --install-lib ${BUILD}/target && "
923 "python setup.py clean && "
924 "cd ../pybindgen-src && "
925 "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
926 "./waf configure --prefix=${BUILD}/target -d release && "
930 "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
931 "rm -rf ${BUILD}/target/lib && "
932 "cd ../passfd-src && "
933 "python setup.py build && "
934 "python setup.py install --install-lib ${BUILD}/target && "
935 "python setup.py clean && "
937 "./waf configure --prefix=${BUILD}/target --with-pybindgen=../pybindgen-src -d release --disable-examples --disable-tests && "
940 "rm -f ${BUILD}/target/lib/*.so && "
941 "cp -a ${BUILD}/ns3-src/build/release/libns3*.so ${BUILD}/target/lib && "
942 "cp -a ${BUILD}/ns3-src/build/release/bindings/python/ns ${BUILD}/target/lib &&"
946 pybindgen_source_url = server.shell_escape(pybindgen_source_url),
947 pygccxml_source_url = server.shell_escape(pygccxml_source_url),
948 ns3_source_url = server.shell_escape(ns3_source_url),
949 passfd_source_url = server.shell_escape(passfd_source_url),
952 # Just move ${BUILD}/target
956 " python -c 'import pygccxml, pybindgen, passfd' && "
957 " test -f lib/ns/_core.so && "
958 " test -f lib/ns/__init__.py && "
959 " test -f lib/ns/core.py && "
960 " test -f lib/libns3-core.so && "
961 " LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
963 # Not working, reinstall
964 "test -d ${BUILD}/target && "
965 "[[ \"x\" != \"x$(find ${BUILD}/target -mindepth 1 -print -quit)\" ]] &&"
966 "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
967 "mv -f ${BUILD}/target/* ${SOURCES}"
971 # Set extra environment paths
972 self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
973 self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib"
977 if self._tarball is None:
978 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
979 if shared_tar is not None:
980 self._tarball = shared_tar
982 # Build an ad-hoc tarball
987 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
989 proc = subprocess.Popen(
990 ["tar", "czf", shared_tar.name,
991 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
993 stdout = open("/dev/null","w"),
994 stdin = open("/dev/null","r"))
997 raise RuntimeError, "Failed to create nepi tarball"
999 self._tarball = self._shared_nepi_tar = shared_tar
1001 return self._tarball
1003 class YumDependency(Dependency):
1005 This dependency is an internal helper class used to
1006 efficiently distribute yum-downloaded rpms.
1008 It temporarily sets the yum cache as persistent in the
1009 build master, and installs all the required packages.
1011 The rpm packages left in the yum cache are gathered and
1012 distributed by the underlying Dependency in an efficient
1013 manner. Build slaves will then install those rpms back in
1014 the cache before issuing the install command.
1016 When packages have been installed already, nothing but an
1017 empty tar is distributed.
1020 # Class attribute holding a *weak* reference to the shared NEPI tar file
1021 # so that they may share it. Don't operate on the file itself, it would
1022 # be a mess, just use its path.
1023 _shared_nepi_tar = None
1025 def _build_get(self):
1026 # canonical representation of dependencies
1027 depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
1029 # download rpms and pack into a tar archive
1031 "sudo -S nice yum -y makecache && "
1032 "sudo -S sed -i -r 's/keepcache *= *0/keepcache=1/' /etc/yum.conf && "
1034 "sudo -S nice yum -y install %s ; "
1035 "rm -f ${BUILD}/packages.tar ; "
1036 "tar -C /var/cache/yum -rf ${BUILD}/packages.tar $(cd /var/cache/yum ; find -iname '*.rpm')"
1037 " ) || /bin/true ) && "
1038 "sudo -S sed -i -r 's/keepcache *= *1/keepcache=0/' /etc/yum.conf && "
1039 "( sudo -S nice yum -y clean packages || /bin/true ) "
1041 def _build_set(self, value):
1044 build = property(_build_get, _build_set)
1046 def _install_get(self):
1047 # canonical representation of dependencies
1048 depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
1050 # unpack cached rpms into yum cache, install, and cleanup
1052 "sudo -S tar -k --keep-newer-files -C /var/cache/yum -xf packages.tar && "
1053 "sudo -S nice yum -y install %s && "
1054 "( sudo -S nice yum -y clean packages || /bin/true ) "
1056 def _install_set(self, value):
1059 install = property(_install_get, _install_set)
1061 def check_bad_host(self, out, err):
1062 badre = re.compile(r'(?:'
1063 r'The GPG keys listed for the ".*" repository are already installed but they are not correct for this package'
1064 r'|Error: Cannot retrieve repository metadata (repomd.xml) for repository: .*[.] Please verify its path and try again'
1065 r'|Error: disk I/O error'
1066 r'|MASTER NODE UNREACHABLE'
1069 return badre.search(out) or badre.search(err)