2 # -*- coding: utf-8 -*-
4 from constants import TESTBED_ID
10 import nepi.util.server as server
21 from nepi.util.constants import ApplicationStatus as AS
23 class Dependency(object):
25 A Dependency is in every respect like an application.
27 It depends on some packages, it may require building binaries, it must deploy
30 But it has no command. Dependencies aren't ever started, or stopped, and have
36 def __init__(self, api=None):
48 self.buildDepends = None
50 self.rpmFusion = False
58 self.add_to_path = True
60 # Those are filled when the app is configured
63 # Those are filled when an actual node is connected
66 # Those are filled when the app is started
67 # Having both pid and ppid makes it harder
68 # for pid rollover to induce tracking mistakes
75 # Spanning tree deployment
77 self._master_passphrase = None
78 self._master_prk = None
79 self._master_puk = None
80 self._master_token = os.urandom(8).encode("hex")
81 self._build_pid = None
82 self._build_ppid = None
85 self._logger = logging.getLogger('nepi.testbeds.planetlab')
90 self.__class__.__name__,
91 ' '.join(filter(bool,(self.depends, self.sources)))
95 if self.home_path is None:
96 raise AssertionError, "Misconfigured application: missing home path"
97 if self.node.ident_path is None or not os.access(self.node.ident_path, os.R_OK):
98 raise AssertionError, "Misconfigured application: missing slice SSH key"
100 raise AssertionError, "Misconfigured application: unconnected node"
101 if self.node.hostname is None:
102 raise AssertionError, "Misconfigured application: misconfigured node"
103 if self.node.slicename is None:
104 raise AssertionError, "Misconfigured application: unspecified slice"
106 def check_bad_host(self, out, err):
108 Called whenever an operation fails, it's given the output to be checked for
109 telltale signs of unhealthy hosts.
113 def remote_trace_path(self, whichtrace):
114 if whichtrace in self.TRACES:
115 tracefile = os.path.join(self.home_path, whichtrace)
121 def remote_trace_name(self, whichtrace):
122 if whichtrace in self.TRACES:
126 def sync_trace(self, local_dir, whichtrace):
127 tracefile = self.remote_trace_path(whichtrace)
131 local_path = os.path.join(local_dir, tracefile)
133 # create parent local folders
134 proc = subprocess.Popen(
135 ["mkdir", "-p", os.path.dirname(local_path)],
136 stdout = open("/dev/null","w"),
137 stdin = open("/dev/null","r"))
140 raise RuntimeError, "Failed to synchronize trace"
145 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
149 except RuntimeError, e:
150 raise RuntimeError, "Failed to synchronize trace: %s %s" \
151 % (e.args[0], e.args[1],)
156 # We assume a correct deployment, so recovery only
157 # means we mark this dependency as deployed
161 self._logger.info("Setting up %s", self)
167 def async_setup(self):
168 if not self._setuper:
173 self._setuper._exc.append(sys.exc_info())
174 self._setuper = threading.Thread(
176 self._setuper._exc = []
177 self._setuper.start()
179 def async_setup_wait(self):
181 self._logger.info("Waiting for %s to be setup", self)
185 if self._setuper._exc:
186 exctyp,exval,exctrace = self._setuper._exc[0]
187 raise exctyp,exval,exctrace
189 raise RuntimeError, "Failed to setup application"
191 self._logger.info("Setup ready: %s at %s", self, self.node.hostname)
195 def _make_home(self):
196 # Make sure all the paths are created where
197 # they have to be created for deployment
200 self._popen_ssh_command(
201 "mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" \
202 % { 'home' : server.shell_escape(self.home_path) },
206 except RuntimeError, e:
207 raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, e.args[0], e.args[1],)
210 # Write program input
213 cStringIO.StringIO(self.stdin),
214 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
215 os.path.join(self.home_path, 'stdin') ),
217 except RuntimeError, e:
218 raise RuntimeError, "Failed to set up application %s: %s %s" \
219 % (self.home_path, e.args[0], e.args[1],)
221 def _replace_paths(self, command):
223 Replace all special path tags with shell-escaped actual paths.
225 # need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
226 root = '' if self.home_path.startswith('/') else "${HOME}/"
228 .replace("${SOURCES}", root+server.shell_escape(self.home_path))
229 .replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
231 def _launch_build(self, trial=0):
232 if self._master is not None:
233 if not trial or self._master_prk is not None:
234 self._do_install_keys()
235 buildscript = self._do_build_slave()
237 buildscript = self._do_build_master()
239 if buildscript is not None:
240 self._logger.info("Building %s at %s", self, self.node.hostname)
242 # upload build script
246 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
247 os.path.join(self.home_path, 'nepi-build.sh') )
249 except RuntimeError, e:
250 raise RuntimeError, "Failed to set up application %s: %s %s" \
251 % (self.home_path, e.args[0], e.args[1],)
254 self._do_launch_build()
256 def _finish_build(self):
257 self._do_wait_build()
260 def _do_build_slave(self):
261 if not self.sources and not self.build:
264 # Create build script
268 sources = self.sources.split(' ')
270 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
271 os.path.join(self._master.home_path, os.path.basename(source)),)
272 for source in sources
277 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
278 os.path.join(self._master.home_path, 'build.tar.gz'),)
281 sshopts = "-o ConnectTimeout=30 -o ConnectionAttempts=3 -o ServerAliveInterval=30 -o TCPKeepAlive=yes"
283 launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
284 " && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
285 " && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" % \
287 'prk' : server.shell_escape(self._master_prk_name),
288 'puk' : server.shell_escape(self._master_puk_name),
291 kill_agent = "kill $SSH_AGENT_PID"
295 "echo 'Checking master reachability' ; "
296 "if ping -c 3 %(master_host)s && (. ./.ssh-agent.sh > /dev/null ; ssh -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s echo MASTER SAYS HI ) ; then "
297 "echo 'Master node reachable' ; "
299 "echo 'MASTER NODE UNREACHABLE' && "
302 ". ./.ssh-agent.sh ; "
303 "while [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s.retcode || /bin/true) != %(token)s ]] ; do sleep 5 ; done ; "
304 "if [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s || /bin/true) != %(token)s ]] ; then echo BAD TOKEN ; exit 1 ; fi ; "
307 'hostkey' : 'master_known_hosts',
308 'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostname),
309 'master_host' : self._master.node.hostname,
310 'token_path' : os.path.join(self._master.home_path, 'build.token'),
311 'token' : server.shell_escape(self._master._master_token),
315 syncfiles = ". ./.ssh-agent.sh && scp -p -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(files)s ." % {
316 'hostkey' : 'master_known_hosts',
317 'files' : ' '.join(files),
321 syncfiles += " && tar xzf build.tar.gz"
322 syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
323 syncfiles += " && ( echo %s > build.token.retcode )" % (server.shell_escape(self._master_token),)
324 syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
326 cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
327 'prk' : server.shell_escape(self._master_prk_name),
328 'puk' : server.shell_escape(self._master_puk_name),
331 slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s ) ; echo %(token)s > build.token.retcode" % {
332 'waitmaster' : waitmaster,
333 'syncfiles' : syncfiles,
335 'kill_agent' : kill_agent,
336 'launch_agent' : launch_agent,
337 'home' : server.shell_escape(self.home_path),
338 'token' : server.shell_escape(self._master_token),
341 return cStringIO.StringIO(slavescript)
343 def _do_launch_build(self):
344 script = "bash ./nepi-build.sh"
345 if self._master_passphrase:
346 script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
347 server.shell_escape(self._master_passphrase),
350 (out,err),proc = rspawn.remote_spawn(
352 pidfile = 'build-pid',
353 home = self.home_path,
356 stderr = rspawn.STDOUT,
358 host = self.node.hostname,
360 user = self.node.slicename,
362 ident_key = self.node.ident_path,
363 server_key = self.node.server_key
367 if self.check_bad_host(out, err):
368 self.node.blacklist()
369 raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
375 pidtuple = rspawn.remote_check_pid(
376 os.path.join(self.home_path,'build-pid'),
377 host = self.node.hostname,
379 user = self.node.slicename,
381 ident_key = self.node.ident_path,
382 server_key = self.node.server_key
387 self._build_pid, self._build_ppid = pidtuple
391 delay = min(30,delay*1.2)
393 raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
395 self._logger.info("Deploying %s at %s", self, self.node.hostname)
397 def _do_wait_build(self, trial=0):
398 pid = self._build_pid
399 ppid = self._build_ppid
406 status = rspawn.remote_status(
408 host = self.node.hostname,
410 user = self.node.slicename,
412 ident_key = self.node.ident_path,
413 server_key = self.node.server_key
416 if status is rspawn.FINISHED:
417 self._build_pid = self._build_ppid = None
419 elif status is not rspawn.RUNNING:
421 time.sleep(delay*(5.5+random.random()))
423 self._build_pid = self._build_ppid = None
427 self._logger.info("Waiting for %s to finish building at %s %s", self, self.node.hostname,
428 "(build slave)" if self._master is not None else "(build master)")
431 time.sleep(delay*(0.5+random.random()))
432 delay = min(30,delay*1.2)
438 (out, err), proc = self._popen_ssh_command(
439 "cat %(token_path)s" % {
440 'token_path' : os.path.join(self.home_path, 'build.token'),
444 if not proc.wait() and out:
445 slave_token = out.strip()
452 if slave_token != self._master_token:
453 # Get buildlog for the error message
455 (buildlog, err), proc = self._popen_ssh_command(
456 "cat %(buildlog)s" % {
457 'buildlog' : os.path.join(self.home_path, 'buildlog'),
458 'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
465 if self.check_bad_host(buildlog, err):
466 self.node.blacklist()
467 elif self._master and trial < 3 and 'BAD TOKEN' in buildlog or 'BAD TOKEN' in err:
468 # bad sync with master, may try again
469 # but first wait for master
470 self._master.async_setup_wait()
471 self._launch_build(trial+1)
472 self._do_wait_build(trial+1)
475 self._master_prk = None
476 self._master_puk = None
478 raise RuntimeError, "Failed to set up application %s: "\
479 "build failed, got wrong token from pid %s/%s "\
480 "(expected %r, got %r), see buildlog at %s:\n%s" % (
481 self.home_path, pid, ppid, self._master_token, slave_token, self.node.hostname, buildlog)
484 self._master_prk = None
485 self._master_puk = None
487 self._logger.info("Built %s at %s", self, self.node.hostname)
489 def _do_kill_build(self):
490 pid = self._build_pid
491 ppid = self._build_ppid
494 self._logger.info("Killing build of %s", self)
497 host = self.node.hostname,
499 user = self.node.slicename,
501 ident_key = self.node.ident_path
505 def _do_build_master(self):
506 if not self.sources and not self.build and not self.buildDepends:
510 sources = self.sources.split(' ')
516 "%s@%s:%s" % (self.node.slicename, self.node.hostname,
517 os.path.join(self.home_path,'.'),)
519 except RuntimeError, e:
520 raise RuntimeError, "Failed upload source file %r: %s %s" \
521 % (sources, e.args[0], e.args[1],)
523 buildscript = cStringIO.StringIO()
525 buildscript.write("(\n")
527 if self.buildDepends:
528 # Install build dependencies
530 "sudo -S yum -y install %(packages)s\n" % {
531 'packages' : self.buildDepends
539 "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
540 'command' : self._replace_paths(self.build),
541 'home' : server.shell_escape(self.home_path),
546 buildscript.write("tar czf build.tar.gz build\n")
549 buildscript.write("echo %(master_token)s > build.token ) ; echo %(master_token)s > build.token.retcode" % {
550 'master_token' : server.shell_escape(self._master_token)
557 def _do_install(self):
559 self._logger.info("Installing %s at %s", self, self.node.hostname)
561 # Install application
563 self._popen_ssh_command(
564 "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/{install,build}log >&2 && false )" % \
566 'command' : self._replace_paths(self.install),
567 'home' : server.shell_escape(self.home_path),
570 except RuntimeError, e:
571 if self.check_bad_host(e.args[0], e.args[1]):
572 self.node.blacklist()
573 raise RuntimeError, "Failed install build sources: %s %s" % (e.args[0], e.args[1],)
575 def set_master(self, master):
576 self._master = master
578 def install_keys(self, prk, puk, passphrase):
580 self._master_passphrase = passphrase
581 self._master_prk = prk
582 self._master_puk = puk
583 self._master_prk_name = os.path.basename(prk.name)
584 self._master_puk_name = os.path.basename(puk.name)
586 def _do_install_keys(self):
587 prk = self._master_prk
588 puk = self._master_puk
592 [ prk.name, puk.name ],
593 '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path )
595 except RuntimeError, e:
596 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
597 % (e.args[0], e.args[1],)
601 cStringIO.StringIO('%s,%s %s\n' % (
602 self._master.node.hostname, socket.gethostbyname(self._master.node.hostname),
603 self._master.node.server_key)),
604 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
605 os.path.join(self.home_path,"master_known_hosts") )
607 except RuntimeError, e:
608 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
609 % (e.args[0], e.args[1],)
613 # make sure there's no leftover build processes
614 self._do_kill_build()
617 self._master_prk = None
618 self._master_puk = None
621 def _popen_scp(self, src, dst, retry = 3):
624 (out,err),proc = server.popen_scp(
629 ident_key = self.node.ident_path,
630 server_key = self.node.server_key
633 if server.eintr_retry(proc.wait)():
634 raise RuntimeError, (out, err)
635 return (out, err), proc
644 def _popen_ssh_command(self, command, retry = 0, noerrors=False, timeout=None):
645 (out,err),proc = server.popen_ssh_command(
647 host = self.node.hostname,
649 user = self.node.slicename,
651 ident_key = self.node.ident_path,
652 server_key = self.node.server_key,
657 if server.eintr_retry(proc.wait)():
659 raise RuntimeError, (out, err)
660 return (out, err), proc
662 class Application(Dependency):
664 An application also has dependencies, but also a command to be ran and monitored.
666 It adds the output of that command as traces.
669 TRACES = ('stdout','stderr','buildlog', 'output')
671 def __init__(self, api=None):
672 super(Application,self).__init__(api)
683 # Those are filled when the app is started
684 # Having both pid and ppid makes it harder
685 # for pid rollover to induce tracking mistakes
686 self._started = False
690 # Do not add to the python path of nodes
691 self.add_to_path = False
694 return "%s<command:%s%s>" % (
695 self.__class__.__name__,
696 "sudo " if self.sudo else "",
701 self._logger.info("Starting %s", self)
703 # Create shell script with the command
704 # This way, complex commands and scripts can be ran seamlessly
706 command = cStringIO.StringIO()
707 command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
708 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
710 command.write('export PATH=$PATH:%s\n' % (
711 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
714 for envkey, envvals in self.node.env.iteritems():
715 for envval in envvals:
716 command.write('export %s=%s\n' % (envkey, envval))
717 command.write(self.command)
723 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
724 os.path.join(self.home_path, "app.sh"))
726 except RuntimeError, e:
727 raise RuntimeError, "Failed to set up application: %s %s" \
728 % (e.args[0], e.args[1],)
730 # Start process in a "daemonized" way, using nohup and heavy
731 # stdin/out redirection to avoid connection issues
732 (out,err),proc = rspawn.remote_spawn(
733 self._replace_paths("bash ./app.sh"),
736 home = self.home_path,
737 stdin = 'stdin' if self.stdin is not None else '/dev/null',
738 stdout = 'stdout' if self.stdout else '/dev/null',
739 stderr = 'stderr' if self.stderr else '/dev/null',
742 host = self.node.hostname,
744 user = self.node.slicename,
746 ident_key = self.node.ident_path,
747 server_key = self.node.server_key
751 if self.check_bad_host(out, err):
752 self.node.blacklist()
753 raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
758 # Assuming the application is running on PlanetLab,
759 # proper pidfiles should be present at the app's home path.
760 # So we mark this application as started, and check the pidfiles
766 # NOTE: wait a bit for the pidfile to be created
767 if self._started and not self._pid or not self._ppid:
768 pidtuple = rspawn.remote_check_pid(
769 os.path.join(self.home_path,'pid'),
770 host = self.node.hostname,
772 user = self.node.slicename,
774 ident_key = self.node.ident_path,
775 server_key = self.node.server_key
779 self._pid, self._ppid = pidtuple
783 if not self._started:
784 return AS.STATUS_NOT_STARTED
785 elif not self._pid or not self._ppid:
786 return AS.STATUS_NOT_STARTED
788 status = rspawn.remote_status(
789 self._pid, self._ppid,
790 host = self.node.hostname,
792 user = self.node.slicename,
794 ident_key = self.node.ident_path,
795 server_key = self.node.server_key
798 if status is rspawn.NOT_STARTED:
799 return AS.STATUS_NOT_STARTED
800 elif status is rspawn.RUNNING:
801 return AS.STATUS_RUNNING
802 elif status is rspawn.FINISHED:
803 return AS.STATUS_FINISHED
806 return AS.STATUS_NOT_STARTED
809 status = self.status()
810 if status == AS.STATUS_RUNNING:
811 # kill by ppid+pid - SIGTERM first, then try SIGKILL
813 self._pid, self._ppid,
814 host = self.node.hostname,
816 user = self.node.slicename,
818 ident_key = self.node.ident_path,
819 server_key = self.node.server_key,
822 self._logger.info("Killed %s", self)
825 class NepiDependency(Dependency):
827 This dependency adds nepi itself to the python path,
828 so that you may run testbeds within PL nodes.
831 # Class attribute holding a *weak* reference to the shared NEPI tar file
832 # so that they may share it. Don't operate on the file itself, it would
833 # be a mess, just use its path.
834 _shared_nepi_tar = None
836 def __init__(self, api = None):
837 super(NepiDependency, self).__init__(api)
841 self.depends = 'python python-ipaddr python-setuptools'
843 # our sources are in our ad-hoc tarball
844 self.sources = self.tarball.name
846 tarname = os.path.basename(self.tarball.name)
848 # it's already built - just move the tarball into place
849 self.build = "mv -f ${SOURCES}/%s ." % (tarname,)
851 # unpack it into sources, and we're done
852 self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,)
856 if self._tarball is None:
857 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
858 if shared_tar is not None:
859 self._tarball = shared_tar
861 # Build an ad-hoc tarball
866 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
868 proc = subprocess.Popen(
869 ["tar", "czf", shared_tar.name,
870 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
872 stdout = open("/dev/null","w"),
873 stdin = open("/dev/null","r"))
876 raise RuntimeError, "Failed to create nepi tarball"
878 self._tarball = self._shared_nepi_tar = shared_tar
882 class NS3Dependency(Dependency):
884 This dependency adds NS3 libraries to the library paths,
885 so that you may run the NS3 testbed within PL nodes.
887 You'll also need the NepiDependency.
890 def __init__(self, api = None):
891 super(NS3Dependency, self).__init__(api)
893 self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip'
895 # We have to download the sources, untar, build...
896 pybindgen_source_url = "http://yans.pl.sophia.inria.fr/trac/nepi/raw-attachment/wiki/WikiStart/pybindgen-r794.tar.gz"
897 pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
898 ns3_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/ns-3.11-nepi/archive/tip.tar.gz"
899 passfd_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/python-passfd/archive/tip.tar.gz"
903 " python -c 'import pygccxml, pybindgen, passfd' && "
904 " test -f lib/ns/_core.so && "
905 " test -f lib/ns/__init__.py && "
906 " test -f lib/ns/core.py && "
907 " test -f lib/libns3-core.so && "
908 " LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
910 # Not working, rebuild
911 # Archive SHA1 sums to check
912 "echo '7158877faff2254e6c094bf18e6b4283cac19137 pygccxml-1.0.0.zip' > archive_sums.txt && "
913 "echo 'a18c2ccffd0df517bc37e2f3a2475092517c43f2 pybindgen-src.tar.gz' >> archive_sums.txt && "
914 " ( " # check existing files
915 " sha1sum -c archive_sums.txt && "
916 " test -f passfd-src.tar.gz && "
917 " test -f ns3-src.tar.gz "
918 " ) || ( " # nope? re-download
919 " rm -f pybindgen-src.zip pygccxml-1.0.0.zip passfd-src.tar.gz ns3-src.tar.gz && "
920 " wget -q -c -O pybindgen-src.tar.gz %(pybindgen_source_url)s && " # continue, to exploit the case when it has already been dl'ed
921 " wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
922 " wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
923 " wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
924 " sha1sum -c archive_sums.txt " # Check SHA1 sums when applicable
926 "unzip -n pygccxml-1.0.0.zip && "
927 "mkdir -p pybindgen-src && "
928 "mkdir -p ns3-src && "
929 "mkdir -p passfd-src && "
930 "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
931 "tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
932 "tar xzf pybindgen-src.tar.gz --strip-components=1 -C pybindgen-src && "
933 "rm -rf target && " # mv doesn't like unclean targets
934 "mkdir -p target && "
935 "cd pygccxml-1.0.0 && "
936 "rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
937 "python setup.py build && "
938 "python setup.py install --install-lib ${BUILD}/target && "
939 "python setup.py clean && "
940 "cd ../pybindgen-src && "
941 "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
942 "./waf configure --prefix=${BUILD}/target -d release && "
946 "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
947 "rm -rf ${BUILD}/target/lib && "
948 "cd ../passfd-src && "
949 "python setup.py build && "
950 "python setup.py install --install-lib ${BUILD}/target && "
951 "python setup.py clean && "
953 "./waf configure --prefix=${BUILD}/target --with-pybindgen=../pybindgen-src -d release --disable-examples --disable-tests && "
956 "rm -f ${BUILD}/target/lib/*.so && "
957 "cp -a ${BUILD}/ns3-src/build/release/libns3*.so ${BUILD}/target/lib && "
958 "cp -a ${BUILD}/ns3-src/build/release/bindings/python/ns ${BUILD}/target/lib &&"
962 pybindgen_source_url = server.shell_escape(pybindgen_source_url),
963 pygccxml_source_url = server.shell_escape(pygccxml_source_url),
964 ns3_source_url = server.shell_escape(ns3_source_url),
965 passfd_source_url = server.shell_escape(passfd_source_url),
968 # Just move ${BUILD}/target
972 " python -c 'import pygccxml, pybindgen, passfd' && "
973 " test -f lib/ns/_core.so && "
974 " test -f lib/ns/__init__.py && "
975 " test -f lib/ns/core.py && "
976 " test -f lib/libns3-core.so && "
977 " LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
979 # Not working, reinstall
980 "test -d ${BUILD}/target && "
981 "[[ \"x\" != \"x$(find ${BUILD}/target -mindepth 1 -print -quit)\" ]] &&"
982 "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
983 "mv -f ${BUILD}/target/* ${SOURCES}"
987 # Set extra environment paths
988 self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
989 self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib"
993 if self._tarball is None:
994 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
995 if shared_tar is not None:
996 self._tarball = shared_tar
998 # Build an ad-hoc tarball
1003 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
1005 proc = subprocess.Popen(
1006 ["tar", "czf", shared_tar.name,
1007 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
1009 stdout = open("/dev/null","w"),
1010 stdin = open("/dev/null","r"))
1013 raise RuntimeError, "Failed to create nepi tarball"
1015 self._tarball = self._shared_nepi_tar = shared_tar
1017 return self._tarball
1019 class YumDependency(Dependency):
1021 This dependency is an internal helper class used to
1022 efficiently distribute yum-downloaded rpms.
1024 It temporarily sets the yum cache as persistent in the
1025 build master, and installs all the required packages.
1027 The rpm packages left in the yum cache are gathered and
1028 distributed by the underlying Dependency in an efficient
1029 manner. Build slaves will then install those rpms back in
1030 the cache before issuing the install command.
1032 When packages have been installed already, nothing but an
1033 empty tar is distributed.
1036 # Class attribute holding a *weak* reference to the shared NEPI tar file
1037 # so that they may share it. Don't operate on the file itself, it would
1038 # be a mess, just use its path.
1039 _shared_nepi_tar = None
1041 def _build_get(self):
1042 # canonical representation of dependencies
1043 depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
1045 # download rpms and pack into a tar archive
1047 "sudo -S nice yum -y makecache && "
1048 "sudo -S sed -i -r 's/keepcache *= *0/keepcache=1/' /etc/yum.conf && "
1050 "sudo -S nice yum -y install %s ; "
1051 "rm -f ${BUILD}/packages.tar ; "
1052 "tar -C /var/cache/yum -rf ${BUILD}/packages.tar $(cd /var/cache/yum ; find -iname '*.rpm')"
1053 " ) || /bin/true ) && "
1054 "sudo -S sed -i -r 's/keepcache *= *1/keepcache=0/' /etc/yum.conf && "
1055 "( sudo -S nice yum -y clean packages || /bin/true ) "
1057 def _build_set(self, value):
1060 build = property(_build_get, _build_set)
1062 def _install_get(self):
1063 # canonical representation of dependencies
1064 depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
1066 # unpack cached rpms into yum cache, install, and cleanup
1068 "sudo -S tar -k --keep-newer-files -C /var/cache/yum -xf packages.tar && "
1069 "sudo -S nice yum -y install %s && "
1070 "( sudo -S nice yum -y clean packages || /bin/true ) "
1072 def _install_set(self, value):
1075 install = property(_install_get, _install_set)
1077 def check_bad_host(self, out, err):
1078 badre = re.compile(r'(?:'
1079 r'The GPG keys listed for the ".*" repository are already installed but they are not correct for this package'
1080 r'|Error: Cannot retrieve repository metadata (repomd.xml) for repository: .*[.] Please verify its path and try again'
1081 r'|Error: disk I/O error'
1082 r'|MASTER NODE UNREACHABLE'
1085 return badre.search(out) or badre.search(err)