2 # -*- coding: utf-8 -*-
4 from constants import TESTBED_ID
10 import nepi.util.server as server
21 from nepi.util.constants import ApplicationStatus as AS
23 class Dependency(object):
25 A Dependency is in every respect like an application.
27 It depends on some packages, it may require building binaries, it must deploy
30 But it has no command. Dependencies aren't ever started, or stopped, and have
36 def __init__(self, api=None):
48 self.buildDepends = None
50 self.rpmFusion = False
58 self.add_to_path = True
60 # Those are filled when the app is configured
63 # Those are filled when an actual node is connected
66 # Those are filled when the app is started
67 # Having both pid and ppid makes it harder
68 # for pid rollover to induce tracking mistakes
75 # Spanning tree deployment
77 self._master_passphrase = None
78 self._master_prk = None
79 self._master_puk = None
80 self._master_token = os.urandom(8).encode("hex")
81 self._build_pid = None
82 self._build_ppid = None
85 self._logger = logging.getLogger('nepi.testbeds.planetlab')
90 self.__class__.__name__,
91 ' '.join(filter(bool,(self.depends, self.sources)))
95 if self.home_path is None:
96 raise AssertionError, "Misconfigured application: missing home path"
97 if self.node.ident_path is None or not os.access(self.node.ident_path, os.R_OK):
98 raise AssertionError, "Misconfigured application: missing slice SSH key"
100 raise AssertionError, "Misconfigured application: unconnected node"
101 if self.node.hostname is None:
102 raise AssertionError, "Misconfigured application: misconfigured node"
103 if self.node.slicename is None:
104 raise AssertionError, "Misconfigured application: unspecified slice"
106 def check_bad_host(self, out, err):
108 Called whenever an operation fails, it's given the output to be checked for
109 telltale signs of unhealthy hosts.
113 def remote_trace_path(self, whichtrace):
114 if whichtrace in self.TRACES:
115 tracefile = os.path.join(self.home_path, whichtrace)
121 def remote_trace_name(self, whichtrace):
122 if whichtrace in self.TRACES:
126 def sync_trace(self, local_dir, whichtrace):
127 tracefile = self.remote_trace_path(whichtrace)
131 local_path = os.path.join(local_dir, tracefile)
133 # create parent local folders
134 proc = subprocess.Popen(
135 ["mkdir", "-p", os.path.dirname(local_path)],
136 stdout = open("/dev/null","w"),
137 stdin = open("/dev/null","r"))
140 raise RuntimeError, "Failed to synchronize trace"
145 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
149 except RuntimeError, e:
150 raise RuntimeError, "Failed to synchronize trace: %s %s" \
151 % (e.args[0], e.args[1],)
156 # We assume a correct deployment, so recovery only
157 # means we mark this dependency as deployed
161 self._logger.info("Setting up %s", self)
167 def async_setup(self):
168 if not self._setuper:
173 self._setuper._exc.append(sys.exc_info())
174 self._setuper = threading.Thread(
176 self._setuper._exc = []
177 self._setuper.start()
179 def async_setup_wait(self):
181 self._logger.info("Waiting for %s to be setup", self)
185 if self._setuper._exc:
186 exctyp,exval,exctrace = self._setuper._exc[0]
187 raise exctyp,exval,exctrace
189 raise RuntimeError, "Failed to setup application"
191 self._logger.info("Setup ready: %s at %s", self, self.node.hostname)
195 def _make_home(self):
196 # Make sure all the paths are created where
197 # they have to be created for deployment
200 self._popen_ssh_command(
201 "mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" \
202 % { 'home' : server.shell_escape(self.home_path) },
206 except RuntimeError, e:
207 raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, e.args[0], e.args[1],)
210 # Write program input
213 cStringIO.StringIO(self.stdin),
214 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
215 os.path.join(self.home_path, 'stdin') ),
217 except RuntimeError, e:
218 raise RuntimeError, "Failed to set up application %s: %s %s" \
219 % (self.home_path, e.args[0], e.args[1],)
221 def _replace_paths(self, command):
223 Replace all special path tags with shell-escaped actual paths.
225 # need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
226 root = '' if self.home_path.startswith('/') else "${HOME}/"
228 .replace("${SOURCES}", root+server.shell_escape(self.home_path))
229 .replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
231 def _launch_build(self, trial=0):
232 if self._master is not None:
234 self._do_install_keys()
235 buildscript = self._do_build_slave()
237 buildscript = self._do_build_master()
239 if buildscript is not None:
240 self._logger.info("Building %s at %s", self, self.node.hostname)
242 # upload build script
246 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
247 os.path.join(self.home_path, 'nepi-build.sh') )
249 except RuntimeError, e:
250 raise RuntimeError, "Failed to set up application %s: %s %s" \
251 % (self.home_path, e.args[0], e.args[1],)
254 self._do_launch_build()
256 def _finish_build(self):
257 self._do_wait_build()
260 def _do_build_slave(self):
261 if not self.sources and not self.build:
264 # Create build script
268 sources = self.sources.split(' ')
270 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
271 os.path.join(self._master.home_path, os.path.basename(source)),)
272 for source in sources
277 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
278 os.path.join(self._master.home_path, 'build.tar.gz'),)
281 sshopts = "-o ConnectTimeout=30 -o ConnectionAttempts=3 -o ServerAliveInterval=30 -o TCPKeepAlive=yes"
283 launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
284 " && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
285 " && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" % \
287 'prk' : server.shell_escape(self._master_prk_name),
288 'puk' : server.shell_escape(self._master_puk_name),
291 kill_agent = "kill $SSH_AGENT_PID"
295 "echo 'Checking master reachability' ; "
296 "if ping -c 3 %(master_host)s && (. ./.ssh-agent.sh > /dev/null ; ssh -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s echo MASTER SAYS HI ) ; then "
297 "echo 'Master node reachable' ; "
299 "echo 'MASTER NODE UNREACHABLE' && "
302 ". ./.ssh-agent.sh ; "
303 "while [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s.retcode || /bin/true) != %(token)s ]] ; do sleep 5 ; done ; "
304 "if [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s || /bin/true) != %(token)s ]] ; then echo BAD TOKEN ; exit 1 ; fi ; "
307 'hostkey' : 'master_known_hosts',
308 'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostname),
309 'master_host' : self._master.node.hostname,
310 'token_path' : os.path.join(self._master.home_path, 'build.token'),
311 'token' : server.shell_escape(self._master._master_token),
315 syncfiles = ". ./.ssh-agent.sh && scp -p -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(files)s ." % {
316 'hostkey' : 'master_known_hosts',
317 'files' : ' '.join(files),
321 syncfiles += " && tar xzf build.tar.gz"
322 syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
323 syncfiles += " && ( echo %s > build.token.retcode )" % (server.shell_escape(self._master_token),)
324 syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
326 cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
327 'prk' : server.shell_escape(self._master_prk_name),
328 'puk' : server.shell_escape(self._master_puk_name),
331 slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s ) ; echo %(token)s > build.token.retcode" % {
332 'waitmaster' : waitmaster,
333 'syncfiles' : syncfiles,
335 'kill_agent' : kill_agent,
336 'launch_agent' : launch_agent,
337 'home' : server.shell_escape(self.home_path),
338 'token' : server.shell_escape(self._master_token),
341 return cStringIO.StringIO(slavescript)
343 def _do_launch_build(self):
344 script = "bash ./nepi-build.sh"
345 if self._master_passphrase:
346 script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
347 server.shell_escape(self._master_passphrase),
350 (out,err),proc = rspawn.remote_spawn(
352 pidfile = 'build-pid',
353 home = self.home_path,
356 stderr = rspawn.STDOUT,
358 host = self.node.hostname,
360 user = self.node.slicename,
362 ident_key = self.node.ident_path,
363 server_key = self.node.server_key
367 if self.check_bad_host(out, err):
368 self.node.blacklist()
369 raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
375 pidtuple = rspawn.remote_check_pid(
376 os.path.join(self.home_path,'build-pid'),
377 host = self.node.hostname,
379 user = self.node.slicename,
381 ident_key = self.node.ident_path,
382 server_key = self.node.server_key
387 self._build_pid, self._build_ppid = pidtuple
391 delay = min(30,delay*1.2)
393 raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
395 self._logger.info("Deploying %s at %s", self, self.node.hostname)
397 def _do_wait_build(self, trial=0):
398 pid = self._build_pid
399 ppid = self._build_ppid
406 status = rspawn.remote_status(
408 host = self.node.hostname,
410 user = self.node.slicename,
412 ident_key = self.node.ident_path,
413 server_key = self.node.server_key
416 if status is rspawn.FINISHED:
417 self._build_pid = self._build_ppid = None
419 elif status is not rspawn.RUNNING:
421 time.sleep(delay*(5.5+random.random()))
423 self._build_pid = self._build_ppid = None
427 self._logger.info("Waiting for %s to finish building at %s %s", self, self.node.hostname,
428 "(build slave)" if self._master is not None else "(build master)")
431 time.sleep(delay*(0.5+random.random()))
432 delay = min(30,delay*1.2)
438 (out, err), proc = self._popen_ssh_command(
439 "cat %(token_path)s" % {
440 'token_path' : os.path.join(self.home_path, 'build.token'),
444 if not proc.wait() and out:
445 slave_token = out.strip()
452 if slave_token != self._master_token:
453 # Get buildlog for the error message
455 (buildlog, err), proc = self._popen_ssh_command(
456 "cat %(buildlog)s" % {
457 'buildlog' : os.path.join(self.home_path, 'buildlog'),
458 'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
465 if self.check_bad_host(buildlog, err):
466 self.node.blacklist()
467 elif self._master and trial < 3 and 'BAD TOKEN' in buildlog or 'BAD TOKEN' in err:
468 # bad sync with master, may try again
469 # but first wait for master
470 self._master.async_setup_wait()
471 self._launch_build(trial+1)
472 self._do_wait_build(trial+1)
474 raise RuntimeError, "Failed to set up application %s: "\
475 "build failed, got wrong token from pid %s/%s "\
476 "(expected %r, got %r), see buildlog at %s:\n%s" % (
477 self.home_path, pid, ppid, self._master_token, slave_token, self.node.hostname, buildlog)
479 self._logger.info("Built %s at %s", self, self.node.hostname)
481 def _do_kill_build(self):
482 pid = self._build_pid
483 ppid = self._build_ppid
486 self._logger.info("Killing build of %s", self)
489 host = self.node.hostname,
491 user = self.node.slicename,
493 ident_key = self.node.ident_path
497 def _do_build_master(self):
498 if not self.sources and not self.build and not self.buildDepends:
502 sources = self.sources.split(' ')
508 "%s@%s:%s" % (self.node.slicename, self.node.hostname,
509 os.path.join(self.home_path,'.'),)
511 except RuntimeError, e:
512 raise RuntimeError, "Failed upload source file %r: %s %s" \
513 % (sources, e.args[0], e.args[1],)
515 buildscript = cStringIO.StringIO()
517 buildscript.write("(\n")
519 if self.buildDepends:
520 # Install build dependencies
522 "sudo -S yum -y install %(packages)s\n" % {
523 'packages' : self.buildDepends
531 "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
532 'command' : self._replace_paths(self.build),
533 'home' : server.shell_escape(self.home_path),
538 buildscript.write("tar czf build.tar.gz build\n")
541 buildscript.write("echo %(master_token)s > build.token ) ; echo %(master_token)s > build.token.retcode" % {
542 'master_token' : server.shell_escape(self._master_token)
549 def _do_install(self):
551 self._logger.info("Installing %s at %s", self, self.node.hostname)
553 # Install application
555 self._popen_ssh_command(
556 "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/{install,build}log >&2 && false )" % \
558 'command' : self._replace_paths(self.install),
559 'home' : server.shell_escape(self.home_path),
562 except RuntimeError, e:
563 if self.check_bad_host(e.args[0], e.args[1]):
564 self.node.blacklist()
565 raise RuntimeError, "Failed install build sources: %s %s" % (e.args[0], e.args[1],)
567 def set_master(self, master):
568 self._master = master
570 def install_keys(self, prk, puk, passphrase):
572 self._master_passphrase = passphrase
573 self._master_prk = prk
574 self._master_puk = puk
575 self._master_prk_name = os.path.basename(prk.name)
576 self._master_puk_name = os.path.basename(puk.name)
578 def _do_install_keys(self):
579 prk = self._master_prk
580 puk = self._master_puk
584 [ prk.name, puk.name ],
585 '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path )
587 except RuntimeError, e:
588 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
589 % (e.args[0], e.args[1],)
593 cStringIO.StringIO('%s,%s %s\n' % (
594 self._master.node.hostname, socket.gethostbyname(self._master.node.hostname),
595 self._master.node.server_key)),
596 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
597 os.path.join(self.home_path,"master_known_hosts") )
599 except RuntimeError, e:
600 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
601 % (e.args[0], e.args[1],)
604 self._master_prk = None
605 self._master_puk = None
608 # make sure there's no leftover build processes
609 self._do_kill_build()
612 def _popen_scp(self, src, dst, retry = 3):
615 (out,err),proc = server.popen_scp(
620 ident_key = self.node.ident_path,
621 server_key = self.node.server_key
624 if server.eintr_retry(proc.wait)():
625 raise RuntimeError, (out, err)
626 return (out, err), proc
635 def _popen_ssh_command(self, command, retry = 0, noerrors=False, timeout=None):
636 (out,err),proc = server.popen_ssh_command(
638 host = self.node.hostname,
640 user = self.node.slicename,
642 ident_key = self.node.ident_path,
643 server_key = self.node.server_key,
648 if server.eintr_retry(proc.wait)():
650 raise RuntimeError, (out, err)
651 return (out, err), proc
653 class Application(Dependency):
655 An application also has dependencies, but also a command to be ran and monitored.
657 It adds the output of that command as traces.
660 TRACES = ('stdout','stderr','buildlog', 'output')
662 def __init__(self, api=None):
663 super(Application,self).__init__(api)
674 # Those are filled when the app is started
675 # Having both pid and ppid makes it harder
676 # for pid rollover to induce tracking mistakes
677 self._started = False
681 # Do not add to the python path of nodes
682 self.add_to_path = False
685 return "%s<command:%s%s>" % (
686 self.__class__.__name__,
687 "sudo " if self.sudo else "",
692 self._logger.info("Starting %s", self)
694 # Create shell script with the command
695 # This way, complex commands and scripts can be ran seamlessly
697 command = cStringIO.StringIO()
698 command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
699 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
701 command.write('export PATH=$PATH:%s\n' % (
702 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
705 for envkey, envvals in self.node.env.iteritems():
706 for envval in envvals:
707 command.write('export %s=%s\n' % (envkey, envval))
708 command.write(self.command)
714 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
715 os.path.join(self.home_path, "app.sh"))
717 except RuntimeError, e:
718 raise RuntimeError, "Failed to set up application: %s %s" \
719 % (e.args[0], e.args[1],)
721 # Start process in a "daemonized" way, using nohup and heavy
722 # stdin/out redirection to avoid connection issues
723 (out,err),proc = rspawn.remote_spawn(
724 self._replace_paths("bash ./app.sh"),
727 home = self.home_path,
728 stdin = 'stdin' if self.stdin is not None else '/dev/null',
729 stdout = 'stdout' if self.stdout else '/dev/null',
730 stderr = 'stderr' if self.stderr else '/dev/null',
733 host = self.node.hostname,
735 user = self.node.slicename,
737 ident_key = self.node.ident_path,
738 server_key = self.node.server_key
742 if self.check_bad_host(out, err):
743 self.node.blacklist()
744 raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
749 # Assuming the application is running on PlanetLab,
750 # proper pidfiles should be present at the app's home path.
751 # So we mark this application as started, and check the pidfiles
757 # NOTE: wait a bit for the pidfile to be created
758 if self._started and not self._pid or not self._ppid:
759 pidtuple = rspawn.remote_check_pid(
760 os.path.join(self.home_path,'pid'),
761 host = self.node.hostname,
763 user = self.node.slicename,
765 ident_key = self.node.ident_path,
766 server_key = self.node.server_key
770 self._pid, self._ppid = pidtuple
774 if not self._started:
775 return AS.STATUS_NOT_STARTED
776 elif not self._pid or not self._ppid:
777 return AS.STATUS_NOT_STARTED
779 status = rspawn.remote_status(
780 self._pid, self._ppid,
781 host = self.node.hostname,
783 user = self.node.slicename,
785 ident_key = self.node.ident_path,
786 server_key = self.node.server_key
789 if status is rspawn.NOT_STARTED:
790 return AS.STATUS_NOT_STARTED
791 elif status is rspawn.RUNNING:
792 return AS.STATUS_RUNNING
793 elif status is rspawn.FINISHED:
794 return AS.STATUS_FINISHED
797 return AS.STATUS_NOT_STARTED
800 status = self.status()
801 if status == AS.STATUS_RUNNING:
802 # kill by ppid+pid - SIGTERM first, then try SIGKILL
804 self._pid, self._ppid,
805 host = self.node.hostname,
807 user = self.node.slicename,
809 ident_key = self.node.ident_path,
810 server_key = self.node.server_key,
813 self._logger.info("Killed %s", self)
816 class NepiDependency(Dependency):
818 This dependency adds nepi itself to the python path,
819 so that you may run testbeds within PL nodes.
822 # Class attribute holding a *weak* reference to the shared NEPI tar file
823 # so that they may share it. Don't operate on the file itself, it would
824 # be a mess, just use its path.
825 _shared_nepi_tar = None
827 def __init__(self, api = None):
828 super(NepiDependency, self).__init__(api)
832 self.depends = 'python python-ipaddr python-setuptools'
834 # our sources are in our ad-hoc tarball
835 self.sources = self.tarball.name
837 tarname = os.path.basename(self.tarball.name)
839 # it's already built - just move the tarball into place
840 self.build = "mv -f ${SOURCES}/%s ." % (tarname,)
842 # unpack it into sources, and we're done
843 self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,)
847 if self._tarball is None:
848 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
849 if shared_tar is not None:
850 self._tarball = shared_tar
852 # Build an ad-hoc tarball
857 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
859 proc = subprocess.Popen(
860 ["tar", "czf", shared_tar.name,
861 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
863 stdout = open("/dev/null","w"),
864 stdin = open("/dev/null","r"))
867 raise RuntimeError, "Failed to create nepi tarball"
869 self._tarball = self._shared_nepi_tar = shared_tar
873 class NS3Dependency(Dependency):
875 This dependency adds NS3 libraries to the library paths,
876 so that you may run the NS3 testbed within PL nodes.
878 You'll also need the NepiDependency.
881 def __init__(self, api = None):
882 super(NS3Dependency, self).__init__(api)
884 self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip'
886 # We have to download the sources, untar, build...
887 pybindgen_source_url = "http://yans.pl.sophia.inria.fr/trac/nepi/raw-attachment/wiki/WikiStart/pybindgen-r794.tar.gz"
888 pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
889 ns3_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/ns-3.11-nepi/archive/tip.tar.gz"
890 passfd_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/python-passfd/archive/tip.tar.gz"
894 " python -c 'import pygccxml, pybindgen, passfd' && "
895 " test -f lib/ns/_core.so && "
896 " test -f lib/ns/__init__.py && "
897 " test -f lib/ns/core.py && "
898 " test -f lib/libns3-core.so && "
899 " LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
901 # Not working, rebuild
902 # Archive SHA1 sums to check
903 "echo '7158877faff2254e6c094bf18e6b4283cac19137 pygccxml-1.0.0.zip' > archive_sums.txt && "
904 "echo 'a18c2ccffd0df517bc37e2f3a2475092517c43f2 pybindgen-src.tar.gz' >> archive_sums.txt && "
905 " ( " # check existing files
906 " sha1sum -c archive_sums.txt && "
907 " test -f passfd-src.tar.gz && "
908 " test -f ns3-src.tar.gz "
909 " ) || ( " # nope? re-download
910 " rm -f pybindgen-src.zip pygccxml-1.0.0.zip passfd-src.tar.gz ns3-src.tar.gz && "
911 " wget -q -c -O pybindgen-src.tar.gz %(pybindgen_source_url)s && " # continue, to exploit the case when it has already been dl'ed
912 " wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
913 " wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
914 " wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
915 " sha1sum -c archive_sums.txt " # Check SHA1 sums when applicable
917 "unzip -n pygccxml-1.0.0.zip && "
918 "mkdir -p pybindgen-src && "
919 "mkdir -p ns3-src && "
920 "mkdir -p passfd-src && "
921 "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
922 "tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
923 "tar xzf pybindgen-src.tar.gz --strip-components=1 -C pybindgen-src && "
924 "rm -rf target && " # mv doesn't like unclean targets
925 "mkdir -p target && "
926 "cd pygccxml-1.0.0 && "
927 "rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
928 "python setup.py build && "
929 "python setup.py install --install-lib ${BUILD}/target && "
930 "python setup.py clean && "
931 "cd ../pybindgen-src && "
932 "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
933 "./waf configure --prefix=${BUILD}/target -d release && "
937 "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
938 "rm -rf ${BUILD}/target/lib && "
939 "cd ../passfd-src && "
940 "python setup.py build && "
941 "python setup.py install --install-lib ${BUILD}/target && "
942 "python setup.py clean && "
944 "./waf configure --prefix=${BUILD}/target --with-pybindgen=../pybindgen-src -d release --disable-examples --disable-tests && "
947 "rm -f ${BUILD}/target/lib/*.so && "
948 "cp -a ${BUILD}/ns3-src/build/release/libns3*.so ${BUILD}/target/lib && "
949 "cp -a ${BUILD}/ns3-src/build/release/bindings/python/ns ${BUILD}/target/lib &&"
953 pybindgen_source_url = server.shell_escape(pybindgen_source_url),
954 pygccxml_source_url = server.shell_escape(pygccxml_source_url),
955 ns3_source_url = server.shell_escape(ns3_source_url),
956 passfd_source_url = server.shell_escape(passfd_source_url),
959 # Just move ${BUILD}/target
963 " python -c 'import pygccxml, pybindgen, passfd' && "
964 " test -f lib/ns/_core.so && "
965 " test -f lib/ns/__init__.py && "
966 " test -f lib/ns/core.py && "
967 " test -f lib/libns3-core.so && "
968 " LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
970 # Not working, reinstall
971 "test -d ${BUILD}/target && "
972 "[[ \"x\" != \"x$(find ${BUILD}/target -mindepth 1 -print -quit)\" ]] &&"
973 "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
974 "mv -f ${BUILD}/target/* ${SOURCES}"
978 # Set extra environment paths
979 self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
980 self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib"
984 if self._tarball is None:
985 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
986 if shared_tar is not None:
987 self._tarball = shared_tar
989 # Build an ad-hoc tarball
994 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
996 proc = subprocess.Popen(
997 ["tar", "czf", shared_tar.name,
998 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
1000 stdout = open("/dev/null","w"),
1001 stdin = open("/dev/null","r"))
1004 raise RuntimeError, "Failed to create nepi tarball"
1006 self._tarball = self._shared_nepi_tar = shared_tar
1008 return self._tarball
1010 class YumDependency(Dependency):
1012 This dependency is an internal helper class used to
1013 efficiently distribute yum-downloaded rpms.
1015 It temporarily sets the yum cache as persistent in the
1016 build master, and installs all the required packages.
1018 The rpm packages left in the yum cache are gathered and
1019 distributed by the underlying Dependency in an efficient
1020 manner. Build slaves will then install those rpms back in
1021 the cache before issuing the install command.
1023 When packages have been installed already, nothing but an
1024 empty tar is distributed.
1027 # Class attribute holding a *weak* reference to the shared NEPI tar file
1028 # so that they may share it. Don't operate on the file itself, it would
1029 # be a mess, just use its path.
1030 _shared_nepi_tar = None
1032 def _build_get(self):
1033 # canonical representation of dependencies
1034 depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
1036 # download rpms and pack into a tar archive
1038 "sudo -S nice yum -y makecache && "
1039 "sudo -S sed -i -r 's/keepcache *= *0/keepcache=1/' /etc/yum.conf && "
1041 "sudo -S nice yum -y install %s ; "
1042 "rm -f ${BUILD}/packages.tar ; "
1043 "tar -C /var/cache/yum -rf ${BUILD}/packages.tar $(cd /var/cache/yum ; find -iname '*.rpm')"
1044 " ) || /bin/true ) && "
1045 "sudo -S sed -i -r 's/keepcache *= *1/keepcache=0/' /etc/yum.conf && "
1046 "( sudo -S nice yum -y clean packages || /bin/true ) "
1048 def _build_set(self, value):
1051 build = property(_build_get, _build_set)
1053 def _install_get(self):
1054 # canonical representation of dependencies
1055 depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
1057 # unpack cached rpms into yum cache, install, and cleanup
1059 "sudo -S tar -k --keep-newer-files -C /var/cache/yum -xf packages.tar && "
1060 "sudo -S nice yum -y install %s && "
1061 "( sudo -S nice yum -y clean packages || /bin/true ) "
1063 def _install_set(self, value):
1066 install = property(_install_get, _install_set)
1068 def check_bad_host(self, out, err):
1069 badre = re.compile(r'(?:'
1070 r'The GPG keys listed for the ".*" repository are already installed but they are not correct for this package'
1071 r'|Error: Cannot retrieve repository metadata (repomd.xml) for repository: .*[.] Please verify its path and try again'
1072 r'|Error: disk I/O error'
1073 r'|MASTER NODE UNREACHABLE'
1076 return badre.search(out) or badre.search(err)