1 # -*- coding: utf-8 -*-
3 from constants import TESTBED_ID
9 import nepi.util.server as server
20 from nepi.util.constants import ApplicationStatus as AS
22 class Dependency(object):
24 A Dependency is in every respect like an application.
26 It depends on some packages, it may require building binaries, it must deploy
29 But it has no command. Dependencies aren't ever started, or stopped, and have
35 def __init__(self, api=None):
47 self.buildDepends = None
49 self.rpmFusion = False
57 self.add_to_path = True
59 # Those are filled when the app is configured
62 # Those are filled when an actual node is connected
65 # Those are filled when the app is started
66 # Having both pid and ppid makes it harder
67 # for pid rollover to induce tracking mistakes
74 # Spanning tree deployment
76 self._master_passphrase = None
77 self._master_prk = None
78 self._master_puk = None
79 self._master_token = os.urandom(8).encode("hex")
80 self._build_pid = None
81 self._build_ppid = None
84 self._logger = logging.getLogger('nepi.testbeds.planetlab')
89 self.__class__.__name__,
90 ' '.join(filter(bool,(self.depends, self.sources)))
94 if self.home_path is None:
95 raise AssertionError, "Misconfigured application: missing home path"
96 if self.node.ident_path is None or not os.access(self.node.ident_path, os.R_OK):
97 raise AssertionError, "Misconfigured application: missing slice SSH key"
99 raise AssertionError, "Misconfigured application: unconnected node"
100 if self.node.hostname is None:
101 raise AssertionError, "Misconfigured application: misconfigured node"
102 if self.node.slicename is None:
103 raise AssertionError, "Misconfigured application: unspecified slice"
105 def check_bad_host(self, out, err):
107 Called whenever an operation fails, it's given the output to be checked for
108 telltale signs of unhealthy hosts.
112 def remote_trace_path(self, whichtrace):
113 if whichtrace in self.TRACES:
114 tracefile = os.path.join(self.home_path, whichtrace)
120 def remote_trace_name(self, whichtrace):
121 if whichtrace in self.TRACES:
125 def sync_trace(self, local_dir, whichtrace):
126 tracefile = self.remote_trace_path(whichtrace)
130 local_path = os.path.join(local_dir, tracefile)
132 # create parent local folders
133 proc = subprocess.Popen(
134 ["mkdir", "-p", os.path.dirname(local_path)],
135 stdout = open("/dev/null","w"),
136 stdin = open("/dev/null","r"))
139 raise RuntimeError, "Failed to synchronize trace"
144 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
148 except RuntimeError, e:
149 raise RuntimeError, "Failed to synchronize trace: %s %s" \
150 % (e.args[0], e.args[1],)
155 # We assume a correct deployment, so recovery only
156 # means we mark this dependency as deployed
160 self._logger.info("Setting up %s", self)
166 def async_setup(self):
167 if not self._setuper:
172 self._setuper._exc.append(sys.exc_info())
173 self._setuper = threading.Thread(
175 self._setuper._exc = []
176 self._setuper.start()
178 def async_setup_wait(self):
180 self._logger.info("Waiting for %s to be setup", self)
184 if self._setuper._exc:
185 exctyp,exval,exctrace = self._setuper._exc[0]
186 raise exctyp,exval,exctrace
188 raise RuntimeError, "Failed to setup application"
190 self._logger.info("Setup ready: %s at %s", self, self.node.hostname)
194 def _make_home(self):
195 # Make sure all the paths are created where
196 # they have to be created for deployment
199 self._popen_ssh_command(
200 "mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" \
201 % { 'home' : server.shell_escape(self.home_path) },
205 except RuntimeError, e:
206 raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, e.args[0], e.args[1],)
209 # Write program input
212 cStringIO.StringIO(self.stdin),
213 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
214 os.path.join(self.home_path, 'stdin') ),
216 except RuntimeError, e:
217 raise RuntimeError, "Failed to set up application %s: %s %s" \
218 % (self.home_path, e.args[0], e.args[1],)
220 def _replace_paths(self, command):
222 Replace all special path tags with shell-escaped actual paths.
224 # need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
225 root = '' if self.home_path.startswith('/') else "${HOME}/"
227 .replace("${SOURCES}", root+server.shell_escape(self.home_path))
228 .replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
230 def _launch_build(self, trial=0):
231 if self._master is not None:
232 if not trial or self._master_prk is not None:
233 self._do_install_keys()
234 buildscript = self._do_build_slave()
236 buildscript = self._do_build_master()
238 if buildscript is not None:
239 self._logger.info("Building %s at %s", self, self.node.hostname)
241 # upload build script
245 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
246 os.path.join(self.home_path, 'nepi-build.sh') )
248 except RuntimeError, e:
249 raise RuntimeError, "Failed to set up application %s: %s %s" \
250 % (self.home_path, e.args[0], e.args[1],)
253 self._do_launch_build()
255 def _finish_build(self):
256 self._do_wait_build()
259 def _do_build_slave(self):
260 if not self.sources and not self.build:
263 # Create build script
267 sources = self.sources.split(' ')
269 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostip,
270 os.path.join(self._master.home_path, os.path.basename(source)),)
271 for source in sources
276 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostip,
277 os.path.join(self._master.home_path, 'build.tar.gz'),)
280 sshopts = "-o ConnectTimeout=30 -o ConnectionAttempts=3 -o ServerAliveInterval=30 -o TCPKeepAlive=yes"
282 launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
283 " && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
284 " && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" % \
286 'prk' : server.shell_escape(self._master_prk_name),
287 'puk' : server.shell_escape(self._master_puk_name),
290 kill_agent = "kill $SSH_AGENT_PID"
294 "echo 'Checking master reachability' ; "
295 "if ping -c 3 %(master_host)s && (. ./.ssh-agent.sh > /dev/null ; ssh -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s echo MASTER SAYS HI ) ; then "
296 "echo 'Master node reachable' ; "
298 "echo 'MASTER NODE UNREACHABLE' && "
301 ". ./.ssh-agent.sh ; "
302 "while [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s.retcode || /bin/true) != %(token)s ]] ; do sleep 5 ; done ; "
303 "if [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s || /bin/true) != %(token)s ]] ; then echo BAD TOKEN ; exit 1 ; fi ; "
306 'hostkey' : 'master_known_hosts',
307 'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostip),
308 'master_host' : self._master.node.hostip,
309 'token_path' : os.path.join(self._master.home_path, 'build.token'),
310 'token' : server.shell_escape(self._master._master_token),
314 syncfiles = ". ./.ssh-agent.sh && scp -p -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(files)s ." % {
315 'hostkey' : 'master_known_hosts',
316 'files' : ' '.join(files),
320 syncfiles += " && tar xzf build.tar.gz"
321 syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
322 syncfiles += " && ( echo %s > build.token.retcode )" % (server.shell_escape(self._master_token),)
323 syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
325 cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
326 'prk' : server.shell_escape(self._master_prk_name),
327 'puk' : server.shell_escape(self._master_puk_name),
330 slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s ) ; echo %(token)s > build.token.retcode" % {
331 'waitmaster' : waitmaster,
332 'syncfiles' : syncfiles,
334 'kill_agent' : kill_agent,
335 'launch_agent' : launch_agent,
336 'home' : server.shell_escape(self.home_path),
337 'token' : server.shell_escape(self._master_token),
340 return cStringIO.StringIO(slavescript)
342 def _do_launch_build(self):
343 script = "bash ./nepi-build.sh"
344 if self._master_passphrase:
345 script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
346 server.shell_escape(self._master_passphrase),
349 (out,err),proc = rspawn.remote_spawn(
351 pidfile = 'build-pid',
352 home = self.home_path,
355 stderr = rspawn.STDOUT,
357 host = self.node.hostname,
359 user = self.node.slicename,
361 ident_key = self.node.ident_path,
362 server_key = self.node.server_key,
363 hostip = self.node.hostip,
367 if self.check_bad_host(out, err):
368 self.node.blacklist()
369 raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
375 pidtuple = rspawn.remote_check_pid(
376 os.path.join(self.home_path,'build-pid'),
377 host = self.node.hostname,
379 user = self.node.slicename,
381 ident_key = self.node.ident_path,
382 server_key = self.node.server_key,
383 hostip = self.node.hostip
388 self._build_pid, self._build_ppid = pidtuple
392 delay = min(30,delay*1.2)
394 raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
396 self._logger.info("Deploying %s at %s", self, self.node.hostname)
398 def _do_wait_build(self, trial=0):
399 pid = self._build_pid
400 ppid = self._build_ppid
407 status = rspawn.remote_status(
409 host = self.node.hostname,
411 user = self.node.slicename,
413 ident_key = self.node.ident_path,
414 server_key = self.node.server_key,
415 hostip = self.node.hostip
418 if status is rspawn.FINISHED:
419 self._build_pid = self._build_ppid = None
421 elif status is not rspawn.RUNNING:
422 self._logger.warn("Busted waiting for %s to finish building at %s %s", self, self.node.hostname,
423 "(build slave)" if self._master is not None else "(build master)")
425 time.sleep(delay*(5.5+random.random()))
427 self._build_pid = self._build_ppid = None
431 self._logger.info("Waiting for %s to finish building at %s %s", self, self.node.hostname,
432 "(build slave)" if self._master is not None else "(build master)")
435 time.sleep(delay*(0.5+random.random()))
436 delay = min(30,delay*1.2)
442 (out, err), proc = self._popen_ssh_command(
443 "cat %(token_path)s" % {
444 'token_path' : os.path.join(self.home_path, 'build.token'),
448 if not proc.wait() and out:
449 slave_token = out.strip()
456 if slave_token != self._master_token:
457 # Get buildlog for the error message
459 (buildlog, err), proc = self._popen_ssh_command(
460 "cat %(buildlog)s" % {
461 'buildlog' : os.path.join(self.home_path, 'buildlog'),
462 'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
469 if self.check_bad_host(buildlog, err):
470 self.node.blacklist()
471 elif self._master and trial < 3 and 'BAD TOKEN' in buildlog or 'BAD TOKEN' in err:
472 # bad sync with master, may try again
473 # but first wait for master
474 self._master.async_setup_wait()
475 self._launch_build(trial+1)
476 return self._do_wait_build(trial+1)
478 return self._do_wait_build(trial+1)
481 self._master_prk = None
482 self._master_puk = None
484 raise RuntimeError, "Failed to set up application %s: "\
485 "build failed, got wrong token from pid %s/%s "\
486 "(expected %r, got %r), see buildlog at %s:\n%s" % (
487 self.home_path, pid, ppid, self._master_token, slave_token, self.node.hostname, buildlog)
490 self._master_prk = None
491 self._master_puk = None
493 self._logger.info("Built %s at %s", self, self.node.hostname)
495 def _do_kill_build(self):
496 pid = self._build_pid
497 ppid = self._build_ppid
500 self._logger.info("Killing build of %s", self)
503 host = self.node.hostname,
505 user = self.node.slicename,
507 ident_key = self.node.ident_path,
508 hostip = self.node.hostip
512 def _do_build_master(self):
513 if not self.sources and not self.build and not self.buildDepends:
517 sources = self.sources.split(' ')
523 "%s@%s:%s" % (self.node.slicename, self.node.hostname,
524 os.path.join(self.home_path,'.'),)
526 except RuntimeError, e:
527 raise RuntimeError, "Failed upload source file %r: %s %s" \
528 % (sources, e.args[0], e.args[1],)
530 buildscript = cStringIO.StringIO()
532 buildscript.write("(\n")
534 if self.buildDepends:
535 # Install build dependencies
537 "sudo -S yum -y install %(packages)s\n" % {
538 'packages' : self.buildDepends
546 "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
547 'command' : self._replace_paths(self.build),
548 'home' : server.shell_escape(self.home_path),
553 buildscript.write("tar czf build.tar.gz build\n")
556 buildscript.write("echo %(master_token)s > build.token ) ; echo %(master_token)s > build.token.retcode" % {
557 'master_token' : server.shell_escape(self._master_token)
564 def _do_install(self):
566 self._logger.info("Installing %s at %s", self, self.node.hostname)
568 # Install application
570 self._popen_ssh_command(
571 "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/{install,build}log >&2 && false )" % \
573 'command' : self._replace_paths(self.install),
574 'home' : server.shell_escape(self.home_path),
577 except RuntimeError, e:
578 if self.check_bad_host(e.args[0], e.args[1]):
579 self.node.blacklist()
580 raise RuntimeError, "Failed install build sources: %s %s" % (e.args[0], e.args[1],)
582 def set_master(self, master):
583 self._master = master
585 def install_keys(self, prk, puk, passphrase):
587 self._master_passphrase = passphrase
588 self._master_prk = prk
589 self._master_puk = puk
590 self._master_prk_name = os.path.basename(prk.name)
591 self._master_puk_name = os.path.basename(puk.name)
593 def _do_install_keys(self):
594 prk = self._master_prk
595 puk = self._master_puk
599 [ prk.name, puk.name ],
600 '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path )
602 except RuntimeError, e:
603 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
604 % (e.args[0], e.args[1],)
608 cStringIO.StringIO('%s,%s %s\n' % (
609 self._master.node.hostname, self._master.node.hostip,
610 self._master.node.server_key)),
611 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
612 os.path.join(self.home_path,"master_known_hosts") )
614 except RuntimeError, e:
615 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
616 % (e.args[0], e.args[1],)
620 # make sure there's no leftover build processes
621 self._do_kill_build()
624 self._master_prk = None
625 self._master_puk = None
628 def _popen_scp(self, src, dst, retry = 3):
631 (out,err),proc = server.popen_scp(
636 ident_key = self.node.ident_path,
637 server_key = self.node.server_key
640 if server.eintr_retry(proc.wait)():
641 raise RuntimeError, (out, err)
642 return (out, err), proc
651 def _popen_ssh_command(self, command, retry = 0, noerrors=False, timeout=None):
652 (out,err),proc = server.popen_ssh_command(
654 host = self.node.hostname,
656 user = self.node.slicename,
658 ident_key = self.node.ident_path,
659 server_key = self.node.server_key,
664 if server.eintr_retry(proc.wait)():
666 raise RuntimeError, (out, err)
667 return (out, err), proc
669 class Application(Dependency):
671 An application also has dependencies, but also a command to be ran and monitored.
673 It adds the output of that command as traces.
676 TRACES = ('stdout','stderr','buildlog', 'output')
678 def __init__(self, api=None):
679 super(Application,self).__init__(api)
690 # Those are filled when the app is started
691 # Having both pid and ppid makes it harder
692 # for pid rollover to induce tracking mistakes
693 self._started = False
697 # Do not add to the python path of nodes
698 self.add_to_path = False
701 return "%s<command:%s%s>" % (
702 self.__class__.__name__,
703 "sudo " if self.sudo else "",
708 self._logger.info("Starting %s", self)
710 # Create shell script with the command
711 # This way, complex commands and scripts can be ran seamlessly
713 command = cStringIO.StringIO()
714 command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
715 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
717 command.write('export PATH=$PATH:%s\n' % (
718 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
721 for envkey, envvals in self.node.env.iteritems():
722 for envval in envvals:
723 command.write('export %s=%s\n' % (envkey, envval))
724 command.write(self.command)
730 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
731 os.path.join(self.home_path, "app.sh"))
733 except RuntimeError, e:
734 raise RuntimeError, "Failed to set up application: %s %s" \
735 % (e.args[0], e.args[1],)
737 # Start process in a "daemonized" way, using nohup and heavy
738 # stdin/out redirection to avoid connection issues
739 (out,err),proc = rspawn.remote_spawn(
740 self._replace_paths("bash ./app.sh"),
743 home = self.home_path,
744 stdin = 'stdin' if self.stdin is not None else '/dev/null',
745 stdout = 'stdout' if self.stdout else '/dev/null',
746 stderr = 'stderr' if self.stderr else '/dev/null',
749 host = self.node.hostname,
751 user = self.node.slicename,
753 ident_key = self.node.ident_path,
754 server_key = self.node.server_key
758 if self.check_bad_host(out, err):
759 self.node.blacklist()
760 raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
765 # Assuming the application is running on PlanetLab,
766 # proper pidfiles should be present at the app's home path.
767 # So we mark this application as started, and check the pidfiles
773 # NOTE: wait a bit for the pidfile to be created
774 if self._started and not self._pid or not self._ppid:
775 pidtuple = rspawn.remote_check_pid(
776 os.path.join(self.home_path,'pid'),
777 host = self.node.hostname,
779 user = self.node.slicename,
781 ident_key = self.node.ident_path,
782 server_key = self.node.server_key
786 self._pid, self._ppid = pidtuple
790 if not self._started:
791 return AS.STATUS_NOT_STARTED
792 elif not self._pid or not self._ppid:
793 return AS.STATUS_NOT_STARTED
795 status = rspawn.remote_status(
796 self._pid, self._ppid,
797 host = self.node.hostname,
799 user = self.node.slicename,
801 ident_key = self.node.ident_path,
802 server_key = self.node.server_key
805 if status is rspawn.NOT_STARTED:
806 return AS.STATUS_NOT_STARTED
807 elif status is rspawn.RUNNING:
808 return AS.STATUS_RUNNING
809 elif status is rspawn.FINISHED:
810 return AS.STATUS_FINISHED
813 return AS.STATUS_NOT_STARTED
816 status = self.status()
817 if status == AS.STATUS_RUNNING:
818 # kill by ppid+pid - SIGTERM first, then try SIGKILL
820 self._pid, self._ppid,
821 host = self.node.hostname,
823 user = self.node.slicename,
825 ident_key = self.node.ident_path,
826 server_key = self.node.server_key,
829 self._logger.info("Killed %s", self)
832 class NepiDependency(Dependency):
834 This dependency adds nepi itself to the python path,
835 so that you may run testbeds within PL nodes.
838 # Class attribute holding a *weak* reference to the shared NEPI tar file
839 # so that they may share it. Don't operate on the file itself, it would
840 # be a mess, just use its path.
841 _shared_nepi_tar = None
843 def __init__(self, api = None):
844 super(NepiDependency, self).__init__(api)
848 self.depends = 'python python-ipaddr python-setuptools'
850 # our sources are in our ad-hoc tarball
851 self.sources = self.tarball.name
853 tarname = os.path.basename(self.tarball.name)
855 # it's already built - just move the tarball into place
856 self.build = "mv -f ${SOURCES}/%s ." % (tarname,)
858 # unpack it into sources, and we're done
859 self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,)
863 if self._tarball is None:
864 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
865 if shared_tar is not None:
866 self._tarball = shared_tar
868 # Build an ad-hoc tarball
873 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
875 proc = subprocess.Popen(
876 ["tar", "czf", shared_tar.name,
877 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
879 stdout = open("/dev/null","w"),
880 stdin = open("/dev/null","r"))
883 raise RuntimeError, "Failed to create nepi tarball"
885 self._tarball = self._shared_nepi_tar = shared_tar
889 class NS3Dependency(Dependency):
891 This dependency adds NS3 libraries to the library paths,
892 so that you may run the NS3 testbed within PL nodes.
894 You'll also need the NepiDependency.
897 def __init__(self, api = None):
898 super(NS3Dependency, self).__init__(api)
900 self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip'
902 # We have to download the sources, untar, build...
903 #pybindgen_source_url = "http://yans.pl.sophia.inria.fr/trac/nepi/raw-attachment/wiki/WikiStart/pybindgen-r794.tar.gz"
904 pybindgen_source_url = "http://yans.pl.sophia.inria.fr/trac/nepi/raw-attachment/wiki/WikiStart/pybindgen-r796.tar.gz"
905 pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
906 #ns3_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/ns-3.11-nepi/archive/tip.tar.gz"
907 ns3_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/ns-3.11-nepi/archive/ddf78e15b30e.tar.gz"
908 passfd_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/python-passfd/archive/tip.tar.gz"
912 " python -c 'import pygccxml, pybindgen, passfd' && "
913 " test -f lib/ns/_core.so && "
914 " test -f lib/ns/__init__.py && "
915 " test -f lib/ns/core.py && "
916 " test -f lib/libns3-core.so && "
917 " LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
919 # Not working, rebuild
920 # Archive SHA1 sums to check
921 "echo '7158877faff2254e6c094bf18e6b4283cac19137 pygccxml-1.0.0.zip' > archive_sums.txt && "
922 "echo '139ba938c62816bbdb82bf5644e4f3041608c634 pybindgen-src.tar.gz' >> archive_sums.txt && "
923 " ( " # check existing files
924 " sha1sum -c archive_sums.txt && "
925 " test -f passfd-src.tar.gz && "
926 " test -f ns3-src.tar.gz "
927 " ) || ( " # nope? re-download
928 " rm -f pybindgen-src.zip pygccxml-1.0.0.zip passfd-src.tar.gz ns3-src.tar.gz && "
929 " wget -q -c -O pybindgen-src.tar.gz %(pybindgen_source_url)s && " # continue, to exploit the case when it has already been dl'ed
930 " wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
931 " wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
932 " wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
933 " sha1sum -c archive_sums.txt " # Check SHA1 sums when applicable
935 "unzip -n pygccxml-1.0.0.zip && "
936 "mkdir -p pybindgen-src && "
937 "mkdir -p ns3-src && "
938 "mkdir -p passfd-src && "
939 "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
940 "tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
941 "tar xzf pybindgen-src.tar.gz --strip-components=1 -C pybindgen-src && "
942 "rm -rf target && " # mv doesn't like unclean targets
943 "mkdir -p target && "
944 "cd pygccxml-1.0.0 && "
945 "rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
946 "python setup.py build && "
947 "python setup.py install --install-lib ${BUILD}/target && "
948 "python setup.py clean && "
949 "cd ../pybindgen-src && "
950 "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
951 "./waf configure --prefix=${BUILD}/target -d release && "
955 "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
956 "rm -rf ${BUILD}/target/lib && "
957 "cd ../passfd-src && "
958 "python setup.py build && "
959 "python setup.py install --install-lib ${BUILD}/target && "
960 "python setup.py clean && "
962 "./waf configure --prefix=${BUILD}/target --with-pybindgen=../pybindgen-src -d release --disable-examples --disable-tests && "
965 "rm -f ${BUILD}/target/lib/*.so && "
966 "cp -a ${BUILD}/ns3-src/build/release/libns3*.so ${BUILD}/target/lib && "
967 "cp -a ${BUILD}/ns3-src/build/release/bindings/python/ns ${BUILD}/target/lib &&"
971 pybindgen_source_url = server.shell_escape(pybindgen_source_url),
972 pygccxml_source_url = server.shell_escape(pygccxml_source_url),
973 ns3_source_url = server.shell_escape(ns3_source_url),
974 passfd_source_url = server.shell_escape(passfd_source_url),
977 # Just move ${BUILD}/target
981 " python -c 'import pygccxml, pybindgen, passfd' && "
982 " test -f lib/ns/_core.so && "
983 " test -f lib/ns/__init__.py && "
984 " test -f lib/ns/core.py && "
985 " test -f lib/libns3-core.so && "
986 " LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
988 # Not working, reinstall
989 "test -d ${BUILD}/target && "
990 "[[ \"x\" != \"x$(find ${BUILD}/target -mindepth 1 -print -quit)\" ]] &&"
991 "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
992 "mv -f ${BUILD}/target/* ${SOURCES}"
996 # Set extra environment paths
997 self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
998 self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib"
1002 if self._tarball is None:
1003 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
1004 if shared_tar is not None:
1005 self._tarball = shared_tar
1007 # Build an ad-hoc tarball
1012 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
1014 proc = subprocess.Popen(
1015 ["tar", "czf", shared_tar.name,
1016 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
1018 stdout = open("/dev/null","w"),
1019 stdin = open("/dev/null","r"))
1022 raise RuntimeError, "Failed to create nepi tarball"
1024 self._tarball = self._shared_nepi_tar = shared_tar
1026 return self._tarball
1028 class YumDependency(Dependency):
1030 This dependency is an internal helper class used to
1031 efficiently distribute yum-downloaded rpms.
1033 It temporarily sets the yum cache as persistent in the
1034 build master, and installs all the required packages.
1036 The rpm packages left in the yum cache are gathered and
1037 distributed by the underlying Dependency in an efficient
1038 manner. Build slaves will then install those rpms back in
1039 the cache before issuing the install command.
1041 When packages have been installed already, nothing but an
1042 empty tar is distributed.
1045 # Class attribute holding a *weak* reference to the shared NEPI tar file
1046 # so that they may share it. Don't operate on the file itself, it would
1047 # be a mess, just use its path.
1048 _shared_nepi_tar = None
1050 def _build_get(self):
1051 # canonical representation of dependencies
1052 depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
1054 # download rpms and pack into a tar archive
1056 "sudo -S nice yum -y makecache && "
1057 "sudo -S sed -i -r 's/keepcache *= *0/keepcache=1/' /etc/yum.conf && "
1059 "sudo -S nice yum -y install %s ; "
1060 "rm -f ${BUILD}/packages.tar ; "
1061 "tar -C /var/cache/yum -rf ${BUILD}/packages.tar $(cd /var/cache/yum ; find -iname '*.rpm')"
1062 " ) || /bin/true ) && "
1063 "sudo -S sed -i -r 's/keepcache *= *1/keepcache=0/' /etc/yum.conf && "
1064 "( sudo -S nice yum -y clean packages || /bin/true ) "
1066 def _build_set(self, value):
1069 build = property(_build_get, _build_set)
1071 def _install_get(self):
1072 # canonical representation of dependencies
1073 depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
1075 # unpack cached rpms into yum cache, install, and cleanup
1077 "sudo -S tar -k --keep-newer-files -C /var/cache/yum -xf packages.tar && "
1078 "sudo -S nice yum -y install %s && "
1079 "( sudo -S nice yum -y clean packages || /bin/true ) "
1081 def _install_set(self, value):
1084 install = property(_install_get, _install_set)
1086 def check_bad_host(self, out, err):
1087 badre = re.compile(r'(?:'
1088 r'The GPG keys listed for the ".*" repository are already installed but they are not correct for this package'
1089 r'|Error: Cannot retrieve repository metadata (repomd.xml) for repository: .*[.] Please verify its path and try again'
1090 r'|Error: disk I/O error'
1091 r'|MASTER NODE UNREACHABLE'
1094 return badre.search(out) or badre.search(err) or self.node.check_bad_host(out,err)