2 # -*- coding: utf-8 -*-
4 from constants import TESTBED_ID
10 import nepi.util.server as server
21 from nepi.util.constants import ApplicationStatus as AS
23 class Dependency(object):
25 A Dependency is in every respect like an application.
27 It depends on some packages, it may require building binaries, it must deploy
30 But it has no command. Dependencies aren't ever started, or stopped, and have
36 def __init__(self, api=None):
48 self.buildDepends = None
50 self.rpmFusion = False
58 self.add_to_path = True
60 # Those are filled when the app is configured
63 # Those are filled when an actual node is connected
66 # Those are filled when the app is started
67 # Having both pid and ppid makes it harder
68 # for pid rollover to induce tracking mistakes
75 # Spanning tree deployment
77 self._master_passphrase = None
78 self._master_prk = None
79 self._master_puk = None
80 self._master_token = os.urandom(8).encode("hex")
81 self._build_pid = None
82 self._build_ppid = None
85 self._logger = logging.getLogger('nepi.testbeds.planetlab')
90 self.__class__.__name__,
91 ' '.join(filter(bool,(self.depends, self.sources)))
95 if self.home_path is None:
96 raise AssertionError, "Misconfigured application: missing home path"
97 if self.node.ident_path is None or not os.access(self.node.ident_path, os.R_OK):
98 raise AssertionError, "Misconfigured application: missing slice SSH key"
100 raise AssertionError, "Misconfigured application: unconnected node"
101 if self.node.hostname is None:
102 raise AssertionError, "Misconfigured application: misconfigured node"
103 if self.node.slicename is None:
104 raise AssertionError, "Misconfigured application: unspecified slice"
106 def check_bad_host(self, out, err):
108 Called whenever an operation fails, it's given the output to be checked for
109 telltale signs of unhealthy hosts.
113 def remote_trace_path(self, whichtrace):
114 if whichtrace in self.TRACES:
115 tracefile = os.path.join(self.home_path, whichtrace)
121 def remote_trace_name(self, whichtrace):
122 if whichtrace in self.TRACES:
126 def sync_trace(self, local_dir, whichtrace):
127 tracefile = self.remote_trace_path(whichtrace)
131 local_path = os.path.join(local_dir, tracefile)
133 # create parent local folders
134 proc = subprocess.Popen(
135 ["mkdir", "-p", os.path.dirname(local_path)],
136 stdout = open("/dev/null","w"),
137 stdin = open("/dev/null","r"))
140 raise RuntimeError, "Failed to synchronize trace"
145 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
149 except RuntimeError, e:
150 raise RuntimeError, "Failed to synchronize trace: %s %s" \
151 % (e.args[0], e.args[1],)
156 # We assume a correct deployment, so recovery only
157 # means we mark this dependency as deployed
161 self._logger.info("Setting up %s", self)
167 def async_setup(self):
168 if not self._setuper:
173 self._setuper._exc.append(sys.exc_info())
174 self._setuper = threading.Thread(
176 self._setuper._exc = []
177 self._setuper.start()
179 def async_setup_wait(self):
181 self._logger.info("Waiting for %s to be setup", self)
185 if self._setuper._exc:
186 exctyp,exval,exctrace = self._setuper._exc[0]
187 raise exctyp,exval,exctrace
189 raise RuntimeError, "Failed to setup application"
191 self._logger.info("Setup ready: %s at %s", self, self.node.hostname)
195 def _make_home(self):
196 # Make sure all the paths are created where
197 # they have to be created for deployment
200 self._popen_ssh_command(
201 "mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" \
202 % { 'home' : server.shell_escape(self.home_path) },
206 except RuntimeError, e:
207 raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, e.args[0], e.args[1],)
210 # Write program input
213 cStringIO.StringIO(self.stdin),
214 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
215 os.path.join(self.home_path, 'stdin') ),
217 except RuntimeError, e:
218 raise RuntimeError, "Failed to set up application %s: %s %s" \
219 % (self.home_path, e.args[0], e.args[1],)
221 def _replace_paths(self, command):
223 Replace all special path tags with shell-escaped actual paths.
225 # need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
226 root = '' if self.home_path.startswith('/') else "${HOME}/"
228 .replace("${SOURCES}", root+server.shell_escape(self.home_path))
229 .replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
231 def _launch_build(self, trial=0):
232 if self._master is not None:
233 if not trial or self._master_prk is not None:
234 self._do_install_keys()
235 buildscript = self._do_build_slave()
237 buildscript = self._do_build_master()
239 if buildscript is not None:
240 self._logger.info("Building %s at %s", self, self.node.hostname)
242 # upload build script
246 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
247 os.path.join(self.home_path, 'nepi-build.sh') )
249 except RuntimeError, e:
250 raise RuntimeError, "Failed to set up application %s: %s %s" \
251 % (self.home_path, e.args[0], e.args[1],)
254 self._do_launch_build()
256 def _finish_build(self):
257 self._do_wait_build()
260 def _do_build_slave(self):
261 if not self.sources and not self.build:
264 # Create build script
268 sources = self.sources.split(' ')
270 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostip,
271 os.path.join(self._master.home_path, os.path.basename(source)),)
272 for source in sources
277 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostip,
278 os.path.join(self._master.home_path, 'build.tar.gz'),)
281 sshopts = "-o ConnectTimeout=30 -o ConnectionAttempts=3 -o ServerAliveInterval=30 -o TCPKeepAlive=yes"
283 launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
284 " && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
285 " && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" % \
287 'prk' : server.shell_escape(self._master_prk_name),
288 'puk' : server.shell_escape(self._master_puk_name),
291 kill_agent = "kill $SSH_AGENT_PID"
295 "echo 'Checking master reachability' ; "
296 "if ping -c 3 %(master_host)s && (. ./.ssh-agent.sh > /dev/null ; ssh -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s echo MASTER SAYS HI ) ; then "
297 "echo 'Master node reachable' ; "
299 "echo 'MASTER NODE UNREACHABLE' && "
302 ". ./.ssh-agent.sh ; "
303 "while [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s.retcode || /bin/true) != %(token)s ]] ; do sleep 5 ; done ; "
304 "if [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s || /bin/true) != %(token)s ]] ; then echo BAD TOKEN ; exit 1 ; fi ; "
307 'hostkey' : 'master_known_hosts',
308 'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostip),
309 'master_host' : self._master.node.hostip,
310 'token_path' : os.path.join(self._master.home_path, 'build.token'),
311 'token' : server.shell_escape(self._master._master_token),
315 syncfiles = ". ./.ssh-agent.sh && scp -p -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(files)s ." % {
316 'hostkey' : 'master_known_hosts',
317 'files' : ' '.join(files),
321 syncfiles += " && tar xzf build.tar.gz"
322 syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
323 syncfiles += " && ( echo %s > build.token.retcode )" % (server.shell_escape(self._master_token),)
324 syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
326 cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
327 'prk' : server.shell_escape(self._master_prk_name),
328 'puk' : server.shell_escape(self._master_puk_name),
331 slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s ) ; echo %(token)s > build.token.retcode" % {
332 'waitmaster' : waitmaster,
333 'syncfiles' : syncfiles,
335 'kill_agent' : kill_agent,
336 'launch_agent' : launch_agent,
337 'home' : server.shell_escape(self.home_path),
338 'token' : server.shell_escape(self._master_token),
341 return cStringIO.StringIO(slavescript)
343 def _do_launch_build(self):
344 script = "bash ./nepi-build.sh"
345 if self._master_passphrase:
346 script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
347 server.shell_escape(self._master_passphrase),
350 (out,err),proc = rspawn.remote_spawn(
352 pidfile = 'build-pid',
353 home = self.home_path,
356 stderr = rspawn.STDOUT,
358 host = self.node.hostname,
360 user = self.node.slicename,
362 ident_key = self.node.ident_path,
363 server_key = self.node.server_key,
364 hostip = self.node.hostip,
368 if self.check_bad_host(out, err):
369 self.node.blacklist()
370 raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
376 pidtuple = rspawn.remote_check_pid(
377 os.path.join(self.home_path,'build-pid'),
378 host = self.node.hostname,
380 user = self.node.slicename,
382 ident_key = self.node.ident_path,
383 server_key = self.node.server_key,
384 hostip = self.node.hostip
389 self._build_pid, self._build_ppid = pidtuple
393 delay = min(30,delay*1.2)
395 raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
397 self._logger.info("Deploying %s at %s", self, self.node.hostname)
399 def _do_wait_build(self, trial=0):
400 pid = self._build_pid
401 ppid = self._build_ppid
408 status = rspawn.remote_status(
410 host = self.node.hostname,
412 user = self.node.slicename,
414 ident_key = self.node.ident_path,
415 server_key = self.node.server_key,
416 hostip = self.node.hostip
419 if status is rspawn.FINISHED:
420 self._build_pid = self._build_ppid = None
422 elif status is not rspawn.RUNNING:
423 self._logger.warn("Busted waiting for %s to finish building at %s %s", self, self.node.hostname,
424 "(build slave)" if self._master is not None else "(build master)")
426 time.sleep(delay*(5.5+random.random()))
428 self._build_pid = self._build_ppid = None
432 self._logger.info("Waiting for %s to finish building at %s %s", self, self.node.hostname,
433 "(build slave)" if self._master is not None else "(build master)")
436 time.sleep(delay*(0.5+random.random()))
437 delay = min(30,delay*1.2)
443 (out, err), proc = self._popen_ssh_command(
444 "cat %(token_path)s" % {
445 'token_path' : os.path.join(self.home_path, 'build.token'),
449 if not proc.wait() and out:
450 slave_token = out.strip()
457 if slave_token != self._master_token:
458 # Get buildlog for the error message
460 (buildlog, err), proc = self._popen_ssh_command(
461 "cat %(buildlog)s" % {
462 'buildlog' : os.path.join(self.home_path, 'buildlog'),
463 'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
470 if self.check_bad_host(buildlog, err):
471 self.node.blacklist()
472 elif self._master and trial < 3 and 'BAD TOKEN' in buildlog or 'BAD TOKEN' in err:
473 # bad sync with master, may try again
474 # but first wait for master
475 self._master.async_setup_wait()
476 self._launch_build(trial+1)
477 self._do_wait_build(trial+1)
480 self._master_prk = None
481 self._master_puk = None
483 raise RuntimeError, "Failed to set up application %s: "\
484 "build failed, got wrong token from pid %s/%s "\
485 "(expected %r, got %r), see buildlog at %s:\n%s" % (
486 self.home_path, pid, ppid, self._master_token, slave_token, self.node.hostname, buildlog)
489 self._master_prk = None
490 self._master_puk = None
492 self._logger.info("Built %s at %s", self, self.node.hostname)
494 def _do_kill_build(self):
495 pid = self._build_pid
496 ppid = self._build_ppid
499 self._logger.info("Killing build of %s", self)
502 host = self.node.hostname,
504 user = self.node.slicename,
506 ident_key = self.node.ident_path,
507 hostip = self.node.hostip
511 def _do_build_master(self):
512 if not self.sources and not self.build and not self.buildDepends:
516 sources = self.sources.split(' ')
522 "%s@%s:%s" % (self.node.slicename, self.node.hostname,
523 os.path.join(self.home_path,'.'),)
525 except RuntimeError, e:
526 raise RuntimeError, "Failed upload source file %r: %s %s" \
527 % (sources, e.args[0], e.args[1],)
529 buildscript = cStringIO.StringIO()
531 buildscript.write("(\n")
533 if self.buildDepends:
534 # Install build dependencies
536 "sudo -S yum -y install %(packages)s\n" % {
537 'packages' : self.buildDepends
545 "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
546 'command' : self._replace_paths(self.build),
547 'home' : server.shell_escape(self.home_path),
552 buildscript.write("tar czf build.tar.gz build\n")
555 buildscript.write("echo %(master_token)s > build.token ) ; echo %(master_token)s > build.token.retcode" % {
556 'master_token' : server.shell_escape(self._master_token)
563 def _do_install(self):
565 self._logger.info("Installing %s at %s", self, self.node.hostname)
567 # Install application
569 self._popen_ssh_command(
570 "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/{install,build}log >&2 && false )" % \
572 'command' : self._replace_paths(self.install),
573 'home' : server.shell_escape(self.home_path),
576 except RuntimeError, e:
577 if self.check_bad_host(e.args[0], e.args[1]):
578 self.node.blacklist()
579 raise RuntimeError, "Failed install build sources: %s %s" % (e.args[0], e.args[1],)
581 def set_master(self, master):
582 self._master = master
584 def install_keys(self, prk, puk, passphrase):
586 self._master_passphrase = passphrase
587 self._master_prk = prk
588 self._master_puk = puk
589 self._master_prk_name = os.path.basename(prk.name)
590 self._master_puk_name = os.path.basename(puk.name)
592 def _do_install_keys(self):
593 prk = self._master_prk
594 puk = self._master_puk
598 [ prk.name, puk.name ],
599 '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path )
601 except RuntimeError, e:
602 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
603 % (e.args[0], e.args[1],)
607 cStringIO.StringIO('%s,%s %s\n' % (
608 self._master.node.hostname, self._master.node.hostip,
609 self._master.node.server_key)),
610 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
611 os.path.join(self.home_path,"master_known_hosts") )
613 except RuntimeError, e:
614 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
615 % (e.args[0], e.args[1],)
619 # make sure there's no leftover build processes
620 self._do_kill_build()
623 self._master_prk = None
624 self._master_puk = None
627 def _popen_scp(self, src, dst, retry = 3):
630 (out,err),proc = server.popen_scp(
635 ident_key = self.node.ident_path,
636 server_key = self.node.server_key
639 if server.eintr_retry(proc.wait)():
640 raise RuntimeError, (out, err)
641 return (out, err), proc
650 def _popen_ssh_command(self, command, retry = 0, noerrors=False, timeout=None):
651 (out,err),proc = server.popen_ssh_command(
653 host = self.node.hostname,
655 user = self.node.slicename,
657 ident_key = self.node.ident_path,
658 server_key = self.node.server_key,
663 if server.eintr_retry(proc.wait)():
665 raise RuntimeError, (out, err)
666 return (out, err), proc
668 class Application(Dependency):
670 An application also has dependencies, but also a command to be ran and monitored.
672 It adds the output of that command as traces.
675 TRACES = ('stdout','stderr','buildlog', 'output')
677 def __init__(self, api=None):
678 super(Application,self).__init__(api)
689 # Those are filled when the app is started
690 # Having both pid and ppid makes it harder
691 # for pid rollover to induce tracking mistakes
692 self._started = False
696 # Do not add to the python path of nodes
697 self.add_to_path = False
700 return "%s<command:%s%s>" % (
701 self.__class__.__name__,
702 "sudo " if self.sudo else "",
707 self._logger.info("Starting %s", self)
709 # Create shell script with the command
710 # This way, complex commands and scripts can be ran seamlessly
712 command = cStringIO.StringIO()
713 command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
714 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
716 command.write('export PATH=$PATH:%s\n' % (
717 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
720 for envkey, envvals in self.node.env.iteritems():
721 for envval in envvals:
722 command.write('export %s=%s\n' % (envkey, envval))
723 command.write(self.command)
729 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
730 os.path.join(self.home_path, "app.sh"))
732 except RuntimeError, e:
733 raise RuntimeError, "Failed to set up application: %s %s" \
734 % (e.args[0], e.args[1],)
736 # Start process in a "daemonized" way, using nohup and heavy
737 # stdin/out redirection to avoid connection issues
738 (out,err),proc = rspawn.remote_spawn(
739 self._replace_paths("bash ./app.sh"),
742 home = self.home_path,
743 stdin = 'stdin' if self.stdin is not None else '/dev/null',
744 stdout = 'stdout' if self.stdout else '/dev/null',
745 stderr = 'stderr' if self.stderr else '/dev/null',
748 host = self.node.hostname,
750 user = self.node.slicename,
752 ident_key = self.node.ident_path,
753 server_key = self.node.server_key
757 if self.check_bad_host(out, err):
758 self.node.blacklist()
759 raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
764 # Assuming the application is running on PlanetLab,
765 # proper pidfiles should be present at the app's home path.
766 # So we mark this application as started, and check the pidfiles
772 # NOTE: wait a bit for the pidfile to be created
773 if self._started and not self._pid or not self._ppid:
774 pidtuple = rspawn.remote_check_pid(
775 os.path.join(self.home_path,'pid'),
776 host = self.node.hostname,
778 user = self.node.slicename,
780 ident_key = self.node.ident_path,
781 server_key = self.node.server_key
785 self._pid, self._ppid = pidtuple
789 if not self._started:
790 return AS.STATUS_NOT_STARTED
791 elif not self._pid or not self._ppid:
792 return AS.STATUS_NOT_STARTED
794 status = rspawn.remote_status(
795 self._pid, self._ppid,
796 host = self.node.hostname,
798 user = self.node.slicename,
800 ident_key = self.node.ident_path,
801 server_key = self.node.server_key
804 if status is rspawn.NOT_STARTED:
805 return AS.STATUS_NOT_STARTED
806 elif status is rspawn.RUNNING:
807 return AS.STATUS_RUNNING
808 elif status is rspawn.FINISHED:
809 return AS.STATUS_FINISHED
812 return AS.STATUS_NOT_STARTED
815 status = self.status()
816 if status == AS.STATUS_RUNNING:
817 # kill by ppid+pid - SIGTERM first, then try SIGKILL
819 self._pid, self._ppid,
820 host = self.node.hostname,
822 user = self.node.slicename,
824 ident_key = self.node.ident_path,
825 server_key = self.node.server_key,
828 self._logger.info("Killed %s", self)
831 class NepiDependency(Dependency):
833 This dependency adds nepi itself to the python path,
834 so that you may run testbeds within PL nodes.
837 # Class attribute holding a *weak* reference to the shared NEPI tar file
838 # so that they may share it. Don't operate on the file itself, it would
839 # be a mess, just use its path.
840 _shared_nepi_tar = None
842 def __init__(self, api = None):
843 super(NepiDependency, self).__init__(api)
847 self.depends = 'python python-ipaddr python-setuptools'
849 # our sources are in our ad-hoc tarball
850 self.sources = self.tarball.name
852 tarname = os.path.basename(self.tarball.name)
854 # it's already built - just move the tarball into place
855 self.build = "mv -f ${SOURCES}/%s ." % (tarname,)
857 # unpack it into sources, and we're done
858 self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,)
862 if self._tarball is None:
863 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
864 if shared_tar is not None:
865 self._tarball = shared_tar
867 # Build an ad-hoc tarball
872 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
874 proc = subprocess.Popen(
875 ["tar", "czf", shared_tar.name,
876 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
878 stdout = open("/dev/null","w"),
879 stdin = open("/dev/null","r"))
882 raise RuntimeError, "Failed to create nepi tarball"
884 self._tarball = self._shared_nepi_tar = shared_tar
888 class NS3Dependency(Dependency):
890 This dependency adds NS3 libraries to the library paths,
891 so that you may run the NS3 testbed within PL nodes.
893 You'll also need the NepiDependency.
896 def __init__(self, api = None):
897 super(NS3Dependency, self).__init__(api)
899 self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip'
901 # We have to download the sources, untar, build...
902 pybindgen_source_url = "http://yans.pl.sophia.inria.fr/trac/nepi/raw-attachment/wiki/WikiStart/pybindgen-r794.tar.gz"
903 pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
904 ns3_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/ns-3.11-nepi/archive/tip.tar.gz"
905 passfd_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/python-passfd/archive/tip.tar.gz"
909 " python -c 'import pygccxml, pybindgen, passfd' && "
910 " test -f lib/ns/_core.so && "
911 " test -f lib/ns/__init__.py && "
912 " test -f lib/ns/core.py && "
913 " test -f lib/libns3-core.so && "
914 " LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
916 # Not working, rebuild
917 # Archive SHA1 sums to check
918 "echo '7158877faff2254e6c094bf18e6b4283cac19137 pygccxml-1.0.0.zip' > archive_sums.txt && "
919 "echo 'a18c2ccffd0df517bc37e2f3a2475092517c43f2 pybindgen-src.tar.gz' >> archive_sums.txt && "
920 " ( " # check existing files
921 " sha1sum -c archive_sums.txt && "
922 " test -f passfd-src.tar.gz && "
923 " test -f ns3-src.tar.gz "
924 " ) || ( " # nope? re-download
925 " rm -f pybindgen-src.zip pygccxml-1.0.0.zip passfd-src.tar.gz ns3-src.tar.gz && "
926 " wget -q -c -O pybindgen-src.tar.gz %(pybindgen_source_url)s && " # continue, to exploit the case when it has already been dl'ed
927 " wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
928 " wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
929 " wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
930 " sha1sum -c archive_sums.txt " # Check SHA1 sums when applicable
932 "unzip -n pygccxml-1.0.0.zip && "
933 "mkdir -p pybindgen-src && "
934 "mkdir -p ns3-src && "
935 "mkdir -p passfd-src && "
936 "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
937 "tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
938 "tar xzf pybindgen-src.tar.gz --strip-components=1 -C pybindgen-src && "
939 "rm -rf target && " # mv doesn't like unclean targets
940 "mkdir -p target && "
941 "cd pygccxml-1.0.0 && "
942 "rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
943 "python setup.py build && "
944 "python setup.py install --install-lib ${BUILD}/target && "
945 "python setup.py clean && "
946 "cd ../pybindgen-src && "
947 "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
948 "./waf configure --prefix=${BUILD}/target -d release && "
952 "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
953 "rm -rf ${BUILD}/target/lib && "
954 "cd ../passfd-src && "
955 "python setup.py build && "
956 "python setup.py install --install-lib ${BUILD}/target && "
957 "python setup.py clean && "
959 "./waf configure --prefix=${BUILD}/target --with-pybindgen=../pybindgen-src -d release --disable-examples --disable-tests && "
962 "rm -f ${BUILD}/target/lib/*.so && "
963 "cp -a ${BUILD}/ns3-src/build/release/libns3*.so ${BUILD}/target/lib && "
964 "cp -a ${BUILD}/ns3-src/build/release/bindings/python/ns ${BUILD}/target/lib &&"
968 pybindgen_source_url = server.shell_escape(pybindgen_source_url),
969 pygccxml_source_url = server.shell_escape(pygccxml_source_url),
970 ns3_source_url = server.shell_escape(ns3_source_url),
971 passfd_source_url = server.shell_escape(passfd_source_url),
974 # Just move ${BUILD}/target
978 " python -c 'import pygccxml, pybindgen, passfd' && "
979 " test -f lib/ns/_core.so && "
980 " test -f lib/ns/__init__.py && "
981 " test -f lib/ns/core.py && "
982 " test -f lib/libns3-core.so && "
983 " LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
985 # Not working, reinstall
986 "test -d ${BUILD}/target && "
987 "[[ \"x\" != \"x$(find ${BUILD}/target -mindepth 1 -print -quit)\" ]] &&"
988 "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
989 "mv -f ${BUILD}/target/* ${SOURCES}"
993 # Set extra environment paths
994 self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
995 self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib"
999 if self._tarball is None:
1000 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
1001 if shared_tar is not None:
1002 self._tarball = shared_tar
1004 # Build an ad-hoc tarball
1009 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
1011 proc = subprocess.Popen(
1012 ["tar", "czf", shared_tar.name,
1013 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
1015 stdout = open("/dev/null","w"),
1016 stdin = open("/dev/null","r"))
1019 raise RuntimeError, "Failed to create nepi tarball"
1021 self._tarball = self._shared_nepi_tar = shared_tar
1023 return self._tarball
1025 class YumDependency(Dependency):
1027 This dependency is an internal helper class used to
1028 efficiently distribute yum-downloaded rpms.
1030 It temporarily sets the yum cache as persistent in the
1031 build master, and installs all the required packages.
1033 The rpm packages left in the yum cache are gathered and
1034 distributed by the underlying Dependency in an efficient
1035 manner. Build slaves will then install those rpms back in
1036 the cache before issuing the install command.
1038 When packages have been installed already, nothing but an
1039 empty tar is distributed.
1042 # Class attribute holding a *weak* reference to the shared NEPI tar file
1043 # so that they may share it. Don't operate on the file itself, it would
1044 # be a mess, just use its path.
1045 _shared_nepi_tar = None
1047 def _build_get(self):
1048 # canonical representation of dependencies
1049 depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
1051 # download rpms and pack into a tar archive
1053 "sudo -S nice yum -y makecache && "
1054 "sudo -S sed -i -r 's/keepcache *= *0/keepcache=1/' /etc/yum.conf && "
1056 "sudo -S nice yum -y install %s ; "
1057 "rm -f ${BUILD}/packages.tar ; "
1058 "tar -C /var/cache/yum -rf ${BUILD}/packages.tar $(cd /var/cache/yum ; find -iname '*.rpm')"
1059 " ) || /bin/true ) && "
1060 "sudo -S sed -i -r 's/keepcache *= *1/keepcache=0/' /etc/yum.conf && "
1061 "( sudo -S nice yum -y clean packages || /bin/true ) "
1063 def _build_set(self, value):
1066 build = property(_build_get, _build_set)
1068 def _install_get(self):
1069 # canonical representation of dependencies
1070 depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
1072 # unpack cached rpms into yum cache, install, and cleanup
1074 "sudo -S tar -k --keep-newer-files -C /var/cache/yum -xf packages.tar && "
1075 "sudo -S nice yum -y install %s && "
1076 "( sudo -S nice yum -y clean packages || /bin/true ) "
1078 def _install_set(self, value):
1081 install = property(_install_get, _install_set)
1083 def check_bad_host(self, out, err):
1084 badre = re.compile(r'(?:'
1085 r'The GPG keys listed for the ".*" repository are already installed but they are not correct for this package'
1086 r'|Error: Cannot retrieve repository metadata (repomd.xml) for repository: .*[.] Please verify its path and try again'
1087 r'|Error: disk I/O error'
1088 r'|MASTER NODE UNREACHABLE'
1091 return badre.search(out) or badre.search(err)