1 # -*- coding: utf-8 -*-
3 from constants import TESTBED_ID
9 import nepi.util.server as server
20 from nepi.util.constants import ApplicationStatus as AS
22 _ccnre = re.compile("\s*(udp|tcp)\s+(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\s*$")
24 class Dependency(object):
26 A Dependency is in every respect like an application.
28 It depends on some packages, it may require building binaries, it must deploy
31 But it has no command. Dependencies aren't ever started, or stopped, and have
37 def __init__(self, api=None):
49 self.buildDepends = None
51 self.rpmFusion = False
59 self.add_to_path = True
61 # Those are filled when the app is configured
64 # Those are filled when an actual node is connected
67 # Those are filled when the app is started
68 # Having both pid and ppid makes it harder
69 # for pid rollover to induce tracking mistakes
76 # Spanning tree deployment
78 self._master_passphrase = None
79 self._master_prk = None
80 self._master_puk = None
81 self._master_token = os.urandom(8).encode("hex")
82 self._build_pid = None
83 self._build_ppid = None
86 self._logger = logging.getLogger('nepi.testbeds.planetlab')
91 self.__class__.__name__,
92 ' '.join(filter(bool,(self.depends, self.sources)))
99 if self.home_path is None:
100 raise AssertionError, "Misconfigured application: missing home path"
101 if self.node.ident_path is None or not os.access(self.node.ident_path, os.R_OK):
102 raise AssertionError, "Misconfigured application: missing slice SSH key"
103 if self.node is None:
104 raise AssertionError, "Misconfigured application: unconnected node"
105 if self.node.hostname is None:
106 raise AssertionError, "Misconfigured application: misconfigured node"
107 if self.node.slicename is None:
108 raise AssertionError, "Misconfigured application: unspecified slice"
110 def check_bad_host(self, out, err):
112 Called whenever an operation fails, it's given the output to be checked for
113 telltale signs of unhealthy hosts.
117 def remote_trace_path(self, whichtrace):
118 if whichtrace in self.TRACES:
119 tracefile = os.path.join(self.home_path, whichtrace)
125 def remote_trace_name(self, whichtrace):
126 if whichtrace in self.TRACES:
130 def sync_trace(self, local_dir, whichtrace):
131 tracefile = self.remote_trace_path(whichtrace)
135 local_path = os.path.join(local_dir, tracefile)
137 # create parent local folders
138 proc = subprocess.Popen(
139 ["mkdir", "-p", os.path.dirname(local_path)],
140 stdout = open("/dev/null","w"),
141 stdin = open("/dev/null","r"))
144 raise RuntimeError, "Failed to synchronize trace"
149 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
153 except RuntimeError, e:
154 raise RuntimeError, "Failed to synchronize trace: %s %s" \
155 % (e.args[0], e.args[1],)
160 # We assume a correct deployment, so recovery only
161 # means we mark this dependency as deployed
165 self._logger.info("Setting up %s", self)
171 def async_setup(self):
172 if not self._setuper:
177 self._setuper._exc.append(sys.exc_info())
178 self._setuper = threading.Thread(
180 self._setuper._exc = []
181 self._setuper.start()
183 def async_setup_wait(self):
185 self._logger.info("Waiting for %s to be setup", self)
189 if self._setuper._exc:
190 exctyp,exval,exctrace = self._setuper._exc[0]
191 raise exctyp,exval,exctrace
193 raise RuntimeError, "Failed to setup application"
195 self._logger.info("Setup ready: %s at %s", self, self.node.hostname)
199 def _make_home(self):
200 # Make sure all the paths are created where
201 # they have to be created for deployment
204 self._popen_ssh_command(
205 "mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" \
206 % { 'home' : server.shell_escape(self.home_path) },
210 except RuntimeError, e:
211 raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, e.args[0], e.args[1],)
215 if not os.path.isfile(stdin):
216 stdin = cStringIO.StringIO(self.stdin)
218 # Write program input
220 self._popen_scp(stdin,
221 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
222 os.path.join(self.home_path, 'stdin') ),
224 except RuntimeError, e:
225 raise RuntimeError, "Failed to set up application %s: %s %s" \
226 % (self.home_path, e.args[0], e.args[1],)
228 def _replace_paths(self, command):
230 Replace all special path tags with shell-escaped actual paths.
232 # need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
233 root = '' if self.home_path.startswith('/') else "${HOME}/"
235 .replace("${SOURCES}", root+server.shell_escape(self.home_path))
236 .replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
238 def _launch_build(self, trial=0):
239 if self._master is not None:
240 if not trial or self._master_prk is not None:
241 self._do_install_keys()
242 buildscript = self._do_build_slave()
244 buildscript = self._do_build_master()
246 if buildscript is not None:
247 self._logger.info("Building %s at %s", self, self.node.hostname)
249 # upload build script
253 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
254 os.path.join(self.home_path, 'nepi-build.sh') )
256 except RuntimeError, e:
257 raise RuntimeError, "Failed to set up application %s: %s %s" \
258 % (self.home_path, e.args[0], e.args[1],)
261 self._do_launch_build()
263 def _finish_build(self):
264 self._do_wait_build()
267 def _do_build_slave(self):
268 if not self.sources and not self.build:
271 # Create build script
275 sources = self.sources.split(' ')
277 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostip,
278 os.path.join(self._master.home_path, os.path.basename(source)),)
279 for source in sources
284 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostip,
285 os.path.join(self._master.home_path, 'build.tar.gz'),)
288 sshopts = "-o ConnectTimeout=30 -o ConnectionAttempts=3 -o ServerAliveInterval=30 -o TCPKeepAlive=yes"
290 launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
291 " && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
292 " && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" % \
294 'prk' : server.shell_escape(self._master_prk_name),
295 'puk' : server.shell_escape(self._master_puk_name),
298 kill_agent = "kill $SSH_AGENT_PID"
302 "echo 'Checking master reachability' ; "
303 "if ping -c 3 %(master_host)s && (. ./.ssh-agent.sh > /dev/null ; ssh -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s echo MASTER SAYS HI ) ; then "
304 "echo 'Master node reachable' ; "
306 "echo 'MASTER NODE UNREACHABLE' && "
309 ". ./.ssh-agent.sh ; "
310 "while [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s.retcode || /bin/true) != %(token)s ]] ; do sleep 5 ; done ; "
311 "if [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s || /bin/true) != %(token)s ]] ; then echo BAD TOKEN ; exit 1 ; fi ; "
314 'hostkey' : 'master_known_hosts',
315 'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostip),
316 'master_host' : self._master.node.hostip,
317 'token_path' : os.path.join(self._master.home_path, 'build.token'),
318 'token' : server.shell_escape(self._master._master_token),
322 syncfiles = ". ./.ssh-agent.sh && scp -p -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(files)s ." % {
323 'hostkey' : 'master_known_hosts',
324 'files' : ' '.join(files),
328 syncfiles += " && tar xzf build.tar.gz"
329 syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
330 syncfiles += " && ( echo %s > build.token.retcode )" % (server.shell_escape(self._master_token),)
331 syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
333 cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
334 'prk' : server.shell_escape(self._master_prk_name),
335 'puk' : server.shell_escape(self._master_puk_name),
338 slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s ) ; echo %(token)s > build.token.retcode" % {
339 'waitmaster' : waitmaster,
340 'syncfiles' : syncfiles,
342 'kill_agent' : kill_agent,
343 'launch_agent' : launch_agent,
344 'home' : server.shell_escape(self.home_path),
345 'token' : server.shell_escape(self._master_token),
348 return cStringIO.StringIO(slavescript)
350 def _do_launch_build(self):
351 script = "bash ./nepi-build.sh"
352 if self._master_passphrase:
353 script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
354 server.shell_escape(self._master_passphrase),
357 (out,err),proc = rspawn.remote_spawn(
359 pidfile = 'build-pid',
360 home = self.home_path,
363 stderr = rspawn.STDOUT,
365 host = self.node.hostname,
367 user = self.node.slicename,
369 ident_key = self.node.ident_path,
370 server_key = self.node.server_key,
371 hostip = self.node.hostip,
375 if self.check_bad_host(out, err):
376 self.node.blacklist()
377 raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
383 pidtuple = rspawn.remote_check_pid(
384 os.path.join(self.home_path,'build-pid'),
385 host = self.node.hostname,
387 user = self.node.slicename,
389 ident_key = self.node.ident_path,
390 server_key = self.node.server_key,
391 hostip = self.node.hostip
396 self._build_pid, self._build_ppid = pidtuple
400 delay = min(30,delay*1.2)
402 raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
404 self._logger.info("Deploying %s at %s", self, self.node.hostname)
406 def _do_wait_build(self, trial=0):
407 pid = self._build_pid
408 ppid = self._build_ppid
415 status = rspawn.remote_status(
417 host = self.node.hostname,
419 user = self.node.slicename,
421 ident_key = self.node.ident_path,
422 server_key = self.node.server_key,
423 hostip = self.node.hostip
426 if status is rspawn.FINISHED:
427 self._build_pid = self._build_ppid = None
429 elif status is not rspawn.RUNNING:
430 self._logger.warn("Busted waiting for %s to finish building at %s %s", self, self.node.hostname,
431 "(build slave)" if self._master is not None else "(build master)")
433 time.sleep(delay*(5.5+random.random()))
435 self._build_pid = self._build_ppid = None
439 self._logger.info("Waiting for %s to finish building at %s %s", self, self.node.hostname,
440 "(build slave)" if self._master is not None else "(build master)")
443 time.sleep(delay*(0.5+random.random()))
444 delay = min(30,delay*1.2)
450 (out, err), proc = self._popen_ssh_command(
451 "cat %(token_path)s" % {
452 'token_path' : os.path.join(self.home_path, 'build.token'),
456 if not proc.wait() and out:
457 slave_token = out.strip()
464 if slave_token != self._master_token:
465 # Get buildlog for the error message
467 (buildlog, err), proc = self._popen_ssh_command(
468 "cat %(buildlog)s" % {
469 'buildlog' : os.path.join(self.home_path, 'buildlog'),
470 'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
477 if self.check_bad_host(buildlog, err):
478 self.node.blacklist()
479 elif self._master and trial < 3 and 'BAD TOKEN' in buildlog or 'BAD TOKEN' in err:
480 # bad sync with master, may try again
481 # but first wait for master
482 self._master.async_setup_wait()
483 self._launch_build(trial+1)
484 return self._do_wait_build(trial+1)
486 return self._do_wait_build(trial+1)
489 self._master_prk = None
490 self._master_puk = None
492 raise RuntimeError, "Failed to set up application %s: "\
493 "build failed, got wrong token from pid %s/%s "\
494 "(expected %r, got %r), see buildlog at %s:\n%s" % (
495 self.home_path, pid, ppid, self._master_token, slave_token, self.node.hostname, buildlog)
498 self._master_prk = None
499 self._master_puk = None
501 self._logger.info("Built %s at %s", self, self.node.hostname)
503 def _do_kill_build(self):
504 pid = self._build_pid
505 ppid = self._build_ppid
508 self._logger.info("Killing build of %s", self)
511 host = self.node.hostname,
513 user = self.node.slicename,
515 ident_key = self.node.ident_path,
516 hostip = self.node.hostip
520 def _do_build_master(self):
521 if not self.sources and not self.build and not self.buildDepends:
525 sources = self.sources.split(' ')
527 http_sources = list()
528 for source in list(sources):
529 if source.startswith("http") or source.startswith("https"):
530 http_sources.append(source)
531 sources.remove(source)
533 # Download http sources
535 for source in http_sources:
536 path = os.path.join(self.home_path, source.split("/")[-1])
537 command = "wget -o %s %s" % (path, source)
538 self._popen_ssh(command)
539 except RuntimeError, e:
540 raise RuntimeError, "Failed wget source file %r: %s %s" \
541 % (sources, e.args[0], e.args[1],)
543 # Copy all other sources
547 "%s@%s:%s" % (self.node.slicename, self.node.hostname,
548 os.path.join(self.home_path,'.'),)
550 except RuntimeError, e:
551 raise RuntimeError, "Failed upload source file %r: %s %s" \
552 % (sources, e.args[0], e.args[1],)
554 buildscript = cStringIO.StringIO()
556 buildscript.write("(\n")
558 if self.buildDepends:
559 # Install build dependencies
561 "sudo -S yum -y install %(packages)s\n" % {
562 'packages' : self.buildDepends
570 "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
571 'command' : self._replace_paths(self.build),
572 'home' : server.shell_escape(self.home_path),
577 buildscript.write("tar czf build.tar.gz build\n")
580 buildscript.write("echo %(master_token)s > build.token ) ; echo %(master_token)s > build.token.retcode" % {
581 'master_token' : server.shell_escape(self._master_token)
588 def _do_install(self):
590 self._logger.info("Installing %s at %s", self, self.node.hostname)
592 # Install application
594 self._popen_ssh_command(
595 "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/{install,build}log >&2 && false )" % \
597 'command' : self._replace_paths(self.install),
598 'home' : server.shell_escape(self.home_path),
601 except RuntimeError, e:
602 if self.check_bad_host(e.args[0], e.args[1]):
603 self.node.blacklist()
604 raise RuntimeError, "Failed install build sources on node %s: %s %s" % (
605 self.node.hostname, e.args[0], e.args[1],)
607 def set_master(self, master):
608 self._master = master
610 def install_keys(self, prk, puk, passphrase):
612 self._master_passphrase = passphrase
613 self._master_prk = prk
614 self._master_puk = puk
615 self._master_prk_name = os.path.basename(prk.name)
616 self._master_puk_name = os.path.basename(puk.name)
618 def _do_install_keys(self):
619 prk = self._master_prk
620 puk = self._master_puk
624 [ prk.name, puk.name ],
625 '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path )
627 except RuntimeError, e:
628 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
629 % (e.args[0], e.args[1],)
633 cStringIO.StringIO('%s,%s %s\n' % (
634 self._master.node.hostname, self._master.node.hostip,
635 self._master.node.server_key)),
636 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
637 os.path.join(self.home_path,"master_known_hosts") )
639 except RuntimeError, e:
640 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
641 % (e.args[0], e.args[1],)
645 # make sure there's no leftover build processes
646 self._do_kill_build()
649 self._master_prk = None
650 self._master_puk = None
653 def _popen_scp(self, src, dst, retry = 3):
656 (out,err),proc = server.popen_scp(
661 ident_key = self.node.ident_path,
662 server_key = self.node.server_key
665 if server.eintr_retry(proc.wait)():
666 raise RuntimeError, (out, err)
667 return (out, err), proc
676 def _popen_ssh_command(self, command, retry = 0, noerrors=False, timeout=None):
677 (out,err),proc = server.popen_ssh_command(
679 host = self.node.hostname,
681 user = self.node.slicename,
683 ident_key = self.node.ident_path,
684 server_key = self.node.server_key,
689 if server.eintr_retry(proc.wait)():
691 raise RuntimeError, (out, err)
692 return (out, err), proc
694 class Application(Dependency):
696 An application also has dependencies, but also a command to be ran and monitored.
698 It adds the output of that command as traces.
701 TRACES = ('stdout','stderr','buildlog', 'output')
703 def __init__(self, api=None):
704 super(Application,self).__init__(api)
715 # Those are filled when the app is started
716 # Having both pid and ppid makes it harder
717 # for pid rollover to induce tracking mistakes
718 self._started = False
722 # Do not add to the python path of nodes
723 self.add_to_path = False
726 return "%s<command:%s%s>" % (
727 self.__class__.__name__,
728 "sudo " if self.sudo else "",
733 self._logger.info("Starting %s", self)
735 # Create shell script with the command
736 # This way, complex commands and scripts can be ran seamlessly
738 command = cStringIO.StringIO()
739 command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
740 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
742 command.write('export PATH=$PATH:%s\n' % (
743 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
746 for envkey, envvals in self.node.env.iteritems():
747 for envval in envvals:
748 command.write('export %s=%s\n' % (envkey, envval))
749 command.write(self.command)
755 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
756 os.path.join(self.home_path, "app.sh"))
758 except RuntimeError, e:
759 raise RuntimeError, "Failed to set up application: %s %s" \
760 % (e.args[0], e.args[1],)
762 # Start process in a "daemonized" way, using nohup and heavy
763 # stdin/out redirection to avoid connection issues
764 (out,err),proc = rspawn.remote_spawn(
765 self._replace_paths("bash ./app.sh"),
768 home = self.home_path,
769 stdin = 'stdin' if self.stdin is not None else '/dev/null',
770 stdout = 'stdout' if self.stdout else '/dev/null',
771 stderr = 'stderr' if self.stderr else '/dev/null',
773 host = self.node.hostname,
775 user = self.node.slicename,
777 ident_key = self.node.ident_path,
778 server_key = self.node.server_key
782 if self.check_bad_host(out, err):
783 self.node.blacklist()
784 raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
789 # Assuming the application is running on PlanetLab,
790 # proper pidfiles should be present at the app's home path.
791 # So we mark this application as started, and check the pidfiles
797 # NOTE: wait a bit for the pidfile to be created
798 if self._started and not self._pid or not self._ppid:
799 pidtuple = rspawn.remote_check_pid(
800 os.path.join(self.home_path,'pid'),
801 host = self.node.hostname,
803 user = self.node.slicename,
805 ident_key = self.node.ident_path,
806 server_key = self.node.server_key
810 self._pid, self._ppid = pidtuple
814 if not self._started:
815 return AS.STATUS_NOT_STARTED
816 elif not self._pid or not self._ppid:
817 return AS.STATUS_NOT_STARTED
819 status = rspawn.remote_status(
820 self._pid, self._ppid,
821 host = self.node.hostname,
823 user = self.node.slicename,
825 ident_key = self.node.ident_path,
826 server_key = self.node.server_key
829 if status is rspawn.NOT_STARTED:
830 return AS.STATUS_NOT_STARTED
831 elif status is rspawn.RUNNING:
832 return AS.STATUS_RUNNING
833 elif status is rspawn.FINISHED:
834 return AS.STATUS_FINISHED
837 return AS.STATUS_NOT_STARTED
840 status = self.status()
841 if status == AS.STATUS_RUNNING:
842 # kill by ppid+pid - SIGTERM first, then try SIGKILL
844 self._pid, self._ppid,
845 host = self.node.hostname,
847 user = self.node.slicename,
849 ident_key = self.node.ident_path,
850 server_key = self.node.server_key,
853 self._logger.info("Killed %s", self)
856 class NepiDependency(Dependency):
858 This dependency adds nepi itself to the python path,
859 so that you may run testbeds within PL nodes.
862 # Class attribute holding a *weak* reference to the shared NEPI tar file
863 # so that they may share it. Don't operate on the file itself, it would
864 # be a mess, just use its path.
865 _shared_nepi_tar = None
867 def __init__(self, api = None):
868 super(NepiDependency, self).__init__(api)
872 self.depends = 'python python-ipaddr python-setuptools'
874 # our sources are in our ad-hoc tarball
875 self.sources = self.tarball.name
877 tarname = os.path.basename(self.tarball.name)
879 # it's already built - just move the tarball into place
880 self.build = "mv -f ${SOURCES}/%s ." % (tarname,)
882 # unpack it into sources, and we're done
883 self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,)
887 if self._tarball is None:
888 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
889 if shared_tar is not None:
890 self._tarball = shared_tar
892 # Build an ad-hoc tarball
897 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
899 proc = subprocess.Popen(
900 ["tar", "czf", shared_tar.name,
901 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
903 stdout = open("/dev/null","w"),
904 stdin = open("/dev/null","r"))
907 raise RuntimeError, "Failed to create nepi tarball"
909 self._tarball = self._shared_nepi_tar = shared_tar
913 class NS3Dependency(Dependency):
915 This dependency adds NS3 libraries to the library paths,
916 so that you may run the NS3 testbed within PL nodes.
918 You'll also need the NepiDependency.
921 def __init__(self, api = None):
922 super(NS3Dependency, self).__init__(api)
924 self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip bzr'
926 # We have to download the sources, untar, build...
927 #pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
928 pygccxml_source_url = "http://yans.pl.sophia.inria.fr/libs/pygccxml-1.0.0.zip"
929 ns3_source_url = "http://nepi.inria.fr/code/nepi-ns3.13/archive/tip.tar.gz"
930 passfd_source_url = "http://nepi.inria.fr/code/python-passfd/archive/tip.tar.gz"
932 pybindgen_version = "797"
937 " python -c 'import pygccxml, pybindgen, passfd' && "
938 " test -f lib/ns/_core.so && "
939 " test -f lib/ns/__init__.py && "
940 " test -f lib/ns/core.py && "
941 " test -f lib/libns3-core.so && "
942 " LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
944 # Not working, rebuild
945 # Archive SHA1 sums to check
946 "echo '7158877faff2254e6c094bf18e6b4283cac19137 pygccxml-1.0.0.zip' > archive_sums.txt && "
947 " ( " # check existing files
948 " sha1sum -c archive_sums.txt && "
949 " test -f passfd-src.tar.gz && "
950 " test -f ns3-src.tar.gz "
951 " ) || ( " # nope? re-download
952 " rm -rf pybindgen pygccxml-1.0.0.zip passfd-src.tar.gz ns3-src.tar.gz && "
953 " bzr checkout lp:pybindgen -r %(pybindgen_version)s && " # continue, to exploit the case when it has already been dl'ed
954 " wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
955 " wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
956 " wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
957 " sha1sum -c archive_sums.txt " # Check SHA1 sums when applicable
959 "unzip -n pygccxml-1.0.0.zip && "
960 "mkdir -p ns3-src && "
961 "mkdir -p passfd-src && "
962 "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
963 "tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
964 "rm -rf target && " # mv doesn't like unclean targets
965 "mkdir -p target && "
966 "cd pygccxml-1.0.0 && "
967 "rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
968 "python setup.py build && "
969 "python setup.py install --install-lib ${BUILD}/target && "
970 "python setup.py clean && "
971 "cd ../pybindgen && "
972 "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
973 "./waf configure --prefix=${BUILD}/target -d release && "
977 "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
978 "rm -rf ${BUILD}/target/lib && "
979 "cd ../passfd-src && "
980 "python setup.py build && "
981 "python setup.py install --install-lib ${BUILD}/target && "
982 "python setup.py clean && "
984 "./waf configure --prefix=${BUILD}/target --with-pybindgen=../pybindgen-src -d release --disable-examples --disable-tests && "
987 "rm -f ${BUILD}/target/lib/*.so && "
988 "cp -a ${BUILD}/ns3-src/build/libns3*.so ${BUILD}/target/lib && "
989 "cp -a ${BUILD}/ns3-src/build/bindings/python/ns ${BUILD}/target/lib &&"
993 pybindgen_version = server.shell_escape(pybindgen_version),
994 pygccxml_source_url = server.shell_escape(pygccxml_source_url),
995 ns3_source_url = server.shell_escape(ns3_source_url),
996 passfd_source_url = server.shell_escape(passfd_source_url),
999 # Just move ${BUILD}/target
1003 " python -c 'import pygccxml, pybindgen, passfd' && "
1004 " test -f lib/ns/_core.so && "
1005 " test -f lib/ns/__init__.py && "
1006 " test -f lib/ns/core.py && "
1007 " test -f lib/libns3-core.so && "
1008 " LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
1010 # Not working, reinstall
1011 "test -d ${BUILD}/target && "
1012 "[[ \"x\" != \"x$(find ${BUILD}/target -mindepth 1 -print -quit)\" ]] &&"
1013 "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
1014 "mv -f ${BUILD}/target/* ${SOURCES}"
1018 # Set extra environment paths
1019 self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
1020 self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib"
1024 if self._tarball is None:
1025 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
1026 if shared_tar is not None:
1027 self._tarball = shared_tar
1029 # Build an ad-hoc tarball
1034 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
1036 proc = subprocess.Popen(
1037 ["tar", "czf", shared_tar.name,
1038 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
1040 stdout = open("/dev/null","w"),
1041 stdin = open("/dev/null","r"))
1044 raise RuntimeError, "Failed to create nepi tarball"
1046 self._tarball = self._shared_nepi_tar = shared_tar
1048 return self._tarball
1050 class YumDependency(Dependency):
1052 This dependency is an internal helper class used to
1053 efficiently distribute yum-downloaded rpms.
1055 It temporarily sets the yum cache as persistent in the
1056 build master, and installs all the required packages.
1058 The rpm packages left in the yum cache are gathered and
1059 distributed by the underlying Dependency in an efficient
1060 manner. Build slaves will then install those rpms back in
1061 the cache before issuing the install command.
1063 When packages have been installed already, nothing but an
1064 empty tar is distributed.
1067 # Class attribute holding a *weak* reference to the shared NEPI tar file
1068 # so that they may share it. Don't operate on the file itself, it would
1069 # be a mess, just use its path.
1070 _shared_nepi_tar = None
1072 def _build_get(self):
1073 # canonical representation of dependencies
1074 depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
1076 # download rpms and pack into a tar archive
1078 "sudo -S nice yum -y makecache && "
1079 "sudo -S sed -i -r 's/keepcache *= *0/keepcache=1/' /etc/yum.conf && "
1081 "sudo -S nice yum -y install %s ; "
1082 "rm -f ${BUILD}/packages.tar ; "
1083 "tar -C /var/cache/yum -rf ${BUILD}/packages.tar $(cd /var/cache/yum ; find -iname '*.rpm')"
1084 " ) || /bin/true ) && "
1085 "sudo -S sed -i -r 's/keepcache *= *1/keepcache=0/' /etc/yum.conf && "
1086 "( sudo -S nice yum -y clean packages || /bin/true ) "
1088 def _build_set(self, value):
1091 build = property(_build_get, _build_set)
1093 def _install_get(self):
1094 # canonical representation of dependencies
1095 depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
1097 # unpack cached rpms into yum cache, install, and cleanup
1099 "sudo -S tar -k --keep-newer-files -C /var/cache/yum -xf packages.tar && "
1100 "sudo -S nice yum -y install %s && "
1101 "( sudo -S nice yum -y clean packages || /bin/true ) "
1103 def _install_set(self, value):
1106 install = property(_install_get, _install_set)
1108 def check_bad_host(self, out, err):
1109 badre = re.compile(r'(?:'
1110 r'The GPG keys listed for the ".*" repository are already installed but they are not correct for this package'
1111 r'|Error: Cannot retrieve repository metadata (repomd.xml) for repository: .*[.] Please verify its path and try again'
1112 r'|Error: disk I/O error'
1113 r'|MASTER NODE UNREACHABLE'
1116 return badre.search(out) or badre.search(err) or self.node.check_bad_host(out,err)
1119 class CCNxDaemon(Application):
1121 An application also has dependencies, but also a command to be ran and monitored.
1123 It adds the output of that command as traces.
1126 def __init__(self, api=None):
1127 super(CCNxDaemon,self).__init__(api)
1130 self.ccnLocalPort = None
1131 self.ccnRoutes = None
1132 self.ccnxVersion = "ccnx-0.6.0"
1134 #self.ccnx_0_5_1_sources = "http://www.ccnx.org/releases/ccnx-0.5.1.tar.gz"
1135 self.ccnx_0_5_1_sources = "http://yans.pl.sophia.inria.fr/libs/ccnx-0.5.1.tar.gz"
1136 #self.ccnx_0_6_0_sources = "http://www.ccnx.org/releases/ccnx-0.6.0.tar.gz"
1137 self.ccnx_0_6_0_sources = "http://yans.pl.sophia.inria.fr/libs/ccnx-0.6.0.tar.gz"
1138 #self.buildDepends = 'make gcc development-tools openssl-devel expat-devel libpcap-devel libxml2-devel'
1139 self.buildDepends = 'make gcc openssl-devel expat-devel libpcap-devel libxml2-devel'
1141 self.ccnx_0_5_1_build = (
1144 " test -d ccnx-0.5.1-src/build/bin "
1146 # Not working, rebuild
1148 " mkdir -p ccnx-0.5.1-src && "
1149 " wget -q -c -O ccnx-0.5.1-src.tar.gz %(ccnx_source_url)s &&"
1150 " tar xf ccnx-0.5.1-src.tar.gz --strip-components=1 -C ccnx-0.5.1-src "
1152 "cd ccnx-0.5.1-src && "
1153 "mkdir -p build/include &&"
1154 "mkdir -p build/lib &&"
1155 "mkdir -p build/bin &&"
1157 "INSTALL_BASE=$I ./configure &&"
1158 "make && make install"
1160 ccnx_source_url = server.shell_escape(self.ccnx_0_5_1_sources),
1163 self.ccnx_0_5_1_install = (
1165 " test -d ${BUILD}/ccnx-0.5.1-src/build/bin && "
1166 " cp -r ${BUILD}/ccnx-0.5.1-src/build/bin ${SOURCES}"
1170 self.ccnx_0_6_0_build = (
1173 " test -d ccnx-0.6.0-src/build/bin "
1175 # Not working, rebuild
1177 " mkdir -p ccnx-0.6.0-src && "
1178 " wget -q -c -O ccnx-0.6.0-src.tar.gz %(ccnx_source_url)s &&"
1179 " tar xf ccnx-0.6.0-src.tar.gz --strip-components=1 -C ccnx-0.6.0-src "
1181 "cd ccnx-0.6.0-src && "
1182 "./configure && make"
1184 ccnx_source_url = server.shell_escape(self.ccnx_0_6_0_sources),
1187 self.ccnx_0_6_0_install = (
1189 " test -d ${BUILD}/ccnx-0.6.0-src/bin && "
1190 " cp -r ${BUILD}/ccnx-0.6.0-src/bin ${SOURCES}"
1194 self.env['PATH'] = "$PATH:${SOURCES}/bin"
1197 # setting ccn sources
1199 if self.ccnxVersion == 'ccnx-0.6.0':
1200 self.build = self.ccnx_0_6_0_build
1201 elif self.ccnxVersion == 'ccnx-0.5.1':
1202 self.build = self.ccnx_0_5_1_build
1204 if not self.install:
1205 if self.ccnxVersion == 'ccnx-0.6.0':
1206 self.install = self.ccnx_0_6_0_install
1207 elif self.ccnxVersion == 'ccnx-0.5.1':
1208 self.install = self.ccnx_0_5_1_install
1210 super(CCNxDaemon, self).setup()
1214 if self.ccnLocalPort:
1215 self.command = "export CCN_LOCAL_PORT=%s ; " % self.ccnLocalPort
1216 self.command += " ccndstart "
1218 # configure ccn routes
1220 routes = self.ccnRoutes.split("|")
1222 if self.ccnLocalPort:
1223 routes = map(lambda route: "%s %s" %(route,
1224 self.ccnLocalPort) if _ccnre.match(route) else route,
1227 routes = map(lambda route: "ccndc add ccnx:/ %s" % route,
1230 routescmd = " ; ".join(routes)
1231 self.command += " ; "
1232 self.command += routescmd
1234 # Start will be invoked in prestart step
1235 super(CCNxDaemon, self).start()
1238 self._logger.info("Killing %s", self)
1240 command = "${SOURCES}/bin/ccndstop"
1242 if self.ccnLocalPort:
1243 self.command = "export CCN_LOCAL_PORT=%s; %s" % (self.ccnLocalPort, command)
1245 cmd = self._replace_paths(command)
1246 command = cStringIO.StringIO()
1253 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
1254 os.path.join(self.home_path, "kill.sh"))
1256 except RuntimeError, e:
1257 raise RuntimeError, "Failed to kill ccndxdaemon: %s %s" \
1258 % (e.args[0], e.args[1],)
1261 script = "bash ./kill.sh"
1262 (out,err),proc = rspawn.remote_spawn(
1264 pidfile = 'kill-pid',
1265 home = self.home_path,
1266 stdin = '/dev/null',
1268 stderr = rspawn.STDOUT,
1270 host = self.node.hostname,
1272 user = self.node.slicename,
1274 ident_key = self.node.ident_path,
1275 server_key = self.node.server_key,
1276 hostip = self.node.hostip,
1280 raise RuntimeError, "Failed to kill cnnxdaemon: %s %s" % (out,err,)
1282 super(CCNxDaemon, self).kill()