1 # -*- coding: utf-8 -*-
3 from constants import TESTBED_ID
9 import nepi.util.server as server
20 from nepi.util.constants import ApplicationStatus as AS
22 _ccnre = re.compile("\s*(udp|tcp)\s+(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\s*$")
24 class Dependency(object):
26 A Dependency is in every respect like an application.
28 It depends on some packages, it may require building binaries, it must deploy
31 But it has no command. Dependencies aren't ever started, or stopped, and have
37 def __init__(self, api=None):
49 self.buildDepends = None
51 self.rpmFusion = False
59 self.add_to_path = True
61 # Those are filled when the app is configured
64 # Those are filled when an actual node is connected
67 # Those are filled when the app is started
68 # Having both pid and ppid makes it harder
69 # for pid rollover to induce tracking mistakes
76 # Spanning tree deployment
78 self._master_passphrase = None
79 self._master_prk = None
80 self._master_puk = None
81 self._master_token = os.urandom(8).encode("hex")
82 self._build_pid = None
83 self._build_ppid = None
86 self._logger = logging.getLogger('nepi.testbeds.planetlab')
91 self.__class__.__name__,
92 ' '.join(filter(bool,(self.depends, self.sources)))
99 if self.home_path is None:
100 raise AssertionError, "Misconfigured application: missing home path"
101 if self.node.ident_path is None or not os.access(self.node.ident_path, os.R_OK):
102 raise AssertionError, "Misconfigured application: missing slice SSH key"
103 if self.node is None:
104 raise AssertionError, "Misconfigured application: unconnected node"
105 if self.node.hostname is None:
106 raise AssertionError, "Misconfigured application: misconfigured node"
107 if self.node.slicename is None:
108 raise AssertionError, "Misconfigured application: unspecified slice"
110 def check_bad_host(self, out, err):
112 Called whenever an operation fails, it's given the output to be checked for
113 telltale signs of unhealthy hosts.
117 def remote_trace_path(self, whichtrace):
118 if whichtrace in self.TRACES:
119 tracefile = os.path.join(self.home_path, whichtrace)
125 def remote_trace_name(self, whichtrace):
126 if whichtrace in self.TRACES:
130 def sync_trace(self, local_dir, whichtrace):
131 tracefile = self.remote_trace_path(whichtrace)
135 local_path = os.path.join(local_dir, tracefile)
137 # create parent local folders
138 proc = subprocess.Popen(
139 ["mkdir", "-p", os.path.dirname(local_path)],
140 stdout = open("/dev/null","w"),
141 stdin = open("/dev/null","r"))
144 raise RuntimeError, "Failed to synchronize trace"
149 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
153 except RuntimeError, e:
154 raise RuntimeError, "Failed to synchronize trace: %s %s" \
155 % (e.args[0], e.args[1],)
160 # We assume a correct deployment, so recovery only
161 # means we mark this dependency as deployed
165 self._logger.info("Setting up %s", self)
171 def async_setup(self):
172 if not self._setuper:
177 self._setuper._exc.append(sys.exc_info())
178 self._setuper = threading.Thread(
180 self._setuper._exc = []
181 self._setuper.start()
183 def async_setup_wait(self):
185 self._logger.info("Waiting for %s to be setup", self)
189 if self._setuper._exc:
190 exctyp,exval,exctrace = self._setuper._exc[0]
191 raise exctyp,exval,exctrace
193 raise RuntimeError, "Failed to setup application"
195 self._logger.info("Setup ready: %s at %s", self, self.node.hostname)
199 def _make_home(self):
200 # Make sure all the paths are created where
201 # they have to be created for deployment
204 self._popen_ssh_command(
205 "mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" \
206 % { 'home' : server.shell_escape(self.home_path) },
210 except RuntimeError, e:
211 raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, e.args[0], e.args[1],)
215 if not os.path.isfile(stdin):
216 stdin = cStringIO.StringIO(self.stdin)
218 # Write program input
220 self._popen_scp(stdin,
221 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
222 os.path.join(self.home_path, 'stdin') ),
224 except RuntimeError, e:
225 raise RuntimeError, "Failed to set up application %s: %s %s" \
226 % (self.home_path, e.args[0], e.args[1],)
228 def _replace_paths(self, command):
230 Replace all special path tags with shell-escaped actual paths.
232 # need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
233 root = '' if self.home_path.startswith('/') else "${HOME}/"
235 .replace("${SOURCES}", root+server.shell_escape(self.home_path))
236 .replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
238 def _launch_build(self, trial=0):
239 if self._master is not None:
240 if not trial or self._master_prk is not None:
241 self._do_install_keys()
242 buildscript = self._do_build_slave()
244 buildscript = self._do_build_master()
246 if buildscript is not None:
247 self._logger.info("Building %s at %s", self, self.node.hostname)
249 # upload build script
253 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
254 os.path.join(self.home_path, 'nepi-build.sh') )
256 except RuntimeError, e:
257 raise RuntimeError, "Failed to set up application %s: %s %s" \
258 % (self.home_path, e.args[0], e.args[1],)
261 self._do_launch_build()
263 def _finish_build(self):
264 self._do_wait_build()
267 def _do_build_slave(self):
268 if not self.sources and not self.build:
271 # Create build script
275 sources = self.sources.split(' ')
277 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostip,
278 os.path.join(self._master.home_path, os.path.basename(source)),)
279 for source in sources
284 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostip,
285 os.path.join(self._master.home_path, 'build.tar.gz'),)
288 sshopts = "-o ConnectTimeout=30 -o ConnectionAttempts=3 -o ServerAliveInterval=30 -o TCPKeepAlive=yes"
290 launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
291 " && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
292 " && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" % \
294 'prk' : server.shell_escape(self._master_prk_name),
295 'puk' : server.shell_escape(self._master_puk_name),
298 kill_agent = "kill $SSH_AGENT_PID"
302 "echo 'Checking master reachability' ; "
303 "if ping -c 3 %(master_host)s && (. ./.ssh-agent.sh > /dev/null ; ssh -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s echo MASTER SAYS HI ) ; then "
304 "echo 'Master node reachable' ; "
306 "echo 'MASTER NODE UNREACHABLE' && "
309 ". ./.ssh-agent.sh ; "
310 "while [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s.retcode || /bin/true) != %(token)s ]] ; do sleep 5 ; done ; "
311 "if [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s || /bin/true) != %(token)s ]] ; then echo BAD TOKEN ; exit 1 ; fi ; "
314 'hostkey' : 'master_known_hosts',
315 'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostip),
316 'master_host' : self._master.node.hostip,
317 'token_path' : os.path.join(self._master.home_path, 'build.token'),
318 'token' : server.shell_escape(self._master._master_token),
322 syncfiles = ". ./.ssh-agent.sh && scp -p -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(files)s ." % {
323 'hostkey' : 'master_known_hosts',
324 'files' : ' '.join(files),
328 syncfiles += " && tar xzf build.tar.gz"
329 syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
330 syncfiles += " && ( echo %s > build.token.retcode )" % (server.shell_escape(self._master_token),)
331 syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
333 cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
334 'prk' : server.shell_escape(self._master_prk_name),
335 'puk' : server.shell_escape(self._master_puk_name),
338 slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s ) ; echo %(token)s > build.token.retcode" % {
339 'waitmaster' : waitmaster,
340 'syncfiles' : syncfiles,
342 'kill_agent' : kill_agent,
343 'launch_agent' : launch_agent,
344 'home' : server.shell_escape(self.home_path),
345 'token' : server.shell_escape(self._master_token),
348 return cStringIO.StringIO(slavescript)
350 def _do_launch_build(self):
351 script = "bash ./nepi-build.sh"
352 if self._master_passphrase:
353 script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
354 server.shell_escape(self._master_passphrase),
357 (out,err),proc = rspawn.remote_spawn(
359 pidfile = 'build-pid',
360 home = self.home_path,
363 stderr = rspawn.STDOUT,
365 host = self.node.hostname,
367 user = self.node.slicename,
369 ident_key = self.node.ident_path,
370 server_key = self.node.server_key,
371 hostip = self.node.hostip,
375 if self.check_bad_host(out, err):
376 self.node.blacklist()
377 raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
383 pidtuple = rspawn.remote_check_pid(
384 os.path.join(self.home_path,'build-pid'),
385 host = self.node.hostip,
387 user = self.node.slicename,
389 ident_key = self.node.ident_path,
390 server_key = self.node.server_key,
391 hostip = self.node.hostip
396 self._build_pid, self._build_ppid = pidtuple
400 delay = min(30,delay*1.2)
402 raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
404 self._logger.info("Deploying %s at %s", self, self.node.hostname)
406 def _do_wait_build(self, trial=0):
407 pid = self._build_pid
408 ppid = self._build_ppid
415 status = rspawn.remote_status(
417 host = self.node.hostname,
419 user = self.node.slicename,
421 ident_key = self.node.ident_path,
422 server_key = self.node.server_key,
423 hostip = self.node.hostip
426 if status is rspawn.FINISHED:
427 self._build_pid = self._build_ppid = None
429 elif status is not rspawn.RUNNING:
430 self._logger.warn("Busted waiting for %s to finish building at %s %s", self, self.node.hostname,
431 "(build slave)" if self._master is not None else "(build master)")
433 time.sleep(delay*(5.5+random.random()))
435 self._build_pid = self._build_ppid = None
439 self._logger.info("Waiting for %s to finish building at %s %s", self, self.node.hostname,
440 "(build slave)" if self._master is not None else "(build master)")
443 time.sleep(delay*(0.5+random.random()))
444 delay = min(30,delay*1.2)
450 (out, err), proc = self._popen_ssh_command(
451 "cat %(token_path)s" % {
452 'token_path' : os.path.join(self.home_path, 'build.token'),
456 if not proc.wait() and out:
457 slave_token = out.strip()
464 if slave_token != self._master_token:
465 # Get buildlog for the error message
467 (buildlog, err), proc = self._popen_ssh_command(
468 "cat %(buildlog)s" % {
469 'buildlog' : os.path.join(self.home_path, 'buildlog'),
470 'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
477 if self.check_bad_host(buildlog, err):
478 self.node.blacklist()
479 elif self._master and trial < 3 and 'BAD TOKEN' in buildlog or 'BAD TOKEN' in err:
480 # bad sync with master, may try again
481 # but first wait for master
482 self._master.async_setup_wait()
483 self._launch_build(trial+1)
484 return self._do_wait_build(trial+1)
486 return self._do_wait_build(trial+1)
489 self._master_prk = None
490 self._master_puk = None
492 raise RuntimeError, "Failed to set up application %s: "\
493 "build failed, got wrong token from pid %s/%s "\
494 "(expected %r, got %r), see buildlog at %s:\n%s" % (
495 self.home_path, pid, ppid, self._master_token, slave_token, self.node.hostname, buildlog)
498 self._master_prk = None
499 self._master_puk = None
501 self._logger.info("Built %s at %s", self, self.node.hostname)
503 def _do_kill_build(self):
504 pid = self._build_pid
505 ppid = self._build_ppid
508 self._logger.info("Killing build of %s", self)
511 host = self.node.hostname,
513 user = self.node.slicename,
515 ident_key = self.node.ident_path,
516 hostip = self.node.hostip
520 def _do_build_master(self):
521 if not self.sources and not self.build and not self.buildDepends:
525 sources = self.sources.split(' ')
527 http_sources = list()
528 for source in list(sources):
529 if source.startswith("http") or source.startswith("https"):
530 http_sources.append(source)
531 sources.remove(source)
533 # Download http sources
535 for source in http_sources:
536 path = os.path.join(self.home_path, source.split("/")[-1])
537 command = "wget -o %s %s" % (path, source)
538 self._popen_ssh(command)
539 except RuntimeError, e:
540 raise RuntimeError, "Failed wget source file %r: %s %s" \
541 % (sources, e.args[0], e.args[1],)
543 # Copy all other sources
547 "%s@%s:%s" % (self.node.slicename, self.node.hostname,
548 os.path.join(self.home_path,'.'),)
550 except RuntimeError, e:
551 raise RuntimeError, "Failed upload source file %r: %s %s" \
552 % (sources, e.args[0], e.args[1],)
554 buildscript = cStringIO.StringIO()
556 buildscript.write("(\n")
558 if self.buildDepends:
559 # Install build dependencies
561 "sudo -S yum -y install %(packages)s\n" % {
562 'packages' : self.buildDepends
570 "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
571 'command' : self._replace_paths(self.build),
572 'home' : server.shell_escape(self.home_path),
577 buildscript.write("tar czf build.tar.gz build\n")
580 buildscript.write("echo %(master_token)s > build.token ) ; echo %(master_token)s > build.token.retcode" % {
581 'master_token' : server.shell_escape(self._master_token)
588 def _do_install(self):
590 self._logger.info("Installing %s at %s", self, self.node.hostname)
592 # Install application
594 command = "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/{install,build}log >&2 && false )" % \
596 'command' : self._replace_paths(self.install),
597 'home' : server.shell_escape(self.home_path),
599 self._popen_ssh_command(command)
600 except RuntimeError, e:
601 if self.check_bad_host(e.args[0], e.args[1]):
602 self.node.blacklist()
603 raise RuntimeError, "Failed install build sources on node %s: %s %s. command %s" % (
604 self.node.hostname, e.args[0], e.args[1], command)
606 def set_master(self, master):
607 self._master = master
609 def install_keys(self, prk, puk, passphrase):
611 self._master_passphrase = passphrase
612 self._master_prk = prk
613 self._master_puk = puk
614 self._master_prk_name = os.path.basename(prk.name)
615 self._master_puk_name = os.path.basename(puk.name)
617 def _do_install_keys(self):
618 prk = self._master_prk
619 puk = self._master_puk
623 [ prk.name, puk.name ],
624 '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path )
626 except RuntimeError, e:
627 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
628 % (e.args[0], e.args[1],)
632 cStringIO.StringIO('%s,%s %s\n' % (
633 self._master.node.hostname, self._master.node.hostip,
634 self._master.node.server_key)),
635 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
636 os.path.join(self.home_path,"master_known_hosts") )
638 except RuntimeError, e:
639 raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
640 % (e.args[0], e.args[1],)
644 # make sure there's no leftover build processes
645 self._do_kill_build()
648 self._master_prk = None
649 self._master_puk = None
652 def _popen_scp(self, src, dst, retry = 3):
655 (out,err),proc = server.popen_scp(
660 ident_key = self.node.ident_path,
661 server_key = self.node.server_key
664 if server.eintr_retry(proc.wait)():
665 raise RuntimeError, (out, err)
666 return (out, err), proc
675 def _popen_ssh_command(self, command, retry = 0, noerrors=False, timeout=None):
676 (out,err),proc = server.popen_ssh_command(
678 host = self.node.hostname,
680 user = self.node.slicename,
682 ident_key = self.node.ident_path,
683 server_key = self.node.server_key,
688 if server.eintr_retry(proc.wait)():
690 raise RuntimeError, (out, err)
691 return (out, err), proc
693 class Application(Dependency):
695 An application also has dependencies, but also a command to be ran and monitored.
697 It adds the output of that command as traces.
700 TRACES = ('stdout','stderr','buildlog', 'output')
702 def __init__(self, api=None):
703 super(Application,self).__init__(api)
714 # Those are filled when the app is started
715 # Having both pid and ppid makes it harder
716 # for pid rollover to induce tracking mistakes
717 self._started = False
721 # Do not add to the python path of nodes
722 self.add_to_path = False
725 return "%s<command:%s%s>" % (
726 self.__class__.__name__,
727 "sudo " if self.sudo else "",
732 self._logger.info("Starting %s", self)
734 # Create shell script with the command
735 # This way, complex commands and scripts can be ran seamlessly
737 command = cStringIO.StringIO()
738 command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
739 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
741 command.write('export PATH=$PATH:%s\n' % (
742 ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
745 for envkey, envvals in self.node.env.iteritems():
746 for envval in envvals:
747 command.write('export %s=%s\n' % (envkey, envval))
748 command.write(self.command)
754 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
755 os.path.join(self.home_path, "app.sh"))
757 except RuntimeError, e:
758 raise RuntimeError, "Failed to set up application: %s %s" \
759 % (e.args[0], e.args[1],)
761 # Start process in a "daemonized" way, using nohup and heavy
762 # stdin/out redirection to avoid connection issues
763 (out,err),proc = rspawn.remote_spawn(
764 self._replace_paths("bash ./app.sh"),
767 home = self.home_path,
768 stdin = 'stdin' if self.stdin is not None else '/dev/null',
769 stdout = 'stdout' if self.stdout else '/dev/null',
770 stderr = 'stderr' if self.stderr else '/dev/null',
772 host = self.node.hostname,
774 user = self.node.slicename,
776 ident_key = self.node.ident_path,
777 server_key = self.node.server_key
781 if self.check_bad_host(out, err):
782 self.node.blacklist()
783 raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
788 # Assuming the application is running on PlanetLab,
789 # proper pidfiles should be present at the app's home path.
790 # So we mark this application as started, and check the pidfiles
796 # NOTE: wait a bit for the pidfile to be created
797 if self._started and not self._pid or not self._ppid:
798 pidtuple = rspawn.remote_check_pid(
799 os.path.join(self.home_path,'pid'),
800 host = self.node.hostname,
802 user = self.node.slicename,
804 ident_key = self.node.ident_path,
805 server_key = self.node.server_key
809 self._pid, self._ppid = pidtuple
813 if not self._started:
814 return AS.STATUS_NOT_STARTED
815 elif not self._pid or not self._ppid:
816 return AS.STATUS_NOT_STARTED
818 status = rspawn.remote_status(
819 self._pid, self._ppid,
820 host = self.node.hostname,
822 user = self.node.slicename,
824 ident_key = self.node.ident_path,
825 server_key = self.node.server_key
828 if status is rspawn.NOT_STARTED:
829 return AS.STATUS_NOT_STARTED
830 elif status is rspawn.RUNNING:
831 return AS.STATUS_RUNNING
832 elif status is rspawn.FINISHED:
833 return AS.STATUS_FINISHED
836 return AS.STATUS_NOT_STARTED
839 status = self.status()
840 if status == AS.STATUS_RUNNING:
841 # kill by ppid+pid - SIGTERM first, then try SIGKILL
843 self._pid, self._ppid,
844 host = self.node.hostname,
846 user = self.node.slicename,
848 ident_key = self.node.ident_path,
849 server_key = self.node.server_key,
852 self._logger.info("Killed %s", self)
855 class NepiDependency(Dependency):
857 This dependency adds nepi itself to the python path,
858 so that you may run testbeds within PL nodes.
861 # Class attribute holding a *weak* reference to the shared NEPI tar file
862 # so that they may share it. Don't operate on the file itself, it would
863 # be a mess, just use its path.
864 _shared_nepi_tar = None
866 def __init__(self, api = None):
867 super(NepiDependency, self).__init__(api)
871 self.depends = 'python python-ipaddr python-setuptools'
873 # our sources are in our ad-hoc tarball
874 self.sources = self.tarball.name
876 tarname = os.path.basename(self.tarball.name)
878 # it's already built - just move the tarball into place
879 self.build = "mv -f ${SOURCES}/%s ." % (tarname,)
881 # unpack it into sources, and we're done
882 self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,)
886 if self._tarball is None:
887 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
888 if shared_tar is not None:
889 self._tarball = shared_tar
891 # Build an ad-hoc tarball
896 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
898 proc = subprocess.Popen(
899 ["tar", "czf", shared_tar.name,
900 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
902 stdout = open("/dev/null","w"),
903 stdin = open("/dev/null","r"))
906 raise RuntimeError, "Failed to create nepi tarball"
908 self._tarball = self._shared_nepi_tar = shared_tar
912 class NS3Dependency(Dependency):
914 This dependency adds NS3 libraries to the library paths,
915 so that you may run the NS3 testbed within PL nodes.
917 You'll also need the NepiDependency.
920 def __init__(self, api = None):
921 super(NS3Dependency, self).__init__(api)
923 self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip bzr'
925 # We have to download the sources, untar, build...
926 #pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
927 pygccxml_source_url = "http://yans.pl.sophia.inria.fr/libs/pygccxml-1.0.0.zip"
928 ns3_source_url = "http://nepi.inria.fr/code/nepi-ns3.13/archive/tip.tar.gz"
929 passfd_source_url = "http://nepi.inria.fr/code/python-passfd/archive/tip.tar.gz"
931 pybindgen_version = "797"
936 " python -c 'import pygccxml, pybindgen, passfd' && "
937 " test -f lib/ns/_core.so && "
938 " test -f lib/ns/__init__.py && "
939 " test -f lib/ns/core.py && "
940 " test -f lib/libns3-core.so && "
941 " LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
943 # Not working, rebuild
944 # Archive SHA1 sums to check
945 "echo '7158877faff2254e6c094bf18e6b4283cac19137 pygccxml-1.0.0.zip' > archive_sums.txt && "
946 " ( " # check existing files
947 " sha1sum -c archive_sums.txt && "
948 " test -f passfd-src.tar.gz && "
949 " test -f ns3-src.tar.gz "
950 " ) || ( " # nope? re-download
951 " rm -rf pybindgen pygccxml-1.0.0.zip passfd-src.tar.gz ns3-src.tar.gz && "
952 " bzr checkout lp:pybindgen -r %(pybindgen_version)s && " # continue, to exploit the case when it has already been dl'ed
953 " wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
954 " wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
955 " wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
956 " sha1sum -c archive_sums.txt " # Check SHA1 sums when applicable
958 "unzip -n pygccxml-1.0.0.zip && "
959 "mkdir -p ns3-src && "
960 "mkdir -p passfd-src && "
961 "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
962 "tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
963 "rm -rf target && " # mv doesn't like unclean targets
964 "mkdir -p target && "
965 "cd pygccxml-1.0.0 && "
966 "rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
967 "python setup.py build && "
968 "python setup.py install --install-lib ${BUILD}/target && "
969 "python setup.py clean && "
970 "cd ../pybindgen && "
971 "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
972 "./waf configure --prefix=${BUILD}/target -d release && "
976 "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
977 "rm -rf ${BUILD}/target/lib && "
978 "cd ../passfd-src && "
979 "python setup.py build && "
980 "python setup.py install --install-lib ${BUILD}/target && "
981 "python setup.py clean && "
983 "./waf configure --prefix=${BUILD}/target --with-pybindgen=../pybindgen-src -d release --disable-examples --disable-tests && "
986 "rm -f ${BUILD}/target/lib/*.so && "
987 "cp -a ${BUILD}/ns3-src/build/libns3*.so ${BUILD}/target/lib && "
988 "cp -a ${BUILD}/ns3-src/build/bindings/python/ns ${BUILD}/target/lib &&"
992 pybindgen_version = server.shell_escape(pybindgen_version),
993 pygccxml_source_url = server.shell_escape(pygccxml_source_url),
994 ns3_source_url = server.shell_escape(ns3_source_url),
995 passfd_source_url = server.shell_escape(passfd_source_url),
998 # Just move ${BUILD}/target
1002 " python -c 'import pygccxml, pybindgen, passfd' && "
1003 " test -f lib/ns/_core.so && "
1004 " test -f lib/ns/__init__.py && "
1005 " test -f lib/ns/core.py && "
1006 " test -f lib/libns3-core.so && "
1007 " LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
1009 # Not working, reinstall
1010 "test -d ${BUILD}/target && "
1011 "[[ \"x\" != \"x$(find ${BUILD}/target -mindepth 1 -print -quit)\" ]] &&"
1012 "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
1013 "mv -f ${BUILD}/target/* ${SOURCES}"
1017 # Set extra environment paths
1018 self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
1019 self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib"
1023 if self._tarball is None:
1024 shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
1025 if shared_tar is not None:
1026 self._tarball = shared_tar
1028 # Build an ad-hoc tarball
1033 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
1035 proc = subprocess.Popen(
1036 ["tar", "czf", shared_tar.name,
1037 '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
1039 stdout = open("/dev/null","w"),
1040 stdin = open("/dev/null","r"))
1043 raise RuntimeError, "Failed to create nepi tarball"
1045 self._tarball = self._shared_nepi_tar = shared_tar
1047 return self._tarball
1049 class YumDependency(Dependency):
1051 This dependency is an internal helper class used to
1052 efficiently distribute yum-downloaded rpms.
1054 It temporarily sets the yum cache as persistent in the
1055 build master, and installs all the required packages.
1057 The rpm packages left in the yum cache are gathered and
1058 distributed by the underlying Dependency in an efficient
1059 manner. Build slaves will then install those rpms back in
1060 the cache before issuing the install command.
1062 When packages have been installed already, nothing but an
1063 empty tar is distributed.
1066 # Class attribute holding a *weak* reference to the shared NEPI tar file
1067 # so that they may share it. Don't operate on the file itself, it would
1068 # be a mess, just use its path.
1069 _shared_nepi_tar = None
1071 def _build_get(self):
1072 # canonical representation of dependencies
1073 depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
1075 # download rpms and pack into a tar archive
1077 "sudo -S nice yum -y makecache && "
1078 "sudo -S sed -i -r 's/keepcache *= *0/keepcache=1/' /etc/yum.conf && "
1080 "sudo -S nice yum -y install %s ; "
1081 "rm -f ${BUILD}/packages.tar ; "
1082 "tar -C /var/cache/yum -rf ${BUILD}/packages.tar $(cd /var/cache/yum ; find -iname '*.rpm')"
1083 " ) || /bin/true ) && "
1084 "sudo -S sed -i -r 's/keepcache *= *1/keepcache=0/' /etc/yum.conf && "
1085 "( sudo -S nice yum -y clean packages || /bin/true ) "
1087 def _build_set(self, value):
1090 build = property(_build_get, _build_set)
1092 def _install_get(self):
1093 # canonical representation of dependencies
1094 depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
1096 # unpack cached rpms into yum cache, install, and cleanup
1098 "sudo -S tar -k --keep-newer-files -C /var/cache/yum -xf packages.tar && "
1099 "sudo -S nice yum -y install %s && "
1100 "( sudo -S nice yum -y clean packages || /bin/true ) "
1102 def _install_set(self, value):
1105 install = property(_install_get, _install_set)
1107 def check_bad_host(self, out, err):
1108 badre = re.compile(r'(?:'
1109 r'The GPG keys listed for the ".*" repository are already installed but they are not correct for this package'
1110 r'|Error: Cannot retrieve repository metadata (repomd.xml) for repository: .*[.] Please verify its path and try again'
1111 r'|Error: disk I/O error'
1112 r'|MASTER NODE UNREACHABLE'
1115 return badre.search(out) or badre.search(err) or self.node.check_bad_host(out,err)
1118 class CCNxDaemon(Application):
1120 An application also has dependencies, but also a command to be ran and monitored.
1122 It adds the output of that command as traces.
1125 def __init__(self, api=None):
1126 super(CCNxDaemon,self).__init__(api)
1129 self.ccnLocalPort = None
1130 self.ccnRoutes = None
1131 self.ccnxVersion = "ccnx-0.6.0"
1133 #self.ccnx_0_5_1_sources = "http://www.ccnx.org/releases/ccnx-0.5.1.tar.gz"
1134 self.ccnx_0_5_1_sources = "http://yans.pl.sophia.inria.fr/libs/ccnx-0.5.1.tar.gz"
1135 #self.ccnx_0_6_0_sources = "http://www.ccnx.org/releases/ccnx-0.6.0.tar.gz"
1136 self.ccnx_0_6_0_sources = "http://yans.pl.sophia.inria.fr/libs/ccnx-0.6.0.tar.gz"
1137 #self.buildDepends = 'make gcc development-tools openssl-devel expat-devel libpcap-devel libxml2-devel'
1138 self.buildDepends = 'make gcc openssl-devel expat-devel libpcap-devel libxml2-devel'
1140 self.ccnx_0_5_1_build = (
1143 " test -d ccnx-0.5.1-src/build/bin "
1145 # Not working, rebuild
1147 " mkdir -p ccnx-0.5.1-src && "
1148 " wget -q -c -O ccnx-0.5.1-src.tar.gz %(ccnx_source_url)s &&"
1149 " tar xf ccnx-0.5.1-src.tar.gz --strip-components=1 -C ccnx-0.5.1-src "
1151 "cd ccnx-0.5.1-src && "
1152 "mkdir -p build/include &&"
1153 "mkdir -p build/lib &&"
1154 "mkdir -p build/bin &&"
1156 "INSTALL_BASE=$I ./configure &&"
1157 "make && make install"
1159 ccnx_source_url = server.shell_escape(self.ccnx_0_5_1_sources),
1162 self.ccnx_0_5_1_install = (
1164 " test -d ${BUILD}/ccnx-0.5.1-src/build/bin && "
1165 " cp -r ${BUILD}/ccnx-0.5.1-src/build/bin ${SOURCES}"
1169 self.ccnx_0_6_0_build = (
1172 " test -d ccnx-0.6.0-src/build/bin "
1174 # Not working, rebuild
1176 " mkdir -p ccnx-0.6.0-src && "
1177 " wget -q -c -O ccnx-0.6.0-src.tar.gz %(ccnx_source_url)s &&"
1178 " tar xf ccnx-0.6.0-src.tar.gz --strip-components=1 -C ccnx-0.6.0-src "
1180 "cd ccnx-0.6.0-src && "
1181 "./configure && make"
1183 ccnx_source_url = server.shell_escape(self.ccnx_0_6_0_sources),
1186 self.ccnx_0_6_0_install = (
1188 " test -d ${BUILD}/ccnx-0.6.0-src/bin && "
1189 " cp -r ${BUILD}/ccnx-0.6.0-src/bin ${SOURCES}"
1193 self.env['PATH'] = "$PATH:${SOURCES}/bin"
1196 # setting ccn sources
1198 if self.ccnxVersion == 'ccnx-0.6.0':
1199 self.build = self.ccnx_0_6_0_build
1200 elif self.ccnxVersion == 'ccnx-0.5.1':
1201 self.build = self.ccnx_0_5_1_build
1203 if not self.install:
1204 if self.ccnxVersion == 'ccnx-0.6.0':
1205 self.install = self.ccnx_0_6_0_install
1206 elif self.ccnxVersion == 'ccnx-0.5.1':
1207 self.install = self.ccnx_0_5_1_install
1209 super(CCNxDaemon, self).setup()
1213 if self.ccnLocalPort:
1214 self.command = "export CCN_LOCAL_PORT=%s ; " % self.ccnLocalPort
1215 self.command += " ccndstart "
1217 # configure ccn routes
1219 routes = self.ccnRoutes.split("|")
1221 if self.ccnLocalPort:
1222 routes = map(lambda route: "%s %s" %(route,
1223 self.ccnLocalPort) if _ccnre.match(route) else route,
1226 routes = map(lambda route: "ccndc add ccnx:/ %s" % route,
1229 routescmd = " ; ".join(routes)
1230 self.command += " ; "
1231 self.command += routescmd
1233 # Start will be invoked in prestart step
1234 super(CCNxDaemon, self).start()
1237 self._logger.info("Killing %s", self)
1239 command = "${SOURCES}/bin/ccndstop"
1241 if self.ccnLocalPort:
1242 self.command = "export CCN_LOCAL_PORT=%s; %s" % (self.ccnLocalPort, command)
1244 cmd = self._replace_paths(command)
1245 command = cStringIO.StringIO()
1252 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
1253 os.path.join(self.home_path, "kill.sh"))
1255 except RuntimeError, e:
1256 raise RuntimeError, "Failed to kill ccndxdaemon: %s %s" \
1257 % (e.args[0], e.args[1],)
1260 script = "bash ./kill.sh"
1261 (out,err),proc = rspawn.remote_spawn(
1263 pidfile = 'kill-pid',
1264 home = self.home_path,
1265 stdin = '/dev/null',
1267 stderr = rspawn.STDOUT,
1269 host = self.node.hostname,
1271 user = self.node.slicename,
1273 ident_key = self.node.ident_path,
1274 server_key = self.node.server_key,
1275 hostip = self.node.hostip,
1279 raise RuntimeError, "Failed to kill cnnxdaemon: %s %s" % (out,err,)
1281 super(CCNxDaemon, self).kill()