src/nepi/testbeds/planetlab/application.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from constants import TESTBED_ID
   5 import plcapi
   6 import operator
   7 import os
   8 import os.path
   9 import sys
  10 import nepi.util.server as server
  11 import cStringIO
  12 import subprocess
  13 import rspawn
  14 import random
  15 import time
  16 import socket
  17 import threading
  18
  19 from nepi.util.constants import STATUS_NOT_STARTED, STATUS_RUNNING, \
  20         STATUS_FINISHED
  21
  22 class Dependency(object):
  23     """
  24     A Dependency is in every respect like an application.
  25
  26     It depends on some packages, it may require building binaries, it must deploy
  27     them...
  28
  29     But it has no command. Dependencies aren't ever started, or stopped, and have
  30     no status.
  31     """
  32
  33     TRACES = ('buildlog')
  34
  35     def __init__(self, api=None):
  36         if not api:
  37             api = plcapi.PLCAPI()
  38         self._api = api
  39
  40         # Attributes
  41         self.command = None
  42         self.sudo = False
  43
  44         self.build = None
  45         self.install = None
  46         self.depends = None
  47         self.buildDepends = None
  48         self.sources = None
  49         self.rpmFusion = False
  50         self.env = {}
  51
  52         self.stdin = None
  53         self.stdout = None
  54         self.stderr = None
  55         self.buildlog = None
  56
  57         self.add_to_path = True
  58
  59         # Those are filled when the app is configured
  60         self.home_path = None
  61
  62         # Those are filled when an actual node is connected
  63         self.node = None
  64
  65         # Those are filled when the app is started
  66         #   Having both pid and ppid makes it harder
  67         #   for pid rollover to induce tracking mistakes
  68         self._started = False
  69         self._setup = False
  70         self._setuper = None
  71         self._pid = None
  72         self._ppid = None
  73
  74         # Spanning tree deployment
  75         self._master = None
  76         self._master_passphrase = None
  77         self._master_prk = None
  78         self._master_puk = None
  79         self._master_token = ''.join(map(chr,[rng.randint(0,255)
  80                                       for rng in (random.SystemRandom(),)
  81                                       for i in xrange(8)] )).encode("hex")
  82         self._build_pid = None
  83         self._build_ppid = None
  84
  85
  86     def __str__(self):
  87         return "%s<%s>" % (
  88             self.__class__.__name__,
  89             ' '.join(filter(bool,(self.depends, self.sources)))
  90         )
  91
  92     def validate(self):
  93         if self.home_path is None:
  94             raise AssertionError, "Misconfigured application: missing home path"
  95         if self.node.ident_path is None or not os.access(self.node.ident_path, os.R_OK):
  96             raise AssertionError, "Misconfigured application: missing slice SSH key"
  97         if self.node is None:
  98             raise AssertionError, "Misconfigured application: unconnected node"
  99         if self.node.hostname is None:
 100             raise AssertionError, "Misconfigured application: misconfigured node"
 101         if self.node.slicename is None:
 102             raise AssertionError, "Misconfigured application: unspecified slice"
 103
 104     def remote_trace_path(self, whichtrace):
 105         if whichtrace in self.TRACES:
 106             tracefile = os.path.join(self.home_path, whichtrace)
 107         else:
 108             tracefile = None
 109
 110         return tracefile
 111
 112     def sync_trace(self, local_dir, whichtrace):
 113         tracefile = self.remote_trace_path(whichtrace)
 114         if not tracefile:
 115             return None
 116
 117         local_path = os.path.join(local_dir, tracefile)
 118
 119         # create parent local folders
 120         proc = subprocess.Popen(
 121             ["mkdir", "-p", os.path.dirname(local_path)],
 122             stdout = open("/dev/null","w"),
 123             stdin = open("/dev/null","r"))
 124
 125         if proc.wait():
 126             raise RuntimeError, "Failed to synchronize trace"
 127
 128         # sync files
 129         try:
 130             self._popen_scp(
 131                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 132                     tracefile),
 133                 local_path
 134                 )
 135         except RuntimeError, e:
 136             raise RuntimeError, "Failed to synchronize trace: %s %s" \
 137                     % (e.args[0], e.args[1],)
 138
 139         return local_path
 140
 141     def setup(self):
 142         self._make_home()
 143         self._launch_build()
 144         self._finish_build()
 145         self._setup = True
 146
 147     def async_setup(self):
 148         if not self._setuper:
 149             def setuper():
 150                 try:
 151                     self.setup()
 152                 except:
 153                     self._setuper._exc.append(sys.exc_info())
 154             self._setuper = threading.Thread(
 155                 target = setuper)
 156             self._setuper._exc = []
 157             self._setuper.start()
 158
 159     def async_setup_wait(self):
 160         if not self._setup:
 161             if self._setuper:
 162                 self._setuper.join()
 163                 if not self._setup:
 164                     if self._setuper._exc:
 165                         exctyp,exval,exctrace = self._setuper._exc[0]
 166                         raise exctyp,exval,exctrace
 167                     else:
 168                         raise RuntimeError, "Failed to setup application"
 169             else:
 170                 self.setup()
 171
 172     def _make_home(self):
 173         # Make sure all the paths are created where
 174         # they have to be created for deployment
 175         # sync files
 176         try:
 177             self._popen_ssh_command(
 178                 "mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" \
 179                     % { 'home' : server.shell_escape(self.home_path) }
 180                 )
 181         except RuntimeError, e:
 182             raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, e.args[0], e.args[1],)
 183
 184         if self.stdin:
 185             # Write program input
 186             try:
 187                 self._popen_scp(
 188                     cStringIO.StringIO(self.stdin),
 189                     '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 190                         os.path.join(self.home_path, 'stdin') ),
 191                     )
 192             except RuntimeError, e:
 193                 raise RuntimeError, "Failed to set up application %s: %s %s" \
 194                         % (self.home_path, e.args[0], e.args[1],)
 195
 196     def _replace_paths(self, command):
 197         """
 198         Replace all special path tags with shell-escaped actual paths.
 199         """
 200         # need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
 201         root = '' if self.home_path.startswith('/') else "${HOME}/"
 202         return ( command
 203             .replace("${SOURCES}", root+server.shell_escape(self.home_path))
 204             .replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
 205
 206     def _launch_build(self):
 207         if self._master is not None:
 208             self._do_install_keys()
 209             buildscript = self._do_build_slave()
 210         else:
 211             buildscript = self._do_build_master()
 212
 213         if buildscript is not None:
 214             # upload build script
 215             try:
 216                 self._popen_scp(
 217                     buildscript,
 218                     '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 219                         os.path.join(self.home_path, 'nepi-build.sh') )
 220                     )
 221             except RuntimeError, e:
 222                 raise RuntimeError, "Failed to set up application %s: %s %s" \
 223                         % (self.home_path, e.args[0], e.args[1],)
 224
 225             # launch build
 226             self._do_launch_build()
 227
 228     def _finish_build(self):
 229         self._do_wait_build()
 230         self._do_install()
 231
 232     def _do_build_slave(self):
 233         if not self.sources and not self.build:
 234             return None
 235
 236         # Create build script
 237         files = set()
 238
 239         if self.sources:
 240             sources = self.sources.split(' ')
 241             files.update(
 242                 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
 243                     os.path.join(self._master.home_path, os.path.basename(source)),)
 244                 for source in sources
 245             )
 246
 247         if self.build:
 248             files.add(
 249                 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
 250                     os.path.join(self._master.home_path, 'build.tar.gz'),)
 251             )
 252
 253         launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
 254                         " && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
 255                         " && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" %  \
 256         {
 257             'prk' : server.shell_escape(self._master_prk_name),
 258             'puk' : server.shell_escape(self._master_puk_name),
 259         }
 260
 261         kill_agent = "kill $SSH_AGENT_PID"
 262
 263         waitmaster = "{ . ./.ssh-agent.sh ; while [[ $(ssh -q -o UserKnownHostsFile=%(hostkey)s %(master)s cat %(token_path)s) != %(token)s ]] ; do sleep 5 ; done ; }" % {
 264             'hostkey' : 'master_known_hosts',
 265             'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostname),
 266             'token_path' : os.path.join(self._master.home_path, 'build.token'),
 267             'token' : server.shell_escape(self._master._master_token),
 268         }
 269
 270         syncfiles = "scp -p -o UserKnownHostsFile=%(hostkey)s %(files)s ." % {
 271             'hostkey' : 'master_known_hosts',
 272             'files' : ' '.join(files),
 273         }
 274         if self.build:
 275             syncfiles += " && tar xzf build.tar.gz"
 276         syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
 277         syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
 278
 279         cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
 280             'prk' : server.shell_escape(self._master_prk_name),
 281             'puk' : server.shell_escape(self._master_puk_name),
 282         }
 283
 284         slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s )" % {
 285             'waitmaster' : waitmaster,
 286             'syncfiles' : syncfiles,
 287             'cleanup' : cleanup,
 288             'kill_agent' : kill_agent,
 289             'launch_agent' : launch_agent,
 290             'home' : server.shell_escape(self.home_path),
 291         }
 292
 293         return cStringIO.StringIO(slavescript)
 294
 295     def _do_launch_build(self):
 296         script = "bash ./nepi-build.sh"
 297         if self._master_passphrase:
 298             script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
 299                 server.shell_escape(self._master_passphrase),
 300                 script
 301             )
 302         (out,err),proc = rspawn.remote_spawn(
 303             script,
 304             pidfile = 'build-pid',
 305             home = self.home_path,
 306             stdin = '/dev/null',
 307             stdout = 'buildlog',
 308             stderr = rspawn.STDOUT,
 309
 310             host = self.node.hostname,
 311             port = None,
 312             user = self.node.slicename,
 313             agent = None,
 314             ident_key = self.node.ident_path,
 315             server_key = self.node.server_key
 316             )
 317
 318         if proc.wait():
 319             raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
 320
 321
 322         pid = ppid = None
 323         delay = 1.0
 324         for i in xrange(5):
 325             pidtuple = rspawn.remote_check_pid(
 326                 os.path.join(self.home_path,'build-pid'),
 327                 host = self.node.hostname,
 328                 port = None,
 329                 user = self.node.slicename,
 330                 agent = None,
 331                 ident_key = self.node.ident_path,
 332                 server_key = self.node.server_key
 333                 )
 334
 335             if pidtuple:
 336                 pid, ppid = pidtuple
 337                 self._build_pid, self._build_ppid = pidtuple
 338                 break
 339             else:
 340                 time.sleep(delay)
 341                 delay = min(30,delay*1.2)
 342         else:
 343             raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
 344
 345     def _do_wait_build(self):
 346         pid = self._build_pid
 347         ppid = self._build_ppid
 348
 349         if pid and ppid:
 350             delay = 1.0
 351             while True:
 352                 status = rspawn.remote_status(
 353                     pid, ppid,
 354                     host = self.node.hostname,
 355                     port = None,
 356                     user = self.node.slicename,
 357                     agent = None,
 358                     ident_key = self.node.ident_path,
 359                     server_key = self.node.server_key
 360                     )
 361
 362                 if status is not rspawn.RUNNING:
 363                     self._build_pid = self._build_ppid = None
 364                     break
 365                 else:
 366                     time.sleep(delay*(0.5+random.random()))
 367                     delay = min(30,delay*1.2)
 368
 369             # check build token
 370             (out, err), proc = self._popen_ssh_command(
 371                 "cat %(token_path)s" % {
 372                     'token_path' : os.path.join(self.home_path, 'build.token'),
 373                 },
 374                 noerrors = True)
 375             slave_token = ""
 376             if not proc.wait() and out:
 377                 slave_token = out.strip()
 378
 379             if slave_token != self._master_token:
 380                 # Get buildlog for the error message
 381
 382                 (buildlog, err), proc = self._popen_ssh_command(
 383                     "cat %(buildlog)s" % {
 384                         'buildlog' : os.path.join(self.home_path, 'buildlog'),
 385                         'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
 386                     },
 387                     noerrors = True)
 388
 389                 proc.wait()
 390
 391                 raise RuntimeError, "Failed to set up application %s: "\
 392                         "build failed, got wrong token from pid %s/%s "\
 393                         "(expected %r, got %r), see buildlog: %s" % (
 394                     self.home_path, pid, ppid, self._master_token, slave_token, buildlog)
 395
 396     def _do_kill_build(self):
 397         pid = self._build_pid
 398         ppid = self._build_ppid
 399
 400         if pid and ppid:
 401             rspawn.remote_kill(
 402                 pid, ppid,
 403                 host = self.node.hostname,
 404                 port = None,
 405                 user = self.node.slicename,
 406                 agent = None,
 407                 ident_key = self.node.ident_path
 408                 )
 409
 410
 411     def _do_build_master(self):
 412         if not self.sources and not self.build and not self.buildDepends:
 413             return None
 414
 415         if self.sources:
 416             sources = self.sources.split(' ')
 417
 418             # Copy all sources
 419             try:
 420                 self._popen_scp(
 421                     sources,
 422                     "%s@%s:%s" % (self.node.slicename, self.node.hostname,
 423                         os.path.join(self.home_path,'.'),)
 424                     )
 425             except RuntimeError, e:
 426                 raise RuntimeError, "Failed upload source file %r: %s %s" \
 427                         % (sources, e.args[0], e.args[1],)
 428
 429         buildscript = cStringIO.StringIO()
 430
 431         if self.buildDepends:
 432             # Install build dependencies
 433             buildscript.write(
 434                 "sudo -S yum -y install %(packages)s\n" % {
 435                     'packages' : self.buildDepends
 436                 }
 437             )
 438
 439
 440         if self.build:
 441             # Build sources
 442             buildscript.write(
 443                 "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
 444                     'command' : self._replace_paths(self.build),
 445                     'home' : server.shell_escape(self.home_path),
 446                 }
 447             )
 448
 449             # Make archive
 450             buildscript.write("tar czf build.tar.gz build\n")
 451
 452         # Write token
 453         buildscript.write("echo %(master_token)s > build.token" % {
 454             'master_token' : server.shell_escape(self._master_token)
 455         })
 456
 457         buildscript.seek(0)
 458
 459         return buildscript
 460
 461     def _do_install(self):
 462         if self.install:
 463             # Install application
 464             try:
 465                 self._popen_ssh_command(
 466                     "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/installlog >&2 && false )" % \
 467                         {
 468                         'command' : self._replace_paths(self.install),
 469                         'home' : server.shell_escape(self.home_path),
 470                         },
 471                     )
 472             except RuntimeError, e:
 473                 raise RuntimeError, "Failed install build sources: %s %s" % (e.args[0], e.args[1],)
 474
 475     def set_master(self, master):
 476         self._master = master
 477
 478     def install_keys(self, prk, puk, passphrase):
 479         # Install keys
 480         self._master_passphrase = passphrase
 481         self._master_prk = prk
 482         self._master_puk = puk
 483         self._master_prk_name = os.path.basename(prk.name)
 484         self._master_puk_name = os.path.basename(puk.name)
 485
 486     def _do_install_keys(self):
 487         prk = self._master_prk
 488         puk = self._master_puk
 489
 490         try:
 491             self._popen_scp(
 492                 [ prk.name, puk.name ],
 493                 '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path )
 494                 )
 495         except RuntimeError, e:
 496             raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
 497                     % (e.args[0], e.args[1],)
 498
 499         try:
 500             self._popen_scp(
 501                 cStringIO.StringIO('%s,%s %s\n' % (
 502                     self._master.node.hostname, socket.gethostbyname(self._master.node.hostname),
 503                     self._master.node.server_key)),
 504                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 505                     os.path.join(self.home_path,"master_known_hosts") )
 506                 )
 507         except RuntimeError, e:
 508             raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
 509                     % (e.args[0], e.args[1],)
 510
 511         # No longer need'em
 512         self._master_prk = None
 513         self._master_puk = None
 514
 515     def cleanup(self):
 516         # make sure there's no leftover build processes
 517         self._do_kill_build()
 518
 519     @server.eintr_retry
 520     def _popen_scp(self, src, dst, retry = True):
 521         (out,err),proc = server.popen_scp(
 522             src,
 523             dst,
 524             port = None,
 525             agent = None,
 526             ident_key = self.node.ident_path,
 527             server_key = self.node.server_key
 528             )
 529
 530         if server.eintr_retry(proc.wait)():
 531             raise RuntimeError, (out, err)
 532         return (out, err), proc
 533
 534
 535     @server.eintr_retry
 536     def _popen_ssh_command(self, command, retry = True, noerrors=False):
 537         (out,err),proc = server.popen_ssh_command(
 538             command,
 539             host = self.node.hostname,
 540             port = None,
 541             user = self.node.slicename,
 542             agent = None,
 543             ident_key = self.node.ident_path,
 544             server_key = self.node.server_key
 545             )
 546
 547         if server.eintr_retry(proc.wait)():
 548             if not noerrors:
 549                 raise RuntimeError, (out, err)
 550         return (out, err), proc
 551
 552 class Application(Dependency):
 553     """
 554     An application also has dependencies, but also a command to be ran and monitored.
 555
 556     It adds the output of that command as traces.
 557     """
 558
 559     TRACES = ('stdout','stderr','buildlog')
 560
 561     def __init__(self, api=None):
 562         super(Application,self).__init__(api)
 563
 564         # Attributes
 565         self.command = None
 566         self.sudo = False
 567
 568         self.stdin = None
 569         self.stdout = None
 570         self.stderr = None
 571
 572         # Those are filled when the app is started
 573         #   Having both pid and ppid makes it harder
 574         #   for pid rollover to induce tracking mistakes
 575         self._started = False
 576         self._pid = None
 577         self._ppid = None
 578
 579         # Do not add to the python path of nodes
 580         self.add_to_path = False
 581
 582     def __str__(self):
 583         return "%s<command:%s%s>" % (
 584             self.__class__.__name__,
 585             "sudo " if self.sudo else "",
 586             self.command,
 587         )
 588
 589     def start(self):
 590         # Create shell script with the command
 591         # This way, complex commands and scripts can be ran seamlessly
 592         # sync files
 593         command = cStringIO.StringIO()
 594         command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
 595             ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
 596         ))
 597         command.write('export PATH=$PATH:%s\n' % (
 598             ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
 599         ))
 600         if self.node.env:
 601             for envkey, envvals in self.node.env.iteritems():
 602                 for envval in envvals:
 603                     command.write('export %s=%s\n' % (envkey, envval))
 604         command.write(self.command)
 605         command.seek(0)
 606
 607         try:
 608             self._popen_scp(
 609                 command,
 610                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 611                     os.path.join(self.home_path, "app.sh"))
 612                 )
 613         except RuntimeError, e:
 614             raise RuntimeError, "Failed to set up application: %s %s" \
 615                     % (e.args[0], e.args[1],)
 616
 617         # Start process in a "daemonized" way, using nohup and heavy
 618         # stdin/out redirection to avoid connection issues
 619         (out,err),proc = rspawn.remote_spawn(
 620             self._replace_paths("bash ./app.sh"),
 621
 622             pidfile = './pid',
 623             home = self.home_path,
 624             stdin = 'stdin' if self.stdin is not None else '/dev/null',
 625             stdout = 'stdout' if self.stdout else '/dev/null',
 626             stderr = 'stderr' if self.stderr else '/dev/null',
 627             sudo = self.sudo,
 628
 629             host = self.node.hostname,
 630             port = None,
 631             user = self.node.slicename,
 632             agent = None,
 633             ident_key = self.node.ident_path,
 634             server_key = self.node.server_key
 635             )
 636
 637         if proc.wait():
 638             raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
 639
 640         self._started = True
 641
 642     def checkpid(self):
 643         # Get PID/PPID
 644         # NOTE: wait a bit for the pidfile to be created
 645         if self._started and not self._pid or not self._ppid:
 646             pidtuple = rspawn.remote_check_pid(
 647                 os.path.join(self.home_path,'pid'),
 648                 host = self.node.hostname,
 649                 port = None,
 650                 user = self.node.slicename,
 651                 agent = None,
 652                 ident_key = self.node.ident_path,
 653                 server_key = self.node.server_key
 654                 )
 655
 656             if pidtuple:
 657                 self._pid, self._ppid = pidtuple
 658
 659     def status(self):
 660         self.checkpid()
 661         if not self._started:
 662             return STATUS_NOT_STARTED
 663         elif not self._pid or not self._ppid:
 664             return STATUS_NOT_STARTED
 665         else:
 666             status = rspawn.remote_status(
 667                 self._pid, self._ppid,
 668                 host = self.node.hostname,
 669                 port = None,
 670                 user = self.node.slicename,
 671                 agent = None,
 672                 ident_key = self.node.ident_path,
 673                 server_key = self.node.server_key
 674                 )
 675
 676             if status is rspawn.NOT_STARTED:
 677                 return STATUS_NOT_STARTED
 678             elif status is rspawn.RUNNING:
 679                 return STATUS_RUNNING
 680             elif status is rspawn.FINISHED:
 681                 return STATUS_FINISHED
 682             else:
 683                 # WTF?
 684                 return STATUS_NOT_STARTED
 685
 686     def kill(self):
 687         status = self.status()
 688         if status == STATUS_RUNNING:
 689             # kill by ppid+pid - SIGTERM first, then try SIGKILL
 690             rspawn.remote_kill(
 691                 self._pid, self._ppid,
 692                 host = self.node.hostname,
 693                 port = None,
 694                 user = self.node.slicename,
 695                 agent = None,
 696                 ident_key = self.node.ident_path,
 697                 server_key = self.node.server_key
 698                 )
 699
 700
 701 class NepiDependency(Dependency):
 702     """
 703     This dependency adds nepi itself to the python path,
 704     so that you may run testbeds within PL nodes.
 705     """
 706
 707     # Class attribute holding a *weak* reference to the shared NEPI tar file
 708     # so that they may share it. Don't operate on the file itself, it would
 709     # be a mess, just use its path.
 710     _shared_nepi_tar = None
 711
 712     def __init__(self, api = None):
 713         super(NepiDependency, self).__init__(api)
 714
 715         self._tarball = None
 716
 717         self.depends = 'python python-ipaddr python-setuptools'
 718
 719         # our sources are in our ad-hoc tarball
 720         self.sources = self.tarball.name
 721
 722         tarname = os.path.basename(self.tarball.name)
 723
 724         # it's already built - just move the tarball into place
 725         self.build = "mv -f ${SOURCES}/%s ." % (tarname,)
 726
 727         # unpack it into sources, and we're done
 728         self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,)
 729
 730     @property
 731     def tarball(self):
 732         if self._tarball is None:
 733             shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
 734             if shared_tar is not None:
 735                 self._tarball = shared_tar
 736             else:
 737                 # Build an ad-hoc tarball
 738                 # Prebuilt
 739                 import nepi
 740                 import tempfile
 741
 742                 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
 743
 744                 proc = subprocess.Popen(
 745                     ["tar", "czf", shared_tar.name,
 746                         '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
 747                         'nepi'],
 748                     stdout = open("/dev/null","w"),
 749                     stdin = open("/dev/null","r"))
 750
 751                 if proc.wait():
 752                     raise RuntimeError, "Failed to create nepi tarball"
 753
 754                 self._tarball = self._shared_nepi_tar = shared_tar
 755
 756         return self._tarball
 757
 758 class NS3Dependency(Dependency):
 759     """
 760     This dependency adds NS3 libraries to the library paths,
 761     so that you may run the NS3 testbed within PL nodes.
 762
 763     You'll also need the NepiDependency.
 764     """
 765
 766     def __init__(self, api = None):
 767         super(NS3Dependency, self).__init__(api)
 768
 769         self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip'
 770
 771         # We have to download the sources, untar, build...
 772         pybindgen_source_url = "http://pybindgen.googlecode.com/files/pybindgen-0.15.0.zip"
 773         pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
 774         ns3_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/nepi-ns-3.9/archive/tip.tar.gz"
 775         passfd_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/python-passfd/archive/tip.tar.gz"
 776         self.build =(
 777             " ( "
 778             "  cd .. && "
 779             "  python -c 'import pygccxml, pybindgen, passfd' && "
 780             "  test -f lib/_ns3.so && "
 781             "  test -f lib/libns3.so "
 782             " ) || ( "
 783                 # Not working, rebuild
 784                      "wget -q -c -O pybindgen-src.zip %(pybindgen_source_url)s && " # continue, to exploit the case when it has already been dl'ed
 785                      "wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
 786                      "wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
 787                      "wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
 788                      "unzip -n pybindgen-src.zip && " # Do not overwrite files, to exploit the case when it has already been built
 789                      "unzip -n pygccxml-1.0.0.zip && "
 790                      "mkdir -p ns3-src && "
 791                      "mkdir -p passfd-src && "
 792                      "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
 793                      "tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
 794                      "rm -rf target && "    # mv doesn't like unclean targets
 795                      "mkdir -p target && "
 796                      "cd pygccxml-1.0.0 && "
 797                      "rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
 798                      "python setup.py build && "
 799                      "python setup.py install --install-lib ${BUILD}/target && "
 800                      "python setup.py clean && "
 801                      "cd ../pybindgen-0.15.0 && "
 802                      "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
 803                      "./waf configure --prefix=${BUILD}/target -d release && "
 804                      "./waf && "
 805                      "./waf install && "
 806                      "./waf clean && "
 807                      "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
 808                      "rm -rf ${BUILD}/target/lib && "
 809                      "cd ../passfd-src && "
 810                      "python setup.py build && "
 811                      "python setup.py install --install-lib ${BUILD}/target && "
 812                      "python setup.py clean && "
 813                      "cd ../ns3-src && "
 814                      "./waf configure --prefix=${BUILD}/target -d release --disable-examples --high-precision-as-double && "
 815                      "./waf &&"
 816                      "./waf install && "
 817                      "./waf clean"
 818              " )"
 819                      % dict(
 820                         pybindgen_source_url = server.shell_escape(pybindgen_source_url),
 821                         pygccxml_source_url = server.shell_escape(pygccxml_source_url),
 822                         ns3_source_url = server.shell_escape(ns3_source_url),
 823                         passfd_source_url = server.shell_escape(passfd_source_url),
 824                      ))
 825
 826         # Just move ${BUILD}/target
 827         self.install = (
 828             " ( "
 829             "  cd .. && "
 830             "  python -c 'import pygccxml, pybindgen, passfd' && "
 831             "  test -f lib/_ns3.so && "
 832             "  test -f lib/libns3.so "
 833             " ) || ( "
 834                 # Not working, reinstall
 835                     "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
 836                     "mv -f ${BUILD}/target/* ${SOURCES}"
 837             " )"
 838         )
 839
 840         # Set extra environment paths
 841         self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
 842         self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib/libns3.so"
 843
 844     @property
 845     def tarball(self):
 846         if self._tarball is None:
 847             shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
 848             if shared_tar is not None:
 849                 self._tarball = shared_tar
 850             else:
 851                 # Build an ad-hoc tarball
 852                 # Prebuilt
 853                 import nepi
 854                 import tempfile
 855
 856                 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
 857
 858                 proc = subprocess.Popen(
 859                     ["tar", "czf", shared_tar.name,
 860                         '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
 861                         'nepi'],
 862                     stdout = open("/dev/null","w"),
 863                     stdin = open("/dev/null","r"))
 864
 865                 if proc.wait():
 866                     raise RuntimeError, "Failed to create nepi tarball"
 867
 868                 self._tarball = self._shared_nepi_tar = shared_tar
 869
 870         return self._tarball
 871
 872