src/nepi/testbeds/planetlab/application.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from constants import TESTBED_ID
   5 import plcapi
   6 import operator
   7 import os
   8 import os.path
   9 import sys
  10 import nepi.util.server as server
  11 import cStringIO
  12 import subprocess
  13 import rspawn
  14 import random
  15 import time
  16 import socket
  17 import threading
  18
  19 from nepi.util.constants import STATUS_NOT_STARTED, STATUS_RUNNING, \
  20         STATUS_FINISHED
  21
  22 class Dependency(object):
  23     """
  24     A Dependency is in every respect like an application.
  25
  26     It depends on some packages, it may require building binaries, it must deploy
  27     them...
  28
  29     But it has no command. Dependencies aren't ever started, or stopped, and have
  30     no status.
  31     """
  32
  33     TRACES = ('buildlog')
  34
  35     def __init__(self, api=None):
  36         if not api:
  37             api = plcapi.PLCAPI()
  38         self._api = api
  39
  40         # Attributes
  41         self.command = None
  42         self.sudo = False
  43
  44         self.build = None
  45         self.install = None
  46         self.depends = None
  47         self.buildDepends = None
  48         self.sources = None
  49         self.env = {}
  50
  51         self.stdin = None
  52         self.stdout = None
  53         self.stderr = None
  54         self.buildlog = None
  55
  56         self.add_to_path = True
  57
  58         # Those are filled when the app is configured
  59         self.home_path = None
  60
  61         # Those are filled when an actual node is connected
  62         self.node = None
  63
  64         # Those are filled when the app is started
  65         #   Having both pid and ppid makes it harder
  66         #   for pid rollover to induce tracking mistakes
  67         self._started = False
  68         self._setup = False
  69         self._setuper = None
  70         self._pid = None
  71         self._ppid = None
  72
  73         # Spanning tree deployment
  74         self._master = None
  75         self._master_passphrase = None
  76         self._master_prk = None
  77         self._master_puk = None
  78         self._master_token = ''.join(map(chr,[rng.randint(0,255)
  79                                       for rng in (random.SystemRandom(),)
  80                                       for i in xrange(8)] )).encode("hex")
  81         self._build_pid = None
  82         self._build_ppid = None
  83
  84
  85     def __str__(self):
  86         return "%s<%s>" % (
  87             self.__class__.__name__,
  88             ' '.join(filter(bool,(self.depends, self.sources)))
  89         )
  90
  91     def validate(self):
  92         if self.home_path is None:
  93             raise AssertionError, "Misconfigured application: missing home path"
  94         if self.node.ident_path is None or not os.access(self.node.ident_path, os.R_OK):
  95             raise AssertionError, "Misconfigured application: missing slice SSH key"
  96         if self.node is None:
  97             raise AssertionError, "Misconfigured application: unconnected node"
  98         if self.node.hostname is None:
  99             raise AssertionError, "Misconfigured application: misconfigured node"
 100         if self.node.slicename is None:
 101             raise AssertionError, "Misconfigured application: unspecified slice"
 102
 103     def remote_trace_path(self, whichtrace):
 104         if whichtrace in self.TRACES:
 105             tracefile = os.path.join(self.home_path, whichtrace)
 106         else:
 107             tracefile = None
 108
 109         return tracefile
 110
 111     def sync_trace(self, local_dir, whichtrace):
 112         tracefile = self.remote_trace_path(whichtrace)
 113         if not tracefile:
 114             return None
 115
 116         local_path = os.path.join(local_dir, tracefile)
 117
 118         # create parent local folders
 119         proc = subprocess.Popen(
 120             ["mkdir", "-p", os.path.dirname(local_path)],
 121             stdout = open("/dev/null","w"),
 122             stdin = open("/dev/null","r"))
 123
 124         if proc.wait():
 125             raise RuntimeError, "Failed to synchronize trace"
 126
 127         # sync files
 128         (out,err),proc = server.popen_scp(
 129             '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 130                 tracefile),
 131             local_path,
 132             port = None,
 133             agent = None,
 134             ident_key = self.node.ident_path,
 135             server_key = self.node.server_key
 136             )
 137
 138         if proc.wait():
 139             raise RuntimeError, "Failed to synchronize trace: %s %s" % (out,err,)
 140
 141         return local_path
 142
 143
 144     def setup(self):
 145         self._make_home()
 146         self._launch_build()
 147         self._finish_build()
 148         self._setup = True
 149
 150     def async_setup(self):
 151         if not self._setuper:
 152             def setuper():
 153                 try:
 154                     self.setup()
 155                 except:
 156                     self._setuper._exc.append(sys.exc_info())
 157             self._setuper = threading.Thread(
 158                 target = setuper)
 159             self._setuper._exc = []
 160             self._setuper.start()
 161
 162     def async_setup_wait(self):
 163         if not self._setup:
 164             if self._setuper:
 165                 self._setuper.join()
 166                 if not self._setup:
 167                     if self._setuper._exc:
 168                         exctyp,exval,exctrace = self._setuper._exc[0]
 169                         raise exctyp,exval,exctrace
 170                     else:
 171                         raise RuntimeError, "Failed to setup application"
 172             else:
 173                 self.setup()
 174
 175     def _make_home(self):
 176         # Make sure all the paths are created where
 177         # they have to be created for deployment
 178         (out,err),proc = server.popen_ssh_command(
 179             "mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" % { 'home' : server.shell_escape(self.home_path) },
 180             host = self.node.hostname,
 181             port = None,
 182             user = self.node.slicename,
 183             agent = None,
 184             ident_key = self.node.ident_path,
 185             server_key = self.node.server_key
 186             )
 187
 188         if proc.wait():
 189             raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, out,err,)
 190
 191         if self.stdin:
 192             # Write program input
 193             (out,err),proc = server.popen_scp(
 194                 cStringIO.StringIO(self.stdin),
 195                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 196                     os.path.join(self.home_path, 'stdin') ),
 197                 port = None,
 198                 agent = None,
 199                 ident_key = self.node.ident_path,
 200                 server_key = self.node.server_key
 201                 )
 202
 203             if proc.wait():
 204                 raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, out,err,)
 205
 206     def _replace_paths(self, command):
 207         """
 208         Replace all special path tags with shell-escaped actual paths.
 209         """
 210         # need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
 211         root = '' if self.home_path.startswith('/') else "${HOME}/"
 212         return ( command
 213             .replace("${SOURCES}", root+server.shell_escape(self.home_path))
 214             .replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
 215
 216     def _launch_build(self):
 217         if self._master is not None:
 218             self._do_install_keys()
 219             buildscript = self._do_build_slave()
 220         else:
 221             buildscript = self._do_build_master()
 222
 223         if buildscript is not None:
 224             # upload build script
 225             (out,err),proc = server.popen_scp(
 226                 buildscript,
 227                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 228                     os.path.join(self.home_path, 'nepi-build.sh') ),
 229                 port = None,
 230                 agent = None,
 231                 ident_key = self.node.ident_path,
 232                 server_key = self.node.server_key
 233                 )
 234
 235             if proc.wait():
 236                 raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, out,err,)
 237
 238             # launch build
 239             self._do_launch_build()
 240
 241     def _finish_build(self):
 242         self._do_wait_build()
 243         self._do_install()
 244
 245     def _do_build_slave(self):
 246         if not self.sources and not self.build:
 247             return None
 248
 249         # Create build script
 250         files = set()
 251
 252         if self.sources:
 253             sources = self.sources.split(' ')
 254             files.update(
 255                 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
 256                     os.path.join(self._master.home_path, os.path.basename(source)),)
 257                 for source in sources
 258             )
 259
 260         if self.build:
 261             files.add(
 262                 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
 263                     os.path.join(self._master.home_path, 'build.tar.gz'),)
 264             )
 265
 266         launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
 267                         " && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
 268                         " && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" %  \
 269         {
 270             'prk' : server.shell_escape(self._master_prk_name),
 271             'puk' : server.shell_escape(self._master_puk_name),
 272         }
 273
 274         kill_agent = "kill $SSH_AGENT_PID"
 275
 276         waitmaster = "{ . ./.ssh-agent.sh ; while [[ $(ssh -q -o UserKnownHostsFile=%(hostkey)s %(master)s cat %(token_path)s) != %(token)s ]] ; do sleep 5 ; done ; }" % {
 277             'hostkey' : 'master_known_hosts',
 278             'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostname),
 279             'token_path' : os.path.join(self._master.home_path, 'build.token'),
 280             'token' : server.shell_escape(self._master._master_token),
 281         }
 282
 283         syncfiles = "scp -p -o UserKnownHostsFile=%(hostkey)s %(files)s ." % {
 284             'hostkey' : 'master_known_hosts',
 285             'files' : ' '.join(files),
 286         }
 287         if self.build:
 288             syncfiles += " && tar xzf build.tar.gz"
 289         syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
 290         syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
 291
 292         cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
 293             'prk' : server.shell_escape(self._master_prk_name),
 294             'puk' : server.shell_escape(self._master_puk_name),
 295         }
 296
 297         slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s )" % {
 298             'waitmaster' : waitmaster,
 299             'syncfiles' : syncfiles,
 300             'cleanup' : cleanup,
 301             'kill_agent' : kill_agent,
 302             'launch_agent' : launch_agent,
 303             'home' : server.shell_escape(self.home_path),
 304         }
 305
 306         return cStringIO.StringIO(slavescript)
 307
 308     def _do_launch_build(self):
 309         script = "bash ./nepi-build.sh"
 310         if self._master_passphrase:
 311             script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
 312                 server.shell_escape(self._master_passphrase),
 313                 script
 314             )
 315         (out,err),proc = rspawn.remote_spawn(
 316             script,
 317
 318             pidfile = 'build-pid',
 319             home = self.home_path,
 320             stdin = '/dev/null',
 321             stdout = 'buildlog',
 322             stderr = rspawn.STDOUT,
 323
 324             host = self.node.hostname,
 325             port = None,
 326             user = self.node.slicename,
 327             agent = None,
 328             ident_key = self.node.ident_path,
 329             server_key = self.node.server_key
 330             )
 331
 332         if proc.wait():
 333             raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
 334
 335
 336         pid = ppid = None
 337         delay = 1.0
 338         for i in xrange(5):
 339             pidtuple = rspawn.remote_check_pid(
 340                 os.path.join(self.home_path,'build-pid'),
 341                 host = self.node.hostname,
 342                 port = None,
 343                 user = self.node.slicename,
 344                 agent = None,
 345                 ident_key = self.node.ident_path,
 346                 server_key = self.node.server_key
 347                 )
 348
 349             if pidtuple:
 350                 pid, ppid = pidtuple
 351                 self._build_pid, self._build_ppid = pidtuple
 352                 break
 353             else:
 354                 time.sleep(delay)
 355                 delay = min(30,delay*1.2)
 356         else:
 357             raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
 358
 359     def _do_wait_build(self):
 360         pid = self._build_pid
 361         ppid = self._build_ppid
 362
 363         if pid and ppid:
 364             delay = 1.0
 365             while True:
 366                 status = rspawn.remote_status(
 367                     pid, ppid,
 368                     host = self.node.hostname,
 369                     port = None,
 370                     user = self.node.slicename,
 371                     agent = None,
 372                     ident_key = self.node.ident_path,
 373                     server_key = self.node.server_key
 374                     )
 375
 376                 if status is not rspawn.RUNNING:
 377                     self._build_pid = self._build_ppid = None
 378                     break
 379                 else:
 380                     time.sleep(delay*(0.5+random.random()))
 381                     delay = min(30,delay*1.2)
 382
 383             # check build token
 384
 385             (out,err),proc = server.popen_ssh_command(
 386                 "cat %(token_path)s" % {
 387                     'token_path' : os.path.join(self.home_path, 'build.token'),
 388                 },
 389                 host = self.node.hostname,
 390                 port = None,
 391                 user = self.node.slicename,
 392                 agent = None,
 393                 ident_key = self.node.ident_path,
 394                 server_key = self.node.server_key
 395                 )
 396
 397             slave_token = ""
 398             if not proc.wait() and out:
 399                 slave_token = out.strip()
 400
 401             if slave_token != self._master_token:
 402                 # Get buildlog for the error message
 403
 404                 (buildlog,err),proc = server.popen_ssh_command(
 405                     "cat %(buildlog)s" % {
 406                         'buildlog' : os.path.join(self.home_path, 'buildlog'),
 407                         'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
 408                     },
 409                     host = self.node.hostname,
 410                     port = None,
 411                     user = self.node.slicename,
 412                     agent = None,
 413                     ident_key = self.node.ident_path,
 414                     server_key = self.node.server_key
 415                     )
 416
 417                 proc.wait()
 418
 419                 raise RuntimeError, "Failed to set up application %s: "\
 420                         "build failed, got wrong token from pid %s/%s "\
 421                         "(expected %r, got %r), see buildlog: %s" % (
 422                     self.home_path, pid, ppid, self._master_token, slave_token, buildlog)
 423
 424     def _do_kill_build(self):
 425         pid = self._build_pid
 426         ppid = self._build_ppid
 427
 428         if pid and ppid:
 429             rspawn.remote_kill(
 430                 pid, ppid,
 431                 host = self.node.hostname,
 432                 port = None,
 433                 user = self.node.slicename,
 434                 agent = None,
 435                 ident_key = self.node.ident_path
 436                 )
 437
 438
 439     def _do_build_master(self):
 440         if not self.sources and not self.build and not self.buildDepends:
 441             return None
 442
 443         if self.sources:
 444             sources = self.sources.split(' ')
 445
 446             # Copy all sources
 447             (out,err),proc = server.popen_scp(
 448                 sources,
 449                 "%s@%s:%s" % (self.node.slicename, self.node.hostname,
 450                     os.path.join(self.home_path,'.'),),
 451                 ident_key = self.node.ident_path,
 452                 server_key = self.node.server_key
 453                 )
 454
 455             if proc.wait():
 456                 raise RuntimeError, "Failed upload source file %r: %s %s" % (source, out,err,)
 457
 458         buildscript = cStringIO.StringIO()
 459
 460         if self.buildDepends:
 461             # Install build dependencies
 462             buildscript.write(
 463                 "sudo -S yum -y install %(packages)s\n" % {
 464                     'packages' : self.buildDepends
 465                 }
 466             )
 467
 468
 469         if self.build:
 470             # Build sources
 471             buildscript.write(
 472                 "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
 473                     'command' : self._replace_paths(self.build),
 474                     'home' : server.shell_escape(self.home_path),
 475                 }
 476             )
 477
 478             # Make archive
 479             buildscript.write(
 480                 "tar czf build.tar.gz build && ( echo %(master_token)s > build.token )\n" % {
 481                     'master_token' : server.shell_escape(self._master_token)
 482                 }
 483             )
 484
 485         buildscript.seek(0)
 486
 487         return buildscript
 488
 489
 490     def _do_install(self):
 491         if self.install:
 492             # Install application
 493             (out,err),proc = server.popen_ssh_command(
 494                 "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/installlog >&2 && false )" % {
 495                     'command' : self._replace_paths(self.install),
 496                     'home' : server.shell_escape(self.home_path),
 497                 },
 498                 host = self.node.hostname,
 499                 port = None,
 500                 user = self.node.slicename,
 501                 agent = None,
 502                 ident_key = self.node.ident_path,
 503                 server_key = self.node.server_key
 504                 )
 505
 506             if proc.wait():
 507                 raise RuntimeError, "Failed instal build sources: %s %s" % (out,err,)
 508
 509     def set_master(self, master):
 510         self._master = master
 511
 512     def install_keys(self, prk, puk, passphrase):
 513         # Install keys
 514         self._master_passphrase = passphrase
 515         self._master_prk = prk
 516         self._master_puk = puk
 517         self._master_prk_name = os.path.basename(prk.name)
 518         self._master_puk_name = os.path.basename(puk.name)
 519
 520     def _do_install_keys(self):
 521         prk = self._master_prk
 522         puk = self._master_puk
 523
 524         (out,err),proc = server.popen_scp(
 525             [ prk.name, puk.name ],
 526             '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path ),
 527             port = None,
 528             agent = None,
 529             ident_key = self.node.ident_path,
 530             server_key = self.node.server_key
 531             )
 532
 533         if proc.wait():
 534             raise RuntimeError, "Failed to set up application deployment keys: %s %s" % (out,err,)
 535
 536         (out,err),proc = server.popen_scp(
 537             cStringIO.StringIO('%s,%s %s\n' % (
 538                 self._master.node.hostname, socket.gethostbyname(self._master.node.hostname),
 539                 self._master.node.server_key)),
 540             '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 541                 os.path.join(self.home_path,"master_known_hosts") ),
 542             port = None,
 543             agent = None,
 544             ident_key = self.node.ident_path,
 545             server_key = self.node.server_key
 546             )
 547
 548         if proc.wait():
 549             raise RuntimeError, "Failed to set up application deployment keys: %s %s" % (out,err,)
 550
 551         # No longer need'em
 552         self._master_prk = None
 553         self._master_puk = None
 554
 555     def cleanup(self):
 556         # make sure there's no leftover build processes
 557         self._do_kill_build()
 558
 559
 560 class Application(Dependency):
 561     """
 562     An application also has dependencies, but also a command to be ran and monitored.
 563
 564     It adds the output of that command as traces.
 565     """
 566
 567     TRACES = ('stdout','stderr','buildlog')
 568
 569     def __init__(self, api=None):
 570         super(Application,self).__init__(api)
 571
 572         # Attributes
 573         self.command = None
 574         self.sudo = False
 575
 576         self.stdin = None
 577         self.stdout = None
 578         self.stderr = None
 579
 580         # Those are filled when the app is started
 581         #   Having both pid and ppid makes it harder
 582         #   for pid rollover to induce tracking mistakes
 583         self._started = False
 584         self._pid = None
 585         self._ppid = None
 586
 587         # Do not add to the python path of nodes
 588         self.add_to_path = False
 589
 590     def __str__(self):
 591         return "%s<command:%s%s>" % (
 592             self.__class__.__name__,
 593             "sudo " if self.sudo else "",
 594             self.command,
 595         )
 596
 597     def start(self):
 598         # Create shell script with the command
 599         # This way, complex commands and scripts can be ran seamlessly
 600         # sync files
 601         command = cStringIO.StringIO()
 602         command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
 603             ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
 604         ))
 605         command.write('export PATH=$PATH:%s\n' % (
 606             ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
 607         ))
 608         if self.node.env:
 609             for envkey, envvals in self.node.env.iteritems():
 610                 for envval in envvals:
 611                     command.write('export %s=%s\n' % (envkey, envval))
 612         command.write(self.command)
 613         command.seek(0)
 614
 615         (out,err),proc = server.popen_scp(
 616             command,
 617             '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 618                 os.path.join(self.home_path, "app.sh")),
 619             port = None,
 620             agent = None,
 621             ident_key = self.node.ident_path,
 622             server_key = self.node.server_key
 623             )
 624
 625         if proc.wait():
 626             raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
 627
 628         # Start process in a "daemonized" way, using nohup and heavy
 629         # stdin/out redirection to avoid connection issues
 630         (out,err),proc = rspawn.remote_spawn(
 631             self._replace_paths("bash ./app.sh"),
 632
 633             pidfile = './pid',
 634             home = self.home_path,
 635             stdin = 'stdin' if self.stdin is not None else '/dev/null',
 636             stdout = 'stdout' if self.stdout else '/dev/null',
 637             stderr = 'stderr' if self.stderr else '/dev/null',
 638             sudo = self.sudo,
 639
 640             host = self.node.hostname,
 641             port = None,
 642             user = self.node.slicename,
 643             agent = None,
 644             ident_key = self.node.ident_path,
 645             server_key = self.node.server_key
 646             )
 647
 648         if proc.wait():
 649             raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
 650
 651         self._started = True
 652
 653     def checkpid(self):
 654         # Get PID/PPID
 655         # NOTE: wait a bit for the pidfile to be created
 656         if self._started and not self._pid or not self._ppid:
 657             pidtuple = rspawn.remote_check_pid(
 658                 os.path.join(self.home_path,'pid'),
 659                 host = self.node.hostname,
 660                 port = None,
 661                 user = self.node.slicename,
 662                 agent = None,
 663                 ident_key = self.node.ident_path,
 664                 server_key = self.node.server_key
 665                 )
 666
 667             if pidtuple:
 668                 self._pid, self._ppid = pidtuple
 669
 670     def status(self):
 671         self.checkpid()
 672         if not self._started:
 673             return STATUS_NOT_STARTED
 674         elif not self._pid or not self._ppid:
 675             return STATUS_NOT_STARTED
 676         else:
 677             status = rspawn.remote_status(
 678                 self._pid, self._ppid,
 679                 host = self.node.hostname,
 680                 port = None,
 681                 user = self.node.slicename,
 682                 agent = None,
 683                 ident_key = self.node.ident_path,
 684                 server_key = self.node.server_key
 685                 )
 686
 687             if status is rspawn.NOT_STARTED:
 688                 return STATUS_NOT_STARTED
 689             elif status is rspawn.RUNNING:
 690                 return STATUS_RUNNING
 691             elif status is rspawn.FINISHED:
 692                 return STATUS_FINISHED
 693             else:
 694                 # WTF?
 695                 return STATUS_NOT_STARTED
 696
 697     def kill(self):
 698         status = self.status()
 699         if status == STATUS_RUNNING:
 700             # kill by ppid+pid - SIGTERM first, then try SIGKILL
 701             rspawn.remote_kill(
 702                 self._pid, self._ppid,
 703                 host = self.node.hostname,
 704                 port = None,
 705                 user = self.node.slicename,
 706                 agent = None,
 707                 ident_key = self.node.ident_path,
 708                 server_key = self.node.server_key
 709                 )
 710
 711 class NepiDependency(Dependency):
 712     """
 713     This dependency adds nepi itself to the python path,
 714     so that you may run testbeds within PL nodes.
 715     """
 716
 717     # Class attribute holding a *weak* reference to the shared NEPI tar file
 718     # so that they may share it. Don't operate on the file itself, it would
 719     # be a mess, just use its path.
 720     _shared_nepi_tar = None
 721
 722     def __init__(self, api = None):
 723         super(NepiDependency, self).__init__(api)
 724
 725         self._tarball = None
 726
 727         self.depends = 'python python-ipaddr python-setuptools'
 728
 729         # our sources are in our ad-hoc tarball
 730         self.sources = self.tarball.name
 731
 732         tarname = os.path.basename(self.tarball.name)
 733
 734         # it's already built - just move the tarball into place
 735         self.build = "mv -f ${SOURCES}/%s ." % (tarname,)
 736
 737         # unpack it into sources, and we're done
 738         self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,)
 739
 740     @property
 741     def tarball(self):
 742         if self._tarball is None:
 743             shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
 744             if shared_tar is not None:
 745                 self._tarball = shared_tar
 746             else:
 747                 # Build an ad-hoc tarball
 748                 # Prebuilt
 749                 import nepi
 750                 import tempfile
 751
 752                 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
 753
 754                 proc = subprocess.Popen(
 755                     ["tar", "czf", shared_tar.name,
 756                         '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
 757                         'nepi'],
 758                     stdout = open("/dev/null","w"),
 759                     stdin = open("/dev/null","r"))
 760
 761                 if proc.wait():
 762                     raise RuntimeError, "Failed to create nepi tarball"
 763
 764                 self._tarball = self._shared_nepi_tar = shared_tar
 765
 766         return self._tarball
 767
 768 class NS3Dependency(Dependency):
 769     """
 770     This dependency adds NS3 libraries to the library paths,
 771     so that you may run the NS3 testbed within PL nodes.
 772
 773     You'll also need the NepiDependency.
 774     """
 775
 776     def __init__(self, api = None):
 777         super(NS3Dependency, self).__init__(api)
 778
 779         self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip'
 780
 781         # We have to download the sources, untar, build...
 782         pybindgen_source_url = "http://pybindgen.googlecode.com/files/pybindgen-0.15.0.zip"
 783         pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
 784         ns3_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/nepi-ns-3.9/archive/tip.tar.gz"
 785         passfd_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/python-passfd/archive/tip.tar.gz"
 786         self.build =(
 787             " ( "
 788             "  cd .. && "
 789             "  python -c 'import pygccxml, pybindgen, passfd' && "
 790             "  test -f lib/_ns3.so && "
 791             "  test -f lib/libns3.so "
 792             " ) || ( "
 793                 # Not working, rebuild
 794                      "wget -q -c -O pybindgen-src.zip %(pybindgen_source_url)s && " # continue, to exploit the case when it has already been dl'ed
 795                      "wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
 796                      "wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
 797                      "wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
 798                      "unzip -n pybindgen-src.zip && " # Do not overwrite files, to exploit the case when it has already been built
 799                      "unzip -n pygccxml-1.0.0.zip && "
 800                      "mkdir -p ns3-src && "
 801                      "mkdir -p passfd-src && "
 802                      "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
 803                      "tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
 804                      "rm -rf target && "    # mv doesn't like unclean targets
 805                      "mkdir -p target && "
 806                      "cd pygccxml-1.0.0 && "
 807                      "rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
 808                      "python setup.py build && "
 809                      "python setup.py install --install-lib ${BUILD}/target && "
 810                      "python setup.py clean && "
 811                      "cd ../pybindgen-0.15.0 && "
 812                      "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
 813                      "./waf configure --prefix=${BUILD}/target -d release && "
 814                      "./waf && "
 815                      "./waf install && "
 816                      "./waf clean && "
 817                      "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
 818                      "rm -rf ${BUILD}/target/lib && "
 819                      "cd ../passfd-src && "
 820                      "python setup.py build && "
 821                      "python setup.py install --install-lib ${BUILD}/target && "
 822                      "python setup.py clean && "
 823                      "cd ../ns3-src && "
 824                      "./waf configure --prefix=${BUILD}/target -d release --disable-examples --high-precision-as-double && "
 825                      "./waf &&"
 826                      "./waf install && "
 827                      "./waf clean"
 828              " )"
 829                      % dict(
 830                         pybindgen_source_url = server.shell_escape(pybindgen_source_url),
 831                         pygccxml_source_url = server.shell_escape(pygccxml_source_url),
 832                         ns3_source_url = server.shell_escape(ns3_source_url),
 833                         passfd_source_url = server.shell_escape(passfd_source_url),
 834                      ))
 835
 836         # Just move ${BUILD}/target
 837         self.install = (
 838             " ( "
 839             "  cd .. && "
 840             "  python -c 'import pygccxml, pybindgen, passfd' && "
 841             "  test -f lib/_ns3.so && "
 842             "  test -f lib/libns3.so "
 843             " ) || ( "
 844                 # Not working, reinstall
 845                     "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
 846                     "mv -f ${BUILD}/target/* ${SOURCES}"
 847             " )"
 848         )
 849
 850         # Set extra environment paths
 851         self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
 852         self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib/libns3.so"
 853
 854     @property
 855     def tarball(self):
 856         if self._tarball is None:
 857             shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
 858             if shared_tar is not None:
 859                 self._tarball = shared_tar
 860             else:
 861                 # Build an ad-hoc tarball
 862                 # Prebuilt
 863                 import nepi
 864                 import tempfile
 865
 866                 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
 867
 868                 proc = subprocess.Popen(
 869                     ["tar", "czf", shared_tar.name,
 870                         '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
 871                         'nepi'],
 872                     stdout = open("/dev/null","w"),
 873                     stdin = open("/dev/null","r"))
 874
 875                 if proc.wait():
 876                     raise RuntimeError, "Failed to create nepi tarball"
 877
 878                 self._tarball = self._shared_nepi_tar = shared_tar
 879
 880         return self._tarball
 881
 882