src/nepi/testbeds/planetlab/application.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from constants import TESTBED_ID
   5 import plcapi
   6 import operator
   7 import os
   8 import os.path
   9 import sys
  10 import nepi.util.server as server
  11 import cStringIO
  12 import subprocess
  13 import rspawn
  14 import random
  15 import time
  16 import socket
  17 import threading
  18
  19 from nepi.util.constants import STATUS_NOT_STARTED, STATUS_RUNNING, \
  20         STATUS_FINISHED
  21
  22 class Dependency(object):
  23     """
  24     A Dependency is in every respect like an application.
  25
  26     It depends on some packages, it may require building binaries, it must deploy
  27     them...
  28
  29     But it has no command. Dependencies aren't ever started, or stopped, and have
  30     no status.
  31     """
  32
  33     TRACES = ('buildlog')
  34
  35     def __init__(self, api=None):
  36         if not api:
  37             api = plcapi.PLCAPI()
  38         self._api = api
  39
  40         # Attributes
  41         self.command = None
  42         self.sudo = False
  43
  44         self.build = None
  45         self.install = None
  46         self.depends = None
  47         self.buildDepends = None
  48         self.sources = None
  49         self.rpmFusion = False
  50         self.env = {}
  51
  52         self.stdin = None
  53         self.stdout = None
  54         self.stderr = None
  55         self.buildlog = None
  56
  57         self.add_to_path = True
  58
  59         # Those are filled when the app is configured
  60         self.home_path = None
  61
  62         # Those are filled when an actual node is connected
  63         self.node = None
  64
  65         # Those are filled when the app is started
  66         #   Having both pid and ppid makes it harder
  67         #   for pid rollover to induce tracking mistakes
  68         self._started = False
  69         self._setup = False
  70         self._setuper = None
  71         self._pid = None
  72         self._ppid = None
  73
  74         # Spanning tree deployment
  75         self._master = None
  76         self._master_passphrase = None
  77         self._master_prk = None
  78         self._master_puk = None
  79         self._master_token = ''.join(map(chr,[rng.randint(0,255)
  80                                       for rng in (random.SystemRandom(),)
  81                                       for i in xrange(8)] )).encode("hex")
  82         self._build_pid = None
  83         self._build_ppid = None
  84
  85
  86     def __str__(self):
  87         return "%s<%s>" % (
  88             self.__class__.__name__,
  89             ' '.join(filter(bool,(self.depends, self.sources)))
  90         )
  91
  92     def validate(self):
  93         if self.home_path is None:
  94             raise AssertionError, "Misconfigured application: missing home path"
  95         if self.node.ident_path is None or not os.access(self.node.ident_path, os.R_OK):
  96             raise AssertionError, "Misconfigured application: missing slice SSH key"
  97         if self.node is None:
  98             raise AssertionError, "Misconfigured application: unconnected node"
  99         if self.node.hostname is None:
 100             raise AssertionError, "Misconfigured application: misconfigured node"
 101         if self.node.slicename is None:
 102             raise AssertionError, "Misconfigured application: unspecified slice"
 103
 104     def remote_trace_path(self, whichtrace):
 105         if whichtrace in self.TRACES:
 106             tracefile = os.path.join(self.home_path, whichtrace)
 107         else:
 108             tracefile = None
 109
 110         return tracefile
 111
 112     def sync_trace(self, local_dir, whichtrace):
 113         tracefile = self.remote_trace_path(whichtrace)
 114         if not tracefile:
 115             return None
 116
 117         local_path = os.path.join(local_dir, tracefile)
 118
 119         # create parent local folders
 120         proc = subprocess.Popen(
 121             ["mkdir", "-p", os.path.dirname(local_path)],
 122             stdout = open("/dev/null","w"),
 123             stdin = open("/dev/null","r"))
 124
 125         if proc.wait():
 126             raise RuntimeError, "Failed to synchronize trace"
 127
 128         # sync files
 129         try:
 130             self._popen_scp(
 131                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 132                     tracefile),
 133                 local_path
 134                 )
 135         except RuntimeError, e:
 136             raise RuntimeError, "Failed to synchronize trace: %s %s" \
 137                     % (e.args[0], e.args[1],)
 138
 139         return local_path
 140
 141     def setup(self):
 142         self._make_home()
 143         self._launch_build()
 144         self._finish_build()
 145         self._setup = True
 146
 147     def async_setup(self):
 148         if not self._setuper:
 149             def setuper():
 150                 try:
 151                     self.setup()
 152                 except:
 153                     self._setuper._exc.append(sys.exc_info())
 154             self._setuper = threading.Thread(
 155                 target = setuper)
 156             self._setuper._exc = []
 157             self._setuper.start()
 158
 159     def async_setup_wait(self):
 160         if not self._setup:
 161             if self._setuper:
 162                 self._setuper.join()
 163                 if not self._setup:
 164                     if self._setuper._exc:
 165                         exctyp,exval,exctrace = self._setuper._exc[0]
 166                         raise exctyp,exval,exctrace
 167                     else:
 168                         raise RuntimeError, "Failed to setup application"
 169             else:
 170                 self.setup()
 171
 172     def _make_home(self):
 173         # Make sure all the paths are created where
 174         # they have to be created for deployment
 175         # sync files
 176         try:
 177             self._popen_ssh_command(
 178                 "mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" \
 179                     % { 'home' : server.shell_escape(self.home_path) }
 180                 )
 181         except RuntimeError, e:
 182             raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, e.args[0], e.args[1],)
 183
 184         if self.stdin:
 185             # Write program input
 186             try:
 187                 self._popen_scp(
 188                     cStringIO.StringIO(self.stdin),
 189                     '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 190                         os.path.join(self.home_path, 'stdin') ),
 191                     )
 192             except RuntimeError, e:
 193                 raise RuntimeError, "Failed to set up application %s: %s %s" \
 194                         % (self.home_path, e.args[0], e.args[1],)
 195
 196     def _replace_paths(self, command):
 197         """
 198         Replace all special path tags with shell-escaped actual paths.
 199         """
 200         # need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
 201         root = '' if self.home_path.startswith('/') else "${HOME}/"
 202         return ( command
 203             .replace("${SOURCES}", root+server.shell_escape(self.home_path))
 204             .replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
 205
 206     def _launch_build(self):
 207         if self._master is not None:
 208             self._do_install_keys()
 209             buildscript = self._do_build_slave()
 210         else:
 211             buildscript = self._do_build_master()
 212
 213         if buildscript is not None:
 214             # upload build script
 215             try:
 216                 self._popen_scp(
 217                     buildscript,
 218                     '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 219                         os.path.join(self.home_path, 'nepi-build.sh') )
 220                     )
 221             except RuntimeError, e:
 222                 raise RuntimeError, "Failed to set up application %s: %s %s" \
 223                         % (self.home_path, e.args[0], e.args[1],)
 224
 225             # launch build
 226             self._do_launch_build()
 227
 228     def _finish_build(self):
 229         self._do_wait_build()
 230         self._do_install()
 231
 232     def _do_build_slave(self):
 233         if not self.sources and not self.build:
 234             return None
 235
 236         # Create build script
 237         files = set()
 238
 239         if self.sources:
 240             sources = self.sources.split(' ')
 241             files.update(
 242                 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
 243                     os.path.join(self._master.home_path, os.path.basename(source)),)
 244                 for source in sources
 245             )
 246
 247         if self.build:
 248             files.add(
 249                 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
 250                     os.path.join(self._master.home_path, 'build.tar.gz'),)
 251             )
 252
 253         launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
 254                         " && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
 255                         " && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" %  \
 256         {
 257             'prk' : server.shell_escape(self._master_prk_name),
 258             'puk' : server.shell_escape(self._master_puk_name),
 259         }
 260
 261         kill_agent = "kill $SSH_AGENT_PID"
 262
 263         waitmaster = "{ . ./.ssh-agent.sh ; while [[ $(ssh -q -o UserKnownHostsFile=%(hostkey)s %(master)s cat %(token_path)s) != %(token)s ]] ; do sleep 5 ; done ; }" % {
 264             'hostkey' : 'master_known_hosts',
 265             'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostname),
 266             'token_path' : os.path.join(self._master.home_path, 'build.token'),
 267             'token' : server.shell_escape(self._master._master_token),
 268         }
 269
 270         syncfiles = "scp -p -o UserKnownHostsFile=%(hostkey)s %(files)s ." % {
 271             'hostkey' : 'master_known_hosts',
 272             'files' : ' '.join(files),
 273         }
 274         if self.build:
 275             syncfiles += " && tar xzf build.tar.gz"
 276         syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
 277         syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
 278
 279         cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
 280             'prk' : server.shell_escape(self._master_prk_name),
 281             'puk' : server.shell_escape(self._master_puk_name),
 282         }
 283
 284         slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s )" % {
 285             'waitmaster' : waitmaster,
 286             'syncfiles' : syncfiles,
 287             'cleanup' : cleanup,
 288             'kill_agent' : kill_agent,
 289             'launch_agent' : launch_agent,
 290             'home' : server.shell_escape(self.home_path),
 291         }
 292
 293         return cStringIO.StringIO(slavescript)
 294
 295     def _do_launch_build(self):
 296         script = "bash ./nepi-build.sh"
 297         if self._master_passphrase:
 298             script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
 299                 server.shell_escape(self._master_passphrase),
 300                 script
 301             )
 302         (out,err),proc = rspawn.remote_spawn(
 303             script,
 304             pidfile = 'build-pid',
 305             home = self.home_path,
 306             stdin = '/dev/null',
 307             stdout = 'buildlog',
 308             stderr = rspawn.STDOUT,
 309
 310             host = self.node.hostname,
 311             port = None,
 312             user = self.node.slicename,
 313             agent = None,
 314             ident_key = self.node.ident_path,
 315             server_key = self.node.server_key
 316             )
 317
 318         if proc.wait():
 319             raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
 320
 321
 322         pid = ppid = None
 323         delay = 1.0
 324         for i in xrange(5):
 325             pidtuple = rspawn.remote_check_pid(
 326                 os.path.join(self.home_path,'build-pid'),
 327                 host = self.node.hostname,
 328                 port = None,
 329                 user = self.node.slicename,
 330                 agent = None,
 331                 ident_key = self.node.ident_path,
 332                 server_key = self.node.server_key
 333                 )
 334
 335             if pidtuple:
 336                 pid, ppid = pidtuple
 337                 self._build_pid, self._build_ppid = pidtuple
 338                 break
 339             else:
 340                 time.sleep(delay)
 341                 delay = min(30,delay*1.2)
 342         else:
 343             raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
 344
 345     def _do_wait_build(self):
 346         pid = self._build_pid
 347         ppid = self._build_ppid
 348
 349         if pid and ppid:
 350             delay = 1.0
 351             while True:
 352                 status = rspawn.remote_status(
 353                     pid, ppid,
 354                     host = self.node.hostname,
 355                     port = None,
 356                     user = self.node.slicename,
 357                     agent = None,
 358                     ident_key = self.node.ident_path,
 359                     server_key = self.node.server_key
 360                     )
 361
 362                 if status is not rspawn.RUNNING:
 363                     self._build_pid = self._build_ppid = None
 364                     break
 365                 else:
 366                     time.sleep(delay*(0.5+random.random()))
 367                     delay = min(30,delay*1.2)
 368
 369             # check build token
 370             (out, err), proc = self._popen_ssh_command(
 371                 "cat %(token_path)s" % {
 372                     'token_path' : os.path.join(self.home_path, 'build.token'),
 373                 })
 374             slave_token = ""
 375             if not proc.wait() and out:
 376                 slave_token = out.strip()
 377
 378             if slave_token != self._master_token:
 379                 # Get buildlog for the error message
 380
 381                 (buildlog, err), proc = self._popen_ssh_command(
 382                     "cat %(buildlog)s" % {
 383                         'buildlog' : os.path.join(self.home_path, 'buildlog'),
 384                         'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
 385                     })
 386
 387                 proc.wait()
 388
 389                 raise RuntimeError, "Failed to set up application %s: "\
 390                         "build failed, got wrong token from pid %s/%s "\
 391                         "(expected %r, got %r), see buildlog: %s" % (
 392                     self.home_path, pid, ppid, self._master_token, slave_token, buildlog)
 393
 394     def _do_kill_build(self):
 395         pid = self._build_pid
 396         ppid = self._build_ppid
 397
 398         if pid and ppid:
 399             rspawn.remote_kill(
 400                 pid, ppid,
 401                 host = self.node.hostname,
 402                 port = None,
 403                 user = self.node.slicename,
 404                 agent = None,
 405                 ident_key = self.node.ident_path
 406                 )
 407
 408
 409     def _do_build_master(self):
 410         if not self.sources and not self.build and not self.buildDepends:
 411             return None
 412
 413         if self.sources:
 414             sources = self.sources.split(' ')
 415
 416             # Copy all sources
 417             try:
 418                 self._popen_scp(
 419                     sources,
 420                     "%s@%s:%s" % (self.node.slicename, self.node.hostname,
 421                         os.path.join(self.home_path,'.'),)
 422                     )
 423             except RuntimeError, e:
 424                 raise RuntimeError, "Failed upload source file %r: %s %s" \
 425                         % (sources, e.args[0], e.args[1],)
 426
 427         buildscript = cStringIO.StringIO()
 428
 429         if self.buildDepends:
 430             # Install build dependencies
 431             buildscript.write(
 432                 "sudo -S yum -y install %(packages)s\n" % {
 433                     'packages' : self.buildDepends
 434                 }
 435             )
 436
 437
 438         if self.build:
 439             # Build sources
 440             buildscript.write(
 441                 "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
 442                     'command' : self._replace_paths(self.build),
 443                     'home' : server.shell_escape(self.home_path),
 444                 }
 445             )
 446
 447             # Make archive
 448             buildscript.write(
 449                 "tar czf build.tar.gz build && ( echo %(master_token)s > build.token )\n" % {
 450                     'master_token' : server.shell_escape(self._master_token)
 451                 }
 452             )
 453
 454         buildscript.seek(0)
 455
 456         return buildscript
 457
 458     def _do_install(self):
 459         if self.install:
 460             # Install application
 461             try:
 462                 self._popen_ssh_command(
 463                     "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/installlog >&2 && false )" % \
 464                         {
 465                         'command' : self._replace_paths(self.install),
 466                         'home' : server.shell_escape(self.home_path),
 467                         }
 468                     )
 469             except RuntimeError, e:
 470                 raise RuntimeError, "Failed instal build sources: %s %s" % (e.args[0], e.args[1],)
 471
 472     def set_master(self, master):
 473         self._master = master
 474
 475     def install_keys(self, prk, puk, passphrase):
 476         # Install keys
 477         self._master_passphrase = passphrase
 478         self._master_prk = prk
 479         self._master_puk = puk
 480         self._master_prk_name = os.path.basename(prk.name)
 481         self._master_puk_name = os.path.basename(puk.name)
 482
 483     def _do_install_keys(self):
 484         prk = self._master_prk
 485         puk = self._master_puk
 486
 487         try:
 488             self._popen_scp(
 489                 [ prk.name, puk.name ],
 490                 '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path )
 491                 )
 492         except RuntimeError, e:
 493             raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
 494                     % (e.args[0], e.args[1],)
 495
 496         try:
 497             self._popen_scp(
 498                 cStringIO.StringIO('%s,%s %s\n' % (
 499                     self._master.node.hostname, socket.gethostbyname(self._master.node.hostname),
 500                     self._master.node.server_key)),
 501                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 502                     os.path.join(self.home_path,"master_known_hosts") )
 503                 )
 504         except RuntimeError, e:
 505             raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
 506                     % (e.args[0], e.args[1],)
 507
 508         # No longer need'em
 509         self._master_prk = None
 510         self._master_puk = None
 511
 512     def cleanup(self):
 513         # make sure there's no leftover build processes
 514         self._do_kill_build()
 515
 516     @server.eintr_retry
 517     def _popen_scp(self, src, dst, retry = True):
 518         (out,err),proc = server.popen_scp(
 519             src,
 520             dst,
 521             port = None,
 522             agent = None,
 523             ident_key = self.node.ident_path,
 524             server_key = self.node.server_key
 525             )
 526
 527         if server.eintr_retry(proc.wait)():
 528             raise RuntimeError, (out, err)
 529         return (out, err), proc
 530
 531
 532     @server.eintr_retry
 533     def _popen_ssh_command(self, command, retry = True):
 534         (out,err),proc = server.popen_ssh_command(
 535             command,
 536             host = self.node.hostname,
 537             port = None,
 538             user = self.node.slicename,
 539             agent = None,
 540             ident_key = self.node.ident_path,
 541             server_key = self.node.server_key
 542             )
 543
 544         if server.eintr_retry(proc.wait)():
 545             raise RuntimeError, (out, err)
 546         return (out, err), proc
 547
 548 class Application(Dependency):
 549     """
 550     An application also has dependencies, but also a command to be ran and monitored.
 551
 552     It adds the output of that command as traces.
 553     """
 554
 555     TRACES = ('stdout','stderr','buildlog')
 556
 557     def __init__(self, api=None):
 558         super(Application,self).__init__(api)
 559
 560         # Attributes
 561         self.command = None
 562         self.sudo = False
 563
 564         self.stdin = None
 565         self.stdout = None
 566         self.stderr = None
 567
 568         # Those are filled when the app is started
 569         #   Having both pid and ppid makes it harder
 570         #   for pid rollover to induce tracking mistakes
 571         self._started = False
 572         self._pid = None
 573         self._ppid = None
 574
 575         # Do not add to the python path of nodes
 576         self.add_to_path = False
 577
 578     def __str__(self):
 579         return "%s<command:%s%s>" % (
 580             self.__class__.__name__,
 581             "sudo " if self.sudo else "",
 582             self.command,
 583         )
 584
 585     def start(self):
 586         # Create shell script with the command
 587         # This way, complex commands and scripts can be ran seamlessly
 588         # sync files
 589         command = cStringIO.StringIO()
 590         command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
 591             ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
 592         ))
 593         command.write('export PATH=$PATH:%s\n' % (
 594             ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
 595         ))
 596         if self.node.env:
 597             for envkey, envvals in self.node.env.iteritems():
 598                 for envval in envvals:
 599                     command.write('export %s=%s\n' % (envkey, envval))
 600         command.write(self.command)
 601         command.seek(0)
 602
 603         try:
 604             self._popen_scp(
 605                 command,
 606                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 607                     os.path.join(self.home_path, "app.sh"))
 608                 )
 609         except RuntimeError, e:
 610             raise RuntimeError, "Failed to set up application: %s %s" \
 611                     % (e.args[0], e.args[1],)
 612
 613         # Start process in a "daemonized" way, using nohup and heavy
 614         # stdin/out redirection to avoid connection issues
 615         (out,err),proc = rspawn.remote_spawn(
 616             self._replace_paths("bash ./app.sh"),
 617
 618             pidfile = './pid',
 619             home = self.home_path,
 620             stdin = 'stdin' if self.stdin is not None else '/dev/null',
 621             stdout = 'stdout' if self.stdout else '/dev/null',
 622             stderr = 'stderr' if self.stderr else '/dev/null',
 623             sudo = self.sudo,
 624
 625             host = self.node.hostname,
 626             port = None,
 627             user = self.node.slicename,
 628             agent = None,
 629             ident_key = self.node.ident_path,
 630             server_key = self.node.server_key
 631             )
 632
 633         if proc.wait():
 634             raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
 635
 636         self._started = True
 637
 638     def checkpid(self):
 639         # Get PID/PPID
 640         # NOTE: wait a bit for the pidfile to be created
 641         if self._started and not self._pid or not self._ppid:
 642             pidtuple = rspawn.remote_check_pid(
 643                 os.path.join(self.home_path,'pid'),
 644                 host = self.node.hostname,
 645                 port = None,
 646                 user = self.node.slicename,
 647                 agent = None,
 648                 ident_key = self.node.ident_path,
 649                 server_key = self.node.server_key
 650                 )
 651
 652             if pidtuple:
 653                 self._pid, self._ppid = pidtuple
 654
 655     def status(self):
 656         self.checkpid()
 657         if not self._started:
 658             return STATUS_NOT_STARTED
 659         elif not self._pid or not self._ppid:
 660             return STATUS_NOT_STARTED
 661         else:
 662             status = rspawn.remote_status(
 663                 self._pid, self._ppid,
 664                 host = self.node.hostname,
 665                 port = None,
 666                 user = self.node.slicename,
 667                 agent = None,
 668                 ident_key = self.node.ident_path,
 669                 server_key = self.node.server_key
 670                 )
 671
 672             if status is rspawn.NOT_STARTED:
 673                 return STATUS_NOT_STARTED
 674             elif status is rspawn.RUNNING:
 675                 return STATUS_RUNNING
 676             elif status is rspawn.FINISHED:
 677                 return STATUS_FINISHED
 678             else:
 679                 # WTF?
 680                 return STATUS_NOT_STARTED
 681
 682     def kill(self):
 683         status = self.status()
 684         if status == STATUS_RUNNING:
 685             # kill by ppid+pid - SIGTERM first, then try SIGKILL
 686             rspawn.remote_kill(
 687                 self._pid, self._ppid,
 688                 host = self.node.hostname,
 689                 port = None,
 690                 user = self.node.slicename,
 691                 agent = None,
 692                 ident_key = self.node.ident_path,
 693                 server_key = self.node.server_key
 694                 )
 695
 696
 697 class NepiDependency(Dependency):
 698     """
 699     This dependency adds nepi itself to the python path,
 700     so that you may run testbeds within PL nodes.
 701     """
 702
 703     # Class attribute holding a *weak* reference to the shared NEPI tar file
 704     # so that they may share it. Don't operate on the file itself, it would
 705     # be a mess, just use its path.
 706     _shared_nepi_tar = None
 707
 708     def __init__(self, api = None):
 709         super(NepiDependency, self).__init__(api)
 710
 711         self._tarball = None
 712
 713         self.depends = 'python python-ipaddr python-setuptools'
 714
 715         # our sources are in our ad-hoc tarball
 716         self.sources = self.tarball.name
 717
 718         tarname = os.path.basename(self.tarball.name)
 719
 720         # it's already built - just move the tarball into place
 721         self.build = "mv -f ${SOURCES}/%s ." % (tarname,)
 722
 723         # unpack it into sources, and we're done
 724         self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,)
 725
 726     @property
 727     def tarball(self):
 728         if self._tarball is None:
 729             shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
 730             if shared_tar is not None:
 731                 self._tarball = shared_tar
 732             else:
 733                 # Build an ad-hoc tarball
 734                 # Prebuilt
 735                 import nepi
 736                 import tempfile
 737
 738                 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
 739
 740                 proc = subprocess.Popen(
 741                     ["tar", "czf", shared_tar.name,
 742                         '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
 743                         'nepi'],
 744                     stdout = open("/dev/null","w"),
 745                     stdin = open("/dev/null","r"))
 746
 747                 if proc.wait():
 748                     raise RuntimeError, "Failed to create nepi tarball"
 749
 750                 self._tarball = self._shared_nepi_tar = shared_tar
 751
 752         return self._tarball
 753
 754 class NS3Dependency(Dependency):
 755     """
 756     This dependency adds NS3 libraries to the library paths,
 757     so that you may run the NS3 testbed within PL nodes.
 758
 759     You'll also need the NepiDependency.
 760     """
 761
 762     def __init__(self, api = None):
 763         super(NS3Dependency, self).__init__(api)
 764
 765         self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip'
 766
 767         # We have to download the sources, untar, build...
 768         pybindgen_source_url = "http://pybindgen.googlecode.com/files/pybindgen-0.15.0.zip"
 769         pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
 770         ns3_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/nepi-ns-3.9/archive/tip.tar.gz"
 771         passfd_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/python-passfd/archive/tip.tar.gz"
 772         self.build =(
 773             " ( "
 774             "  cd .. && "
 775             "  python -c 'import pygccxml, pybindgen, passfd' && "
 776             "  test -f lib/_ns3.so && "
 777             "  test -f lib/libns3.so "
 778             " ) || ( "
 779                 # Not working, rebuild
 780                      "wget -q -c -O pybindgen-src.zip %(pybindgen_source_url)s && " # continue, to exploit the case when it has already been dl'ed
 781                      "wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
 782                      "wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
 783                      "wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
 784                      "unzip -n pybindgen-src.zip && " # Do not overwrite files, to exploit the case when it has already been built
 785                      "unzip -n pygccxml-1.0.0.zip && "
 786                      "mkdir -p ns3-src && "
 787                      "mkdir -p passfd-src && "
 788                      "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
 789                      "tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
 790                      "rm -rf target && "    # mv doesn't like unclean targets
 791                      "mkdir -p target && "
 792                      "cd pygccxml-1.0.0 && "
 793                      "rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
 794                      "python setup.py build && "
 795                      "python setup.py install --install-lib ${BUILD}/target && "
 796                      "python setup.py clean && "
 797                      "cd ../pybindgen-0.15.0 && "
 798                      "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
 799                      "./waf configure --prefix=${BUILD}/target -d release && "
 800                      "./waf && "
 801                      "./waf install && "
 802                      "./waf clean && "
 803                      "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
 804                      "rm -rf ${BUILD}/target/lib && "
 805                      "cd ../passfd-src && "
 806                      "python setup.py build && "
 807                      "python setup.py install --install-lib ${BUILD}/target && "
 808                      "python setup.py clean && "
 809                      "cd ../ns3-src && "
 810                      "./waf configure --prefix=${BUILD}/target -d release --disable-examples --high-precision-as-double && "
 811                      "./waf &&"
 812                      "./waf install && "
 813                      "./waf clean"
 814              " )"
 815                      % dict(
 816                         pybindgen_source_url = server.shell_escape(pybindgen_source_url),
 817                         pygccxml_source_url = server.shell_escape(pygccxml_source_url),
 818                         ns3_source_url = server.shell_escape(ns3_source_url),
 819                         passfd_source_url = server.shell_escape(passfd_source_url),
 820                      ))
 821
 822         # Just move ${BUILD}/target
 823         self.install = (
 824             " ( "
 825             "  cd .. && "
 826             "  python -c 'import pygccxml, pybindgen, passfd' && "
 827             "  test -f lib/_ns3.so && "
 828             "  test -f lib/libns3.so "
 829             " ) || ( "
 830                 # Not working, reinstall
 831                     "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
 832                     "mv -f ${BUILD}/target/* ${SOURCES}"
 833             " )"
 834         )
 835
 836         # Set extra environment paths
 837         self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
 838         self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib/libns3.so"
 839
 840     @property
 841     def tarball(self):
 842         if self._tarball is None:
 843             shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
 844             if shared_tar is not None:
 845                 self._tarball = shared_tar
 846             else:
 847                 # Build an ad-hoc tarball
 848                 # Prebuilt
 849                 import nepi
 850                 import tempfile
 851
 852                 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
 853
 854                 proc = subprocess.Popen(
 855                     ["tar", "czf", shared_tar.name,
 856                         '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
 857                         'nepi'],
 858                     stdout = open("/dev/null","w"),
 859                     stdin = open("/dev/null","r"))
 860
 861                 if proc.wait():
 862                     raise RuntimeError, "Failed to create nepi tarball"
 863
 864                 self._tarball = self._shared_nepi_tar = shared_tar
 865
 866         return self._tarball
 867
 868