src/nepi/testbeds/planetlab/application.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from constants import TESTBED_ID
   5 import plcapi
   6 import operator
   7 import os
   8 import os.path
   9 import sys
  10 import nepi.util.server as server
  11 import cStringIO
  12 import subprocess
  13 import rspawn
  14 import random
  15 import time
  16 import socket
  17 import threading
  18 import logging
  19
  20 from nepi.util.constants import ApplicationStatus as AS
  21
  22 class Dependency(object):
  23     """
  24     A Dependency is in every respect like an application.
  25
  26     It depends on some packages, it may require building binaries, it must deploy
  27     them...
  28
  29     But it has no command. Dependencies aren't ever started, or stopped, and have
  30     no status.
  31     """
  32
  33     TRACES = ('buildlog')
  34
  35     def __init__(self, api=None):
  36         if not api:
  37             api = plcapi.PLCAPI()
  38         self._api = api
  39
  40         # Attributes
  41         self.command = None
  42         self.sudo = False
  43
  44         self.build = None
  45         self.install = None
  46         self.depends = None
  47         self.buildDepends = None
  48         self.sources = None
  49         self.rpmFusion = False
  50         self.env = {}
  51
  52         self.stdin = None
  53         self.stdout = None
  54         self.stderr = None
  55         self.buildlog = None
  56
  57         self.add_to_path = True
  58
  59         # Those are filled when the app is configured
  60         self.home_path = None
  61
  62         # Those are filled when an actual node is connected
  63         self.node = None
  64
  65         # Those are filled when the app is started
  66         #   Having both pid and ppid makes it harder
  67         #   for pid rollover to induce tracking mistakes
  68         self._started = False
  69         self._setup = False
  70         self._setuper = None
  71         self._pid = None
  72         self._ppid = None
  73
  74         # Spanning tree deployment
  75         self._master = None
  76         self._master_passphrase = None
  77         self._master_prk = None
  78         self._master_puk = None
  79         self._master_token = ''.join(map(chr,[rng.randint(0,255)
  80                                       for rng in (random.SystemRandom(),)
  81                                       for i in xrange(8)] )).encode("hex")
  82         self._build_pid = None
  83         self._build_ppid = None
  84
  85         # Logging
  86         self._logger = logging.getLogger('nepi.testbeds.planetlab')
  87
  88
  89     def __str__(self):
  90         return "%s<%s>" % (
  91             self.__class__.__name__,
  92             ' '.join(filter(bool,(self.depends, self.sources)))
  93         )
  94
  95     def validate(self):
  96         if self.home_path is None:
  97             raise AssertionError, "Misconfigured application: missing home path"
  98         if self.node.ident_path is None or not os.access(self.node.ident_path, os.R_OK):
  99             raise AssertionError, "Misconfigured application: missing slice SSH key"
 100         if self.node is None:
 101             raise AssertionError, "Misconfigured application: unconnected node"
 102         if self.node.hostname is None:
 103             raise AssertionError, "Misconfigured application: misconfigured node"
 104         if self.node.slicename is None:
 105             raise AssertionError, "Misconfigured application: unspecified slice"
 106
 107     def remote_trace_path(self, whichtrace):
 108         if whichtrace in self.TRACES:
 109             tracefile = os.path.join(self.home_path, whichtrace)
 110         else:
 111             tracefile = None
 112
 113         return tracefile
 114
 115     def sync_trace(self, local_dir, whichtrace):
 116         tracefile = self.remote_trace_path(whichtrace)
 117         if not tracefile:
 118             return None
 119
 120         local_path = os.path.join(local_dir, tracefile)
 121
 122         # create parent local folders
 123         proc = subprocess.Popen(
 124             ["mkdir", "-p", os.path.dirname(local_path)],
 125             stdout = open("/dev/null","w"),
 126             stdin = open("/dev/null","r"))
 127
 128         if proc.wait():
 129             raise RuntimeError, "Failed to synchronize trace"
 130
 131         # sync files
 132         try:
 133             self._popen_scp(
 134                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 135                     tracefile),
 136                 local_path
 137                 )
 138         except RuntimeError, e:
 139             raise RuntimeError, "Failed to synchronize trace: %s %s" \
 140                     % (e.args[0], e.args[1],)
 141
 142         return local_path
 143
 144     def recover(self):
 145         # We assume a correct deployment, so recovery only
 146         # means we mark this dependency as deployed
 147         self._setup = True
 148
 149     def setup(self):
 150         self._logger.info("Setting up %s", self)
 151         self._make_home()
 152         self._launch_build()
 153         self._finish_build()
 154         self._setup = True
 155
 156     def async_setup(self):
 157         if not self._setuper:
 158             def setuper():
 159                 try:
 160                     self.setup()
 161                 except:
 162                     self._setuper._exc.append(sys.exc_info())
 163             self._setuper = threading.Thread(
 164                 target = setuper)
 165             self._setuper._exc = []
 166             self._setuper.start()
 167
 168     def async_setup_wait(self):
 169         if not self._setup:
 170             self._logger.info("Waiting for %s to be setup", self)
 171             if self._setuper:
 172                 self._setuper.join()
 173                 if not self._setup:
 174                     if self._setuper._exc:
 175                         exctyp,exval,exctrace = self._setuper._exc[0]
 176                         raise exctyp,exval,exctrace
 177                     else:
 178                         raise RuntimeError, "Failed to setup application"
 179             else:
 180                 self.setup()
 181
 182     def _make_home(self):
 183         # Make sure all the paths are created where
 184         # they have to be created for deployment
 185         # sync files
 186         try:
 187             self._popen_ssh_command(
 188                 "mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" \
 189                     % { 'home' : server.shell_escape(self.home_path) }
 190                 )
 191         except RuntimeError, e:
 192             raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, e.args[0], e.args[1],)
 193
 194         if self.stdin:
 195             # Write program input
 196             try:
 197                 self._popen_scp(
 198                     cStringIO.StringIO(self.stdin),
 199                     '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 200                         os.path.join(self.home_path, 'stdin') ),
 201                     )
 202             except RuntimeError, e:
 203                 raise RuntimeError, "Failed to set up application %s: %s %s" \
 204                         % (self.home_path, e.args[0], e.args[1],)
 205
 206     def _replace_paths(self, command):
 207         """
 208         Replace all special path tags with shell-escaped actual paths.
 209         """
 210         # need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
 211         root = '' if self.home_path.startswith('/') else "${HOME}/"
 212         return ( command
 213             .replace("${SOURCES}", root+server.shell_escape(self.home_path))
 214             .replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
 215
 216     def _launch_build(self):
 217         if self._master is not None:
 218             self._do_install_keys()
 219             buildscript = self._do_build_slave()
 220         else:
 221             buildscript = self._do_build_master()
 222
 223         if buildscript is not None:
 224             self._logger.info("Building %s", self)
 225
 226             # upload build script
 227             try:
 228                 self._popen_scp(
 229                     buildscript,
 230                     '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 231                         os.path.join(self.home_path, 'nepi-build.sh') )
 232                     )
 233             except RuntimeError, e:
 234                 raise RuntimeError, "Failed to set up application %s: %s %s" \
 235                         % (self.home_path, e.args[0], e.args[1],)
 236
 237             # launch build
 238             self._do_launch_build()
 239
 240     def _finish_build(self):
 241         self._do_wait_build()
 242         self._do_install()
 243
 244     def _do_build_slave(self):
 245         if not self.sources and not self.build:
 246             return None
 247
 248         # Create build script
 249         files = set()
 250
 251         if self.sources:
 252             sources = self.sources.split(' ')
 253             files.update(
 254                 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
 255                     os.path.join(self._master.home_path, os.path.basename(source)),)
 256                 for source in sources
 257             )
 258
 259         if self.build:
 260             files.add(
 261                 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
 262                     os.path.join(self._master.home_path, 'build.tar.gz'),)
 263             )
 264
 265         launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
 266                         " && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
 267                         " && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" %  \
 268         {
 269             'prk' : server.shell_escape(self._master_prk_name),
 270             'puk' : server.shell_escape(self._master_puk_name),
 271         }
 272
 273         kill_agent = "kill $SSH_AGENT_PID"
 274
 275         waitmaster = "{ . ./.ssh-agent.sh ; while [[ $(ssh -q -o UserKnownHostsFile=%(hostkey)s %(master)s cat %(token_path)s) != %(token)s ]] ; do sleep 5 ; done ; }" % {
 276             'hostkey' : 'master_known_hosts',
 277             'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostname),
 278             'token_path' : os.path.join(self._master.home_path, 'build.token'),
 279             'token' : server.shell_escape(self._master._master_token),
 280         }
 281
 282         syncfiles = "scp -p -o UserKnownHostsFile=%(hostkey)s %(files)s ." % {
 283             'hostkey' : 'master_known_hosts',
 284             'files' : ' '.join(files),
 285         }
 286         if self.build:
 287             syncfiles += " && tar xzf build.tar.gz"
 288         syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
 289         syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
 290
 291         cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
 292             'prk' : server.shell_escape(self._master_prk_name),
 293             'puk' : server.shell_escape(self._master_puk_name),
 294         }
 295
 296         slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s )" % {
 297             'waitmaster' : waitmaster,
 298             'syncfiles' : syncfiles,
 299             'cleanup' : cleanup,
 300             'kill_agent' : kill_agent,
 301             'launch_agent' : launch_agent,
 302             'home' : server.shell_escape(self.home_path),
 303         }
 304
 305         return cStringIO.StringIO(slavescript)
 306
 307     def _do_launch_build(self):
 308         script = "bash ./nepi-build.sh"
 309         if self._master_passphrase:
 310             script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
 311                 server.shell_escape(self._master_passphrase),
 312                 script
 313             )
 314         (out,err),proc = rspawn.remote_spawn(
 315             script,
 316             pidfile = 'build-pid',
 317             home = self.home_path,
 318             stdin = '/dev/null',
 319             stdout = 'buildlog',
 320             stderr = rspawn.STDOUT,
 321
 322             host = self.node.hostname,
 323             port = None,
 324             user = self.node.slicename,
 325             agent = None,
 326             ident_key = self.node.ident_path,
 327             server_key = self.node.server_key
 328             )
 329
 330         if proc.wait():
 331             raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
 332
 333
 334         pid = ppid = None
 335         delay = 1.0
 336         for i in xrange(5):
 337             pidtuple = rspawn.remote_check_pid(
 338                 os.path.join(self.home_path,'build-pid'),
 339                 host = self.node.hostname,
 340                 port = None,
 341                 user = self.node.slicename,
 342                 agent = None,
 343                 ident_key = self.node.ident_path,
 344                 server_key = self.node.server_key
 345                 )
 346
 347             if pidtuple:
 348                 pid, ppid = pidtuple
 349                 self._build_pid, self._build_ppid = pidtuple
 350                 break
 351             else:
 352                 time.sleep(delay)
 353                 delay = min(30,delay*1.2)
 354         else:
 355             raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
 356
 357         self._logger.info("Deploying %s", self)
 358
 359     def _do_wait_build(self):
 360         pid = self._build_pid
 361         ppid = self._build_ppid
 362
 363         if pid and ppid:
 364             delay = 1.0
 365             first = True
 366             while True:
 367                 status = rspawn.remote_status(
 368                     pid, ppid,
 369                     host = self.node.hostname,
 370                     port = None,
 371                     user = self.node.slicename,
 372                     agent = None,
 373                     ident_key = self.node.ident_path,
 374                     server_key = self.node.server_key
 375                     )
 376
 377                 if status is not rspawn.RUNNING:
 378                     self._build_pid = self._build_ppid = None
 379                     break
 380                 else:
 381                     if first:
 382                         self._logger.info("Waiting for %s to finish building %s", self,
 383                             "(build slave)" if self._master is not None else "(build master)")
 384
 385                         first = False
 386                     time.sleep(delay*(0.5+random.random()))
 387                     delay = min(30,delay*1.2)
 388
 389             # check build token
 390             (out, err), proc = self._popen_ssh_command(
 391                 "cat %(token_path)s" % {
 392                     'token_path' : os.path.join(self.home_path, 'build.token'),
 393                 },
 394                 noerrors = True)
 395             slave_token = ""
 396             if not proc.wait() and out:
 397                 slave_token = out.strip()
 398
 399             if slave_token != self._master_token:
 400                 # Get buildlog for the error message
 401
 402                 (buildlog, err), proc = self._popen_ssh_command(
 403                     "cat %(buildlog)s" % {
 404                         'buildlog' : os.path.join(self.home_path, 'buildlog'),
 405                         'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
 406                     },
 407                     noerrors = True)
 408
 409                 proc.wait()
 410
 411                 raise RuntimeError, "Failed to set up application %s: "\
 412                         "build failed, got wrong token from pid %s/%s "\
 413                         "(expected %r, got %r), see buildlog: %s" % (
 414                     self.home_path, pid, ppid, self._master_token, slave_token, buildlog)
 415
 416             self._logger.info("Built %s", self)
 417
 418     def _do_kill_build(self):
 419         pid = self._build_pid
 420         ppid = self._build_ppid
 421
 422         if pid and ppid:
 423             self._logger.info("Killing build of %s", self)
 424             rspawn.remote_kill(
 425                 pid, ppid,
 426                 host = self.node.hostname,
 427                 port = None,
 428                 user = self.node.slicename,
 429                 agent = None,
 430                 ident_key = self.node.ident_path
 431                 )
 432
 433
 434     def _do_build_master(self):
 435         if not self.sources and not self.build and not self.buildDepends:
 436             return None
 437
 438         if self.sources:
 439             sources = self.sources.split(' ')
 440
 441             # Copy all sources
 442             try:
 443                 self._popen_scp(
 444                     sources,
 445                     "%s@%s:%s" % (self.node.slicename, self.node.hostname,
 446                         os.path.join(self.home_path,'.'),)
 447                     )
 448             except RuntimeError, e:
 449                 raise RuntimeError, "Failed upload source file %r: %s %s" \
 450                         % (sources, e.args[0], e.args[1],)
 451
 452         buildscript = cStringIO.StringIO()
 453
 454         if self.buildDepends:
 455             # Install build dependencies
 456             buildscript.write(
 457                 "sudo -S yum -y install %(packages)s\n" % {
 458                     'packages' : self.buildDepends
 459                 }
 460             )
 461
 462
 463         if self.build:
 464             # Build sources
 465             buildscript.write(
 466                 "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
 467                     'command' : self._replace_paths(self.build),
 468                     'home' : server.shell_escape(self.home_path),
 469                 }
 470             )
 471
 472             # Make archive
 473             buildscript.write("tar czf build.tar.gz build\n")
 474
 475         # Write token
 476         buildscript.write("echo %(master_token)s > build.token" % {
 477             'master_token' : server.shell_escape(self._master_token)
 478         })
 479
 480         buildscript.seek(0)
 481
 482         return buildscript
 483
 484     def _do_install(self):
 485         if self.install:
 486             self._logger.info("Installing %s", self)
 487
 488             # Install application
 489             try:
 490                 self._popen_ssh_command(
 491                     "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/{install,build}log >&2 && false )" % \
 492                         {
 493                         'command' : self._replace_paths(self.install),
 494                         'home' : server.shell_escape(self.home_path),
 495                         },
 496                     )
 497             except RuntimeError, e:
 498                 raise RuntimeError, "Failed install build sources: %s %s" % (e.args[0], e.args[1],)
 499
 500     def set_master(self, master):
 501         self._master = master
 502
 503     def install_keys(self, prk, puk, passphrase):
 504         # Install keys
 505         self._master_passphrase = passphrase
 506         self._master_prk = prk
 507         self._master_puk = puk
 508         self._master_prk_name = os.path.basename(prk.name)
 509         self._master_puk_name = os.path.basename(puk.name)
 510
 511     def _do_install_keys(self):
 512         prk = self._master_prk
 513         puk = self._master_puk
 514
 515         try:
 516             self._popen_scp(
 517                 [ prk.name, puk.name ],
 518                 '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path )
 519                 )
 520         except RuntimeError, e:
 521             raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
 522                     % (e.args[0], e.args[1],)
 523
 524         try:
 525             self._popen_scp(
 526                 cStringIO.StringIO('%s,%s %s\n' % (
 527                     self._master.node.hostname, socket.gethostbyname(self._master.node.hostname),
 528                     self._master.node.server_key)),
 529                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 530                     os.path.join(self.home_path,"master_known_hosts") )
 531                 )
 532         except RuntimeError, e:
 533             raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
 534                     % (e.args[0], e.args[1],)
 535
 536         # No longer need'em
 537         self._master_prk = None
 538         self._master_puk = None
 539
 540     def cleanup(self):
 541         # make sure there's no leftover build processes
 542         self._do_kill_build()
 543
 544     @server.eintr_retry
 545     def _popen_scp(self, src, dst, retry = True):
 546         (out,err),proc = server.popen_scp(
 547             src,
 548             dst,
 549             port = None,
 550             agent = None,
 551             ident_key = self.node.ident_path,
 552             server_key = self.node.server_key
 553             )
 554
 555         if server.eintr_retry(proc.wait)():
 556             raise RuntimeError, (out, err)
 557         return (out, err), proc
 558
 559
 560     @server.eintr_retry
 561     def _popen_ssh_command(self, command, retry = True, noerrors=False):
 562         (out,err),proc = server.popen_ssh_command(
 563             command,
 564             host = self.node.hostname,
 565             port = None,
 566             user = self.node.slicename,
 567             agent = None,
 568             ident_key = self.node.ident_path,
 569             server_key = self.node.server_key
 570             )
 571
 572         if server.eintr_retry(proc.wait)():
 573             if not noerrors:
 574                 raise RuntimeError, (out, err)
 575         return (out, err), proc
 576
 577 class Application(Dependency):
 578     """
 579     An application also has dependencies, but also a command to be ran and monitored.
 580
 581     It adds the output of that command as traces.
 582     """
 583
 584     TRACES = ('stdout','stderr','buildlog')
 585
 586     def __init__(self, api=None):
 587         super(Application,self).__init__(api)
 588
 589         # Attributes
 590         self.command = None
 591         self.sudo = False
 592
 593         self.stdin = None
 594         self.stdout = None
 595         self.stderr = None
 596
 597         # Those are filled when the app is started
 598         #   Having both pid and ppid makes it harder
 599         #   for pid rollover to induce tracking mistakes
 600         self._started = False
 601         self._pid = None
 602         self._ppid = None
 603
 604         # Do not add to the python path of nodes
 605         self.add_to_path = False
 606
 607     def __str__(self):
 608         return "%s<command:%s%s>" % (
 609             self.__class__.__name__,
 610             "sudo " if self.sudo else "",
 611             self.command,
 612         )
 613
 614     def start(self):
 615         self._logger.info("Starting %s", self)
 616
 617         # Create shell script with the command
 618         # This way, complex commands and scripts can be ran seamlessly
 619         # sync files
 620         command = cStringIO.StringIO()
 621         command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
 622             ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
 623         ))
 624         command.write('export PATH=$PATH:%s\n' % (
 625             ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
 626         ))
 627         if self.node.env:
 628             for envkey, envvals in self.node.env.iteritems():
 629                 for envval in envvals:
 630                     command.write('export %s=%s\n' % (envkey, envval))
 631         command.write(self.command)
 632         command.seek(0)
 633
 634         try:
 635             self._popen_scp(
 636                 command,
 637                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 638                     os.path.join(self.home_path, "app.sh"))
 639                 )
 640         except RuntimeError, e:
 641             raise RuntimeError, "Failed to set up application: %s %s" \
 642                     % (e.args[0], e.args[1],)
 643
 644         # Start process in a "daemonized" way, using nohup and heavy
 645         # stdin/out redirection to avoid connection issues
 646         (out,err),proc = rspawn.remote_spawn(
 647             self._replace_paths("bash ./app.sh"),
 648
 649             pidfile = './pid',
 650             home = self.home_path,
 651             stdin = 'stdin' if self.stdin is not None else '/dev/null',
 652             stdout = 'stdout' if self.stdout else '/dev/null',
 653             stderr = 'stderr' if self.stderr else '/dev/null',
 654             sudo = self.sudo,
 655
 656             host = self.node.hostname,
 657             port = None,
 658             user = self.node.slicename,
 659             agent = None,
 660             ident_key = self.node.ident_path,
 661             server_key = self.node.server_key
 662             )
 663
 664         if proc.wait():
 665             raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
 666
 667         self._started = True
 668
 669     def recover(self):
 670         # Assuming the application is running on PlanetLab,
 671         # proper pidfiles should be present at the app's home path.
 672         # So we mark this application as started, and check the pidfiles
 673         self._started = True
 674         self.checkpid()
 675
 676     def checkpid(self):
 677         # Get PID/PPID
 678         # NOTE: wait a bit for the pidfile to be created
 679         if self._started and not self._pid or not self._ppid:
 680             pidtuple = rspawn.remote_check_pid(
 681                 os.path.join(self.home_path,'pid'),
 682                 host = self.node.hostname,
 683                 port = None,
 684                 user = self.node.slicename,
 685                 agent = None,
 686                 ident_key = self.node.ident_path,
 687                 server_key = self.node.server_key
 688                 )
 689
 690             if pidtuple:
 691                 self._pid, self._ppid = pidtuple
 692
 693     def status(self):
 694         self.checkpid()
 695         if not self._started:
 696             return AS.STATUS_NOT_STARTED
 697         elif not self._pid or not self._ppid:
 698             return AS.STATUS_NOT_STARTED
 699         else:
 700             status = rspawn.remote_status(
 701                 self._pid, self._ppid,
 702                 host = self.node.hostname,
 703                 port = None,
 704                 user = self.node.slicename,
 705                 agent = None,
 706                 ident_key = self.node.ident_path,
 707                 server_key = self.node.server_key
 708                 )
 709
 710             if status is rspawn.NOT_STARTED:
 711                 return AS.STATUS_NOT_STARTED
 712             elif status is rspawn.RUNNING:
 713                 return AS.STATUS_RUNNING
 714             elif status is rspawn.FINISHED:
 715                 return AS.STATUS_FINISHED
 716             else:
 717                 # WTF?
 718                 return AS.STATUS_NOT_STARTED
 719
 720     def kill(self):
 721         status = self.status()
 722         if status == AS.STATUS_RUNNING:
 723             # kill by ppid+pid - SIGTERM first, then try SIGKILL
 724             rspawn.remote_kill(
 725                 self._pid, self._ppid,
 726                 host = self.node.hostname,
 727                 port = None,
 728                 user = self.node.slicename,
 729                 agent = None,
 730                 ident_key = self.node.ident_path,
 731                 server_key = self.node.server_key,
 732                 sudo = self.sudo
 733                 )
 734             self._logger.info("Killed %s", self)
 735
 736
 737 class NepiDependency(Dependency):
 738     """
 739     This dependency adds nepi itself to the python path,
 740     so that you may run testbeds within PL nodes.
 741     """
 742
 743     # Class attribute holding a *weak* reference to the shared NEPI tar file
 744     # so that they may share it. Don't operate on the file itself, it would
 745     # be a mess, just use its path.
 746     _shared_nepi_tar = None
 747
 748     def __init__(self, api = None):
 749         super(NepiDependency, self).__init__(api)
 750
 751         self._tarball = None
 752
 753         self.depends = 'python python-ipaddr python-setuptools'
 754
 755         # our sources are in our ad-hoc tarball
 756         self.sources = self.tarball.name
 757
 758         tarname = os.path.basename(self.tarball.name)
 759
 760         # it's already built - just move the tarball into place
 761         self.build = "mv -f ${SOURCES}/%s ." % (tarname,)
 762
 763         # unpack it into sources, and we're done
 764         self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,)
 765
 766     @property
 767     def tarball(self):
 768         if self._tarball is None:
 769             shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
 770             if shared_tar is not None:
 771                 self._tarball = shared_tar
 772             else:
 773                 # Build an ad-hoc tarball
 774                 # Prebuilt
 775                 import nepi
 776                 import tempfile
 777
 778                 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
 779
 780                 proc = subprocess.Popen(
 781                     ["tar", "czf", shared_tar.name,
 782                         '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
 783                         'nepi'],
 784                     stdout = open("/dev/null","w"),
 785                     stdin = open("/dev/null","r"))
 786
 787                 if proc.wait():
 788                     raise RuntimeError, "Failed to create nepi tarball"
 789
 790                 self._tarball = self._shared_nepi_tar = shared_tar
 791
 792         return self._tarball
 793
 794 class NS3Dependency(Dependency):
 795     """
 796     This dependency adds NS3 libraries to the library paths,
 797     so that you may run the NS3 testbed within PL nodes.
 798
 799     You'll also need the NepiDependency.
 800     """
 801
 802     def __init__(self, api = None):
 803         super(NS3Dependency, self).__init__(api)
 804
 805         self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip'
 806
 807         # We have to download the sources, untar, build...
 808         pybindgen_source_url = "http://yans.pl.sophia.inria.fr/trac/nepi/raw-attachment/wiki/WikiStart/pybindgen-r794.tar.gz"
 809         pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
 810         ns3_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/ns-3.11-nepi/archive/tip.tar.gz"
 811         passfd_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/python-passfd/archive/tip.tar.gz"
 812         self.build =(
 813             " ( "
 814             "  cd .. && "
 815             "  python -c 'import pygccxml, pybindgen, passfd' && "
 816             "  test -f lib/ns/_core.so && "
 817             "  test -f lib/ns/__init__.py && "
 818             "  test -f lib/ns/core.py && "
 819             "  test -f lib/libns3-core.so && "
 820             "  LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
 821             " ) || ( "
 822                 # Not working, rebuild
 823                      # Archive SHA1 sums to check
 824                      "echo '7158877faff2254e6c094bf18e6b4283cac19137  pygccxml-1.0.0.zip' > archive_sums.txt && "
 825                      "echo 'a18c2ccffd0df517bc37e2f3a2475092517c43f2  pybindgen-src.tar.gz' >> archive_sums.txt && "
 826                      " ( " # check existing files
 827                      " sha1sum -c archive_sums.txt && "
 828                      " test -f passfd-src.tar.gz && "
 829                      " test -f ns3-src.tar.gz "
 830                      " ) || ( " # nope? re-download
 831                      " rm -f pybindgen-src.zip pygccxml-1.0.0.zip passfd-src.tar.gz ns3-src.tar.gz && "
 832                      " wget -q -c -O pybindgen-src.tar.gz %(pybindgen_source_url)s && " # continue, to exploit the case when it has already been dl'ed
 833                      " wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
 834                      " wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
 835                      " wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
 836                      " sha1sum -c archive_sums.txt " # Check SHA1 sums when applicable
 837                      " ) && "
 838                      "unzip -n pygccxml-1.0.0.zip && "
 839                      "mkdir -p pybindgen-src && "
 840                      "mkdir -p ns3-src && "
 841                      "mkdir -p passfd-src && "
 842                      "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
 843                      "tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
 844                      "tar xzf pybindgen-src.tar.gz --strip-components=1 -C pybindgen-src && "
 845                      "rm -rf target && "    # mv doesn't like unclean targets
 846                      "mkdir -p target && "
 847                      "cd pygccxml-1.0.0 && "
 848                      "rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
 849                      "python setup.py build && "
 850                      "python setup.py install --install-lib ${BUILD}/target && "
 851                      "python setup.py clean && "
 852                      "cd ../pybindgen-src && "
 853                      "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
 854                      "./waf configure --prefix=${BUILD}/target -d release && "
 855                      "./waf && "
 856                      "./waf install && "
 857                      "./waf clean && "
 858                      "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
 859                      "rm -rf ${BUILD}/target/lib && "
 860                      "cd ../passfd-src && "
 861                      "python setup.py build && "
 862                      "python setup.py install --install-lib ${BUILD}/target && "
 863                      "python setup.py clean && "
 864                      "cd ../ns3-src && "
 865                      "./waf configure --prefix=${BUILD}/target --with-pybindgen=../pybindgen-src -d release --disable-examples --disable-tests --enable-threading && "
 866                      "./waf &&"
 867                      "./waf install && "
 868                      "rm -f ${BUILD}/target/lib/*.so && "
 869                      "cp -a ${BUILD}/ns3-src/build/release/libns3*.so ${BUILD}/target/lib && "
 870                      "cp -a ${BUILD}/ns3-src/build/release/bindings/python/ns ${BUILD}/target/lib &&"
 871                      "./waf clean "
 872              " )"
 873                      % dict(
 874                         pybindgen_source_url = server.shell_escape(pybindgen_source_url),
 875                         pygccxml_source_url = server.shell_escape(pygccxml_source_url),
 876                         ns3_source_url = server.shell_escape(ns3_source_url),
 877                         passfd_source_url = server.shell_escape(passfd_source_url),
 878                      ))
 879
 880         # Just move ${BUILD}/target
 881         self.install = (
 882             " ( "
 883             "  cd .. && "
 884             "  python -c 'import pygccxml, pybindgen, passfd' && "
 885             "  test -f lib/ns/_core.so && "
 886             "  test -f lib/ns/__init__.py && "
 887             "  test -f lib/ns/core.py && "
 888             "  test -f lib/libns3-core.so && "
 889             "  LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
 890             " ) || ( "
 891                 # Not working, reinstall
 892                     "test -d ${BUILD}/target && "
 893                     "[[ \"x\" != \"x$(find ${BUILD}/target -mindepth 1 -print -quit)\" ]] &&"
 894                     "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
 895                     "mv -f ${BUILD}/target/* ${SOURCES}"
 896             " )"
 897         )
 898
 899         # Set extra environment paths
 900         self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
 901         self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib"
 902
 903     @property
 904     def tarball(self):
 905         if self._tarball is None:
 906             shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
 907             if shared_tar is not None:
 908                 self._tarball = shared_tar
 909             else:
 910                 # Build an ad-hoc tarball
 911                 # Prebuilt
 912                 import nepi
 913                 import tempfile
 914
 915                 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
 916
 917                 proc = subprocess.Popen(
 918                     ["tar", "czf", shared_tar.name,
 919                         '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
 920                         'nepi'],
 921                     stdout = open("/dev/null","w"),
 922                     stdin = open("/dev/null","r"))
 923
 924                 if proc.wait():
 925                     raise RuntimeError, "Failed to create nepi tarball"
 926
 927                 self._tarball = self._shared_nepi_tar = shared_tar
 928
 929         return self._tarball
 930
 931 class YumDependency(Dependency):
 932     """
 933     This dependency is an internal helper class used to
 934     efficiently distribute yum-downloaded rpms.
 935
 936     It temporarily sets the yum cache as persistent in the
 937     build master, and installs all the required packages.
 938
 939     The rpm packages left in the yum cache are gathered and
 940     distributed by the underlying Dependency in an efficient
 941     manner. Build slaves will then install those rpms back in
 942     the cache before issuing the install command.
 943
 944     When packages have been installed already, nothing but an
 945     empty tar is distributed.
 946     """
 947
 948     # Class attribute holding a *weak* reference to the shared NEPI tar file
 949     # so that they may share it. Don't operate on the file itself, it would
 950     # be a mess, just use its path.
 951     _shared_nepi_tar = None
 952
 953     def _build_get(self):
 954         # canonical representation of dependencies
 955         depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
 956
 957         # download rpms and pack into a tar archive
 958         return (
 959             "sudo -S nice yum -y makecache && "
 960             "sudo -S sed -i -r 's/keepcache *= *0/keepcache=1/' /etc/yum.conf && "
 961             " ( ( "
 962                 "sudo -S nice yum -y install %s ; "
 963                 "rm -f ${BUILD}/packages.tar ; "
 964                 "tar -C /var/cache/yum -rf ${BUILD}/packages.tar $(cd /var/cache/yum ; find -iname '*.rpm')"
 965             " ) || /bin/true ) && "
 966             "sudo -S sed -i -r 's/keepcache *= *1/keepcache=0/' /etc/yum.conf && "
 967             "sudo -S nice yum -y clean packages "
 968         ) % ( depends, )
 969     def _build_set(self, value):
 970         # ignore
 971         return
 972     build = property(_build_get, _build_set)
 973
 974     def _install_get(self):
 975         # canonical representation of dependencies
 976         depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
 977
 978         # unpack cached rpms into yum cache, install, and cleanup
 979         return (
 980             "sudo -S tar -k --keep-newer-files -C /var/cache/yum -xf packages.tar && "
 981             "sudo -S nice yum -y install %s && "
 982             "sudo -S nice yum -y clean packages "
 983         ) % ( depends, )
 984     def _install_set(self, value):
 985         # ignore
 986         return
 987     install = property(_install_get, _install_set)
 988
 989