src/nepi/testbeds/planetlab/application.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from constants import TESTBED_ID
   5 import plcapi
   6 import operator
   7 import os
   8 import os.path
   9 import sys
  10 import nepi.util.server as server
  11 import cStringIO
  12 import subprocess
  13 import rspawn
  14 import random
  15 import time
  16 import socket
  17 import threading
  18 import logging
  19
  20 from nepi.util.constants import ApplicationStatus as AS
  21
  22 class Dependency(object):
  23     """
  24     A Dependency is in every respect like an application.
  25
  26     It depends on some packages, it may require building binaries, it must deploy
  27     them...
  28
  29     But it has no command. Dependencies aren't ever started, or stopped, and have
  30     no status.
  31     """
  32
  33     TRACES = ('buildlog')
  34
  35     def __init__(self, api=None):
  36         if not api:
  37             api = plcapi.PLCAPI()
  38         self._api = api
  39
  40         # Attributes
  41         self.command = None
  42         self.sudo = False
  43
  44         self.build = None
  45         self.install = None
  46         self.depends = None
  47         self.buildDepends = None
  48         self.sources = None
  49         self.rpmFusion = False
  50         self.env = {}
  51
  52         self.stdin = None
  53         self.stdout = None
  54         self.stderr = None
  55         self.buildlog = None
  56
  57         self.add_to_path = True
  58
  59         # Those are filled when the app is configured
  60         self.home_path = None
  61
  62         # Those are filled when an actual node is connected
  63         self.node = None
  64
  65         # Those are filled when the app is started
  66         #   Having both pid and ppid makes it harder
  67         #   for pid rollover to induce tracking mistakes
  68         self._started = False
  69         self._setup = False
  70         self._setuper = None
  71         self._pid = None
  72         self._ppid = None
  73
  74         # Spanning tree deployment
  75         self._master = None
  76         self._master_passphrase = None
  77         self._master_prk = None
  78         self._master_puk = None
  79         self._master_token = ''.join(map(chr,[rng.randint(0,255)
  80                                       for rng in (random.SystemRandom(),)
  81                                       for i in xrange(8)] )).encode("hex")
  82         self._build_pid = None
  83         self._build_ppid = None
  84
  85         # Logging
  86         self._logger = logging.getLogger('nepi.testbeds.planetlab')
  87
  88
  89     def __str__(self):
  90         return "%s<%s>" % (
  91             self.__class__.__name__,
  92             ' '.join(filter(bool,(self.depends, self.sources)))
  93         )
  94
  95     def validate(self):
  96         if self.home_path is None:
  97             raise AssertionError, "Misconfigured application: missing home path"
  98         if self.node.ident_path is None or not os.access(self.node.ident_path, os.R_OK):
  99             raise AssertionError, "Misconfigured application: missing slice SSH key"
 100         if self.node is None:
 101             raise AssertionError, "Misconfigured application: unconnected node"
 102         if self.node.hostname is None:
 103             raise AssertionError, "Misconfigured application: misconfigured node"
 104         if self.node.slicename is None:
 105             raise AssertionError, "Misconfigured application: unspecified slice"
 106
 107     def remote_trace_path(self, whichtrace):
 108         if whichtrace in self.TRACES:
 109             tracefile = os.path.join(self.home_path, whichtrace)
 110         else:
 111             tracefile = None
 112
 113         return tracefile
 114
 115     def remote_trace_name(self, whichtrace):
 116         if whichtrace in self.TRACES:
 117             return whichtrace
 118         return None
 119
 120     def sync_trace(self, local_dir, whichtrace):
 121         tracefile = self.remote_trace_path(whichtrace)
 122         if not tracefile:
 123             return None
 124
 125         local_path = os.path.join(local_dir, tracefile)
 126
 127         # create parent local folders
 128         proc = subprocess.Popen(
 129             ["mkdir", "-p", os.path.dirname(local_path)],
 130             stdout = open("/dev/null","w"),
 131             stdin = open("/dev/null","r"))
 132
 133         if proc.wait():
 134             raise RuntimeError, "Failed to synchronize trace"
 135
 136         # sync files
 137         try:
 138             self._popen_scp(
 139                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 140                     tracefile),
 141                 local_path
 142                 )
 143         except RuntimeError, e:
 144             raise RuntimeError, "Failed to synchronize trace: %s %s" \
 145                     % (e.args[0], e.args[1],)
 146
 147         return local_path
 148
 149     def recover(self):
 150         # We assume a correct deployment, so recovery only
 151         # means we mark this dependency as deployed
 152         self._setup = True
 153
 154     def setup(self):
 155         self._logger.info("Setting up %s", self)
 156         self._make_home()
 157         self._launch_build()
 158         self._finish_build()
 159         self._setup = True
 160
 161     def async_setup(self):
 162         if not self._setuper:
 163             def setuper():
 164                 try:
 165                     self.setup()
 166                 except:
 167                     self._setuper._exc.append(sys.exc_info())
 168             self._setuper = threading.Thread(
 169                 target = setuper)
 170             self._setuper._exc = []
 171             self._setuper.start()
 172
 173     def async_setup_wait(self):
 174         if not self._setup:
 175             self._logger.info("Waiting for %s to be setup", self)
 176             if self._setuper:
 177                 self._setuper.join()
 178                 if not self._setup:
 179                     if self._setuper._exc:
 180                         exctyp,exval,exctrace = self._setuper._exc[0]
 181                         raise exctyp,exval,exctrace
 182                     else:
 183                         raise RuntimeError, "Failed to setup application"
 184                 else:
 185                     self._logger.info("Setup ready: %s", self)
 186             else:
 187                 self.setup()
 188
 189     def _make_home(self):
 190         # Make sure all the paths are created where
 191         # they have to be created for deployment
 192         # sync files
 193         try:
 194             self._popen_ssh_command(
 195                 "mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" \
 196                     % { 'home' : server.shell_escape(self.home_path) },
 197                 timeout = 120,
 198                 retry = 3
 199                 )
 200         except RuntimeError, e:
 201             raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, e.args[0], e.args[1],)
 202
 203         if self.stdin:
 204             # Write program input
 205             try:
 206                 self._popen_scp(
 207                     cStringIO.StringIO(self.stdin),
 208                     '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 209                         os.path.join(self.home_path, 'stdin') ),
 210                     )
 211             except RuntimeError, e:
 212                 raise RuntimeError, "Failed to set up application %s: %s %s" \
 213                         % (self.home_path, e.args[0], e.args[1],)
 214
 215     def _replace_paths(self, command):
 216         """
 217         Replace all special path tags with shell-escaped actual paths.
 218         """
 219         # need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
 220         root = '' if self.home_path.startswith('/') else "${HOME}/"
 221         return ( command
 222             .replace("${SOURCES}", root+server.shell_escape(self.home_path))
 223             .replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
 224
 225     def _launch_build(self):
 226         if self._master is not None:
 227             self._do_install_keys()
 228             buildscript = self._do_build_slave()
 229         else:
 230             buildscript = self._do_build_master()
 231
 232         if buildscript is not None:
 233             self._logger.info("Building %s", self)
 234
 235             # upload build script
 236             try:
 237                 self._popen_scp(
 238                     buildscript,
 239                     '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 240                         os.path.join(self.home_path, 'nepi-build.sh') )
 241                     )
 242             except RuntimeError, e:
 243                 raise RuntimeError, "Failed to set up application %s: %s %s" \
 244                         % (self.home_path, e.args[0], e.args[1],)
 245
 246             # launch build
 247             self._do_launch_build()
 248
 249     def _finish_build(self):
 250         self._do_wait_build()
 251         self._do_install()
 252
 253     def _do_build_slave(self):
 254         if not self.sources and not self.build:
 255             return None
 256
 257         # Create build script
 258         files = set()
 259
 260         if self.sources:
 261             sources = self.sources.split(' ')
 262             files.update(
 263                 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
 264                     os.path.join(self._master.home_path, os.path.basename(source)),)
 265                 for source in sources
 266             )
 267
 268         if self.build:
 269             files.add(
 270                 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
 271                     os.path.join(self._master.home_path, 'build.tar.gz'),)
 272             )
 273
 274         launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
 275                         " && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
 276                         " && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" %  \
 277         {
 278             'prk' : server.shell_escape(self._master_prk_name),
 279             'puk' : server.shell_escape(self._master_puk_name),
 280         }
 281
 282         kill_agent = "kill $SSH_AGENT_PID"
 283
 284         waitmaster = "{ . ./.ssh-agent.sh ; while [[ $(ssh -q -o UserKnownHostsFile=%(hostkey)s %(master)s cat %(token_path)s) != %(token)s ]] ; do sleep 5 ; done ; }" % {
 285             'hostkey' : 'master_known_hosts',
 286             'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostname),
 287             'token_path' : os.path.join(self._master.home_path, 'build.token'),
 288             'token' : server.shell_escape(self._master._master_token),
 289         }
 290
 291         syncfiles = "scp -p -o UserKnownHostsFile=%(hostkey)s %(files)s ." % {
 292             'hostkey' : 'master_known_hosts',
 293             'files' : ' '.join(files),
 294         }
 295         if self.build:
 296             syncfiles += " && tar xzf build.tar.gz"
 297         syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
 298         syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
 299
 300         cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
 301             'prk' : server.shell_escape(self._master_prk_name),
 302             'puk' : server.shell_escape(self._master_puk_name),
 303         }
 304
 305         slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s )" % {
 306             'waitmaster' : waitmaster,
 307             'syncfiles' : syncfiles,
 308             'cleanup' : cleanup,
 309             'kill_agent' : kill_agent,
 310             'launch_agent' : launch_agent,
 311             'home' : server.shell_escape(self.home_path),
 312         }
 313
 314         return cStringIO.StringIO(slavescript)
 315
 316     def _do_launch_build(self):
 317         script = "bash ./nepi-build.sh"
 318         if self._master_passphrase:
 319             script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
 320                 server.shell_escape(self._master_passphrase),
 321                 script
 322             )
 323         (out,err),proc = rspawn.remote_spawn(
 324             script,
 325             pidfile = 'build-pid',
 326             home = self.home_path,
 327             stdin = '/dev/null',
 328             stdout = 'buildlog',
 329             stderr = rspawn.STDOUT,
 330
 331             host = self.node.hostname,
 332             port = None,
 333             user = self.node.slicename,
 334             agent = None,
 335             ident_key = self.node.ident_path,
 336             server_key = self.node.server_key
 337             )
 338
 339         if proc.wait():
 340             raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
 341
 342
 343         pid = ppid = None
 344         delay = 1.0
 345         for i in xrange(5):
 346             pidtuple = rspawn.remote_check_pid(
 347                 os.path.join(self.home_path,'build-pid'),
 348                 host = self.node.hostname,
 349                 port = None,
 350                 user = self.node.slicename,
 351                 agent = None,
 352                 ident_key = self.node.ident_path,
 353                 server_key = self.node.server_key
 354                 )
 355
 356             if pidtuple:
 357                 pid, ppid = pidtuple
 358                 self._build_pid, self._build_ppid = pidtuple
 359                 break
 360             else:
 361                 time.sleep(delay)
 362                 delay = min(30,delay*1.2)
 363         else:
 364             raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
 365
 366         self._logger.info("Deploying %s", self)
 367
 368     def _do_wait_build(self):
 369         pid = self._build_pid
 370         ppid = self._build_ppid
 371
 372         if pid and ppid:
 373             delay = 1.0
 374             first = True
 375             while True:
 376                 status = rspawn.remote_status(
 377                     pid, ppid,
 378                     host = self.node.hostname,
 379                     port = None,
 380                     user = self.node.slicename,
 381                     agent = None,
 382                     ident_key = self.node.ident_path,
 383                     server_key = self.node.server_key
 384                     )
 385
 386                 if status is not rspawn.RUNNING:
 387                     self._build_pid = self._build_ppid = None
 388                     break
 389                 else:
 390                     if first:
 391                         self._logger.info("Waiting for %s to finish building %s", self,
 392                             "(build slave)" if self._master is not None else "(build master)")
 393
 394                         first = False
 395                     time.sleep(delay*(0.5+random.random()))
 396                     delay = min(30,delay*1.2)
 397
 398             # check build token
 399             (out, err), proc = self._popen_ssh_command(
 400                 "cat %(token_path)s" % {
 401                     'token_path' : os.path.join(self.home_path, 'build.token'),
 402                 },
 403                 timeout = 120,
 404                 noerrors = True)
 405             slave_token = ""
 406             if not proc.wait() and out:
 407                 slave_token = out.strip()
 408
 409             if slave_token != self._master_token:
 410                 # Get buildlog for the error message
 411
 412                 (buildlog, err), proc = self._popen_ssh_command(
 413                     "cat %(buildlog)s" % {
 414                         'buildlog' : os.path.join(self.home_path, 'buildlog'),
 415                         'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
 416                     },
 417                     timeout = 120,
 418                     noerrors = True)
 419
 420                 proc.wait()
 421
 422                 raise RuntimeError, "Failed to set up application %s: "\
 423                         "build failed, got wrong token from pid %s/%s "\
 424                         "(expected %r, got %r), see buildlog: %s" % (
 425                     self.home_path, pid, ppid, self._master_token, slave_token, buildlog)
 426
 427             self._logger.info("Built %s", self)
 428
 429     def _do_kill_build(self):
 430         pid = self._build_pid
 431         ppid = self._build_ppid
 432
 433         if pid and ppid:
 434             self._logger.info("Killing build of %s", self)
 435             rspawn.remote_kill(
 436                 pid, ppid,
 437                 host = self.node.hostname,
 438                 port = None,
 439                 user = self.node.slicename,
 440                 agent = None,
 441                 ident_key = self.node.ident_path
 442                 )
 443
 444
 445     def _do_build_master(self):
 446         if not self.sources and not self.build and not self.buildDepends:
 447             return None
 448
 449         if self.sources:
 450             sources = self.sources.split(' ')
 451
 452             # Copy all sources
 453             try:
 454                 self._popen_scp(
 455                     sources,
 456                     "%s@%s:%s" % (self.node.slicename, self.node.hostname,
 457                         os.path.join(self.home_path,'.'),)
 458                     )
 459             except RuntimeError, e:
 460                 raise RuntimeError, "Failed upload source file %r: %s %s" \
 461                         % (sources, e.args[0], e.args[1],)
 462
 463         buildscript = cStringIO.StringIO()
 464
 465         if self.buildDepends:
 466             # Install build dependencies
 467             buildscript.write(
 468                 "sudo -S yum -y install %(packages)s\n" % {
 469                     'packages' : self.buildDepends
 470                 }
 471             )
 472
 473
 474         if self.build:
 475             # Build sources
 476             buildscript.write(
 477                 "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
 478                     'command' : self._replace_paths(self.build),
 479                     'home' : server.shell_escape(self.home_path),
 480                 }
 481             )
 482
 483             # Make archive
 484             buildscript.write("tar czf build.tar.gz build\n")
 485
 486         # Write token
 487         buildscript.write("echo %(master_token)s > build.token" % {
 488             'master_token' : server.shell_escape(self._master_token)
 489         })
 490
 491         buildscript.seek(0)
 492
 493         return buildscript
 494
 495     def _do_install(self):
 496         if self.install:
 497             self._logger.info("Installing %s", self)
 498
 499             # Install application
 500             try:
 501                 self._popen_ssh_command(
 502                     "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/{install,build}log >&2 && false )" % \
 503                         {
 504                         'command' : self._replace_paths(self.install),
 505                         'home' : server.shell_escape(self.home_path),
 506                         },
 507                     )
 508             except RuntimeError, e:
 509                 raise RuntimeError, "Failed install build sources: %s %s" % (e.args[0], e.args[1],)
 510
 511     def set_master(self, master):
 512         self._master = master
 513
 514     def install_keys(self, prk, puk, passphrase):
 515         # Install keys
 516         self._master_passphrase = passphrase
 517         self._master_prk = prk
 518         self._master_puk = puk
 519         self._master_prk_name = os.path.basename(prk.name)
 520         self._master_puk_name = os.path.basename(puk.name)
 521
 522     def _do_install_keys(self):
 523         prk = self._master_prk
 524         puk = self._master_puk
 525
 526         try:
 527             self._popen_scp(
 528                 [ prk.name, puk.name ],
 529                 '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path )
 530                 )
 531         except RuntimeError, e:
 532             raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
 533                     % (e.args[0], e.args[1],)
 534
 535         try:
 536             self._popen_scp(
 537                 cStringIO.StringIO('%s,%s %s\n' % (
 538                     self._master.node.hostname, socket.gethostbyname(self._master.node.hostname),
 539                     self._master.node.server_key)),
 540                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 541                     os.path.join(self.home_path,"master_known_hosts") )
 542                 )
 543         except RuntimeError, e:
 544             raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
 545                     % (e.args[0], e.args[1],)
 546
 547         # No longer need'em
 548         self._master_prk = None
 549         self._master_puk = None
 550
 551     def cleanup(self):
 552         # make sure there's no leftover build processes
 553         self._do_kill_build()
 554
 555     @server.eintr_retry
 556     def _popen_scp(self, src, dst, retry = 3):
 557         while 1:
 558             try:
 559                 (out,err),proc = server.popen_scp(
 560                     src,
 561                     dst,
 562                     port = None,
 563                     agent = None,
 564                     ident_key = self.node.ident_path,
 565                     server_key = self.node.server_key
 566                     )
 567
 568                 if server.eintr_retry(proc.wait)():
 569                     raise RuntimeError, (out, err)
 570                 return (out, err), proc
 571             except:
 572                 if retry <= 0:
 573                     raise
 574                 else:
 575                     retry -= 1
 576
 577
 578     @server.eintr_retry
 579     def _popen_ssh_command(self, command, retry = 0, noerrors=False, timeout=None):
 580         (out,err),proc = server.popen_ssh_command(
 581             command,
 582             host = self.node.hostname,
 583             port = None,
 584             user = self.node.slicename,
 585             agent = None,
 586             ident_key = self.node.ident_path,
 587             server_key = self.node.server_key,
 588             timeout = timeout,
 589             retry = retry
 590             )
 591
 592         if server.eintr_retry(proc.wait)():
 593             if not noerrors:
 594                 raise RuntimeError, (out, err)
 595         return (out, err), proc
 596
 597 class Application(Dependency):
 598     """
 599     An application also has dependencies, but also a command to be ran and monitored.
 600
 601     It adds the output of that command as traces.
 602     """
 603
 604     TRACES = ('stdout','stderr','buildlog', 'output')
 605
 606     def __init__(self, api=None):
 607         super(Application,self).__init__(api)
 608
 609         # Attributes
 610         self.command = None
 611         self.sudo = False
 612
 613         self.stdin = None
 614         self.stdout = None
 615         self.stderr = None
 616         self.output = None
 617
 618         # Those are filled when the app is started
 619         #   Having both pid and ppid makes it harder
 620         #   for pid rollover to induce tracking mistakes
 621         self._started = False
 622         self._pid = None
 623         self._ppid = None
 624
 625         # Do not add to the python path of nodes
 626         self.add_to_path = False
 627
 628     def __str__(self):
 629         return "%s<command:%s%s>" % (
 630             self.__class__.__name__,
 631             "sudo " if self.sudo else "",
 632             self.command,
 633         )
 634
 635     def start(self):
 636         self._logger.info("Starting %s", self)
 637
 638         # Create shell script with the command
 639         # This way, complex commands and scripts can be ran seamlessly
 640         # sync files
 641         command = cStringIO.StringIO()
 642         command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
 643             ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
 644         ))
 645         command.write('export PATH=$PATH:%s\n' % (
 646             ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
 647         ))
 648         if self.node.env:
 649             for envkey, envvals in self.node.env.iteritems():
 650                 for envval in envvals:
 651                     command.write('export %s=%s\n' % (envkey, envval))
 652         command.write(self.command)
 653         command.seek(0)
 654
 655         try:
 656             self._popen_scp(
 657                 command,
 658                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 659                     os.path.join(self.home_path, "app.sh"))
 660                 )
 661         except RuntimeError, e:
 662             raise RuntimeError, "Failed to set up application: %s %s" \
 663                     % (e.args[0], e.args[1],)
 664
 665         # Start process in a "daemonized" way, using nohup and heavy
 666         # stdin/out redirection to avoid connection issues
 667         (out,err),proc = rspawn.remote_spawn(
 668             self._replace_paths("bash ./app.sh"),
 669
 670             pidfile = './pid',
 671             home = self.home_path,
 672             stdin = 'stdin' if self.stdin is not None else '/dev/null',
 673             stdout = 'stdout' if self.stdout else '/dev/null',
 674             stderr = 'stderr' if self.stderr else '/dev/null',
 675             sudo = self.sudo,
 676
 677             host = self.node.hostname,
 678             port = None,
 679             user = self.node.slicename,
 680             agent = None,
 681             ident_key = self.node.ident_path,
 682             server_key = self.node.server_key
 683             )
 684
 685         if proc.wait():
 686             raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
 687
 688         self._started = True
 689
 690     def recover(self):
 691         # Assuming the application is running on PlanetLab,
 692         # proper pidfiles should be present at the app's home path.
 693         # So we mark this application as started, and check the pidfiles
 694         self._started = True
 695         self.checkpid()
 696
 697     def checkpid(self):
 698         # Get PID/PPID
 699         # NOTE: wait a bit for the pidfile to be created
 700         if self._started and not self._pid or not self._ppid:
 701             pidtuple = rspawn.remote_check_pid(
 702                 os.path.join(self.home_path,'pid'),
 703                 host = self.node.hostname,
 704                 port = None,
 705                 user = self.node.slicename,
 706                 agent = None,
 707                 ident_key = self.node.ident_path,
 708                 server_key = self.node.server_key
 709                 )
 710
 711             if pidtuple:
 712                 self._pid, self._ppid = pidtuple
 713
 714     def status(self):
 715         self.checkpid()
 716         if not self._started:
 717             return AS.STATUS_NOT_STARTED
 718         elif not self._pid or not self._ppid:
 719             return AS.STATUS_NOT_STARTED
 720         else:
 721             status = rspawn.remote_status(
 722                 self._pid, self._ppid,
 723                 host = self.node.hostname,
 724                 port = None,
 725                 user = self.node.slicename,
 726                 agent = None,
 727                 ident_key = self.node.ident_path,
 728                 server_key = self.node.server_key
 729                 )
 730
 731             if status is rspawn.NOT_STARTED:
 732                 return AS.STATUS_NOT_STARTED
 733             elif status is rspawn.RUNNING:
 734                 return AS.STATUS_RUNNING
 735             elif status is rspawn.FINISHED:
 736                 return AS.STATUS_FINISHED
 737             else:
 738                 # WTF?
 739                 return AS.STATUS_NOT_STARTED
 740
 741     def kill(self):
 742         status = self.status()
 743         if status == AS.STATUS_RUNNING:
 744             # kill by ppid+pid - SIGTERM first, then try SIGKILL
 745             rspawn.remote_kill(
 746                 self._pid, self._ppid,
 747                 host = self.node.hostname,
 748                 port = None,
 749                 user = self.node.slicename,
 750                 agent = None,
 751                 ident_key = self.node.ident_path,
 752                 server_key = self.node.server_key,
 753                 sudo = self.sudo
 754                 )
 755             self._logger.info("Killed %s", self)
 756
 757
 758 class NepiDependency(Dependency):
 759     """
 760     This dependency adds nepi itself to the python path,
 761     so that you may run testbeds within PL nodes.
 762     """
 763
 764     # Class attribute holding a *weak* reference to the shared NEPI tar file
 765     # so that they may share it. Don't operate on the file itself, it would
 766     # be a mess, just use its path.
 767     _shared_nepi_tar = None
 768
 769     def __init__(self, api = None):
 770         super(NepiDependency, self).__init__(api)
 771
 772         self._tarball = None
 773
 774         self.depends = 'python python-ipaddr python-setuptools'
 775
 776         # our sources are in our ad-hoc tarball
 777         self.sources = self.tarball.name
 778
 779         tarname = os.path.basename(self.tarball.name)
 780
 781         # it's already built - just move the tarball into place
 782         self.build = "mv -f ${SOURCES}/%s ." % (tarname,)
 783
 784         # unpack it into sources, and we're done
 785         self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,)
 786
 787     @property
 788     def tarball(self):
 789         if self._tarball is None:
 790             shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
 791             if shared_tar is not None:
 792                 self._tarball = shared_tar
 793             else:
 794                 # Build an ad-hoc tarball
 795                 # Prebuilt
 796                 import nepi
 797                 import tempfile
 798
 799                 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
 800
 801                 proc = subprocess.Popen(
 802                     ["tar", "czf", shared_tar.name,
 803                         '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
 804                         'nepi'],
 805                     stdout = open("/dev/null","w"),
 806                     stdin = open("/dev/null","r"))
 807
 808                 if proc.wait():
 809                     raise RuntimeError, "Failed to create nepi tarball"
 810
 811                 self._tarball = self._shared_nepi_tar = shared_tar
 812
 813         return self._tarball
 814
 815 class NS3Dependency(Dependency):
 816     """
 817     This dependency adds NS3 libraries to the library paths,
 818     so that you may run the NS3 testbed within PL nodes.
 819
 820     You'll also need the NepiDependency.
 821     """
 822
 823     def __init__(self, api = None):
 824         super(NS3Dependency, self).__init__(api)
 825
 826         self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip'
 827
 828         # We have to download the sources, untar, build...
 829         pybindgen_source_url = "http://yans.pl.sophia.inria.fr/trac/nepi/raw-attachment/wiki/WikiStart/pybindgen-r794.tar.gz"
 830         pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
 831         ns3_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/ns-3.11-nepi/archive/tip.tar.gz"
 832         passfd_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/python-passfd/archive/tip.tar.gz"
 833         self.build =(
 834             " ( "
 835             "  cd .. && "
 836             "  python -c 'import pygccxml, pybindgen, passfd' && "
 837             "  test -f lib/ns/_core.so && "
 838             "  test -f lib/ns/__init__.py && "
 839             "  test -f lib/ns/core.py && "
 840             "  test -f lib/libns3-core.so && "
 841             "  LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
 842             " ) || ( "
 843                 # Not working, rebuild
 844                      # Archive SHA1 sums to check
 845                      "echo '7158877faff2254e6c094bf18e6b4283cac19137  pygccxml-1.0.0.zip' > archive_sums.txt && "
 846                      "echo 'a18c2ccffd0df517bc37e2f3a2475092517c43f2  pybindgen-src.tar.gz' >> archive_sums.txt && "
 847                      " ( " # check existing files
 848                      " sha1sum -c archive_sums.txt && "
 849                      " test -f passfd-src.tar.gz && "
 850                      " test -f ns3-src.tar.gz "
 851                      " ) || ( " # nope? re-download
 852                      " rm -f pybindgen-src.zip pygccxml-1.0.0.zip passfd-src.tar.gz ns3-src.tar.gz && "
 853                      " wget -q -c -O pybindgen-src.tar.gz %(pybindgen_source_url)s && " # continue, to exploit the case when it has already been dl'ed
 854                      " wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
 855                      " wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
 856                      " wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
 857                      " sha1sum -c archive_sums.txt " # Check SHA1 sums when applicable
 858                      " ) && "
 859                      "unzip -n pygccxml-1.0.0.zip && "
 860                      "mkdir -p pybindgen-src && "
 861                      "mkdir -p ns3-src && "
 862                      "mkdir -p passfd-src && "
 863                      "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
 864                      "tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
 865                      "tar xzf pybindgen-src.tar.gz --strip-components=1 -C pybindgen-src && "
 866                      "rm -rf target && "    # mv doesn't like unclean targets
 867                      "mkdir -p target && "
 868                      "cd pygccxml-1.0.0 && "
 869                      "rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
 870                      "python setup.py build && "
 871                      "python setup.py install --install-lib ${BUILD}/target && "
 872                      "python setup.py clean && "
 873                      "cd ../pybindgen-src && "
 874                      "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
 875                      "./waf configure --prefix=${BUILD}/target -d release && "
 876                      "./waf && "
 877                      "./waf install && "
 878                      "./waf clean && "
 879                      "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
 880                      "rm -rf ${BUILD}/target/lib && "
 881                      "cd ../passfd-src && "
 882                      "python setup.py build && "
 883                      "python setup.py install --install-lib ${BUILD}/target && "
 884                      "python setup.py clean && "
 885                      "cd ../ns3-src && "
 886                      "./waf configure --prefix=${BUILD}/target --with-pybindgen=../pybindgen-src -d release --disable-examples --disable-tests && "
 887                      "./waf &&"
 888                      "./waf install && "
 889                      "rm -f ${BUILD}/target/lib/*.so && "
 890                      "cp -a ${BUILD}/ns3-src/build/release/libns3*.so ${BUILD}/target/lib && "
 891                      "cp -a ${BUILD}/ns3-src/build/release/bindings/python/ns ${BUILD}/target/lib &&"
 892                      "./waf clean "
 893              " )"
 894                      % dict(
 895                         pybindgen_source_url = server.shell_escape(pybindgen_source_url),
 896                         pygccxml_source_url = server.shell_escape(pygccxml_source_url),
 897                         ns3_source_url = server.shell_escape(ns3_source_url),
 898                         passfd_source_url = server.shell_escape(passfd_source_url),
 899                      ))
 900
 901         # Just move ${BUILD}/target
 902         self.install = (
 903             " ( "
 904             "  cd .. && "
 905             "  python -c 'import pygccxml, pybindgen, passfd' && "
 906             "  test -f lib/ns/_core.so && "
 907             "  test -f lib/ns/__init__.py && "
 908             "  test -f lib/ns/core.py && "
 909             "  test -f lib/libns3-core.so && "
 910             "  LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
 911             " ) || ( "
 912                 # Not working, reinstall
 913                     "test -d ${BUILD}/target && "
 914                     "[[ \"x\" != \"x$(find ${BUILD}/target -mindepth 1 -print -quit)\" ]] &&"
 915                     "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
 916                     "mv -f ${BUILD}/target/* ${SOURCES}"
 917             " )"
 918         )
 919
 920         # Set extra environment paths
 921         self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
 922         self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib"
 923
 924     @property
 925     def tarball(self):
 926         if self._tarball is None:
 927             shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
 928             if shared_tar is not None:
 929                 self._tarball = shared_tar
 930             else:
 931                 # Build an ad-hoc tarball
 932                 # Prebuilt
 933                 import nepi
 934                 import tempfile
 935
 936                 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
 937
 938                 proc = subprocess.Popen(
 939                     ["tar", "czf", shared_tar.name,
 940                         '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
 941                         'nepi'],
 942                     stdout = open("/dev/null","w"),
 943                     stdin = open("/dev/null","r"))
 944
 945                 if proc.wait():
 946                     raise RuntimeError, "Failed to create nepi tarball"
 947
 948                 self._tarball = self._shared_nepi_tar = shared_tar
 949
 950         return self._tarball
 951
 952 class YumDependency(Dependency):
 953     """
 954     This dependency is an internal helper class used to
 955     efficiently distribute yum-downloaded rpms.
 956
 957     It temporarily sets the yum cache as persistent in the
 958     build master, and installs all the required packages.
 959
 960     The rpm packages left in the yum cache are gathered and
 961     distributed by the underlying Dependency in an efficient
 962     manner. Build slaves will then install those rpms back in
 963     the cache before issuing the install command.
 964
 965     When packages have been installed already, nothing but an
 966     empty tar is distributed.
 967     """
 968
 969     # Class attribute holding a *weak* reference to the shared NEPI tar file
 970     # so that they may share it. Don't operate on the file itself, it would
 971     # be a mess, just use its path.
 972     _shared_nepi_tar = None
 973
 974     def _build_get(self):
 975         # canonical representation of dependencies
 976         depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
 977
 978         # download rpms and pack into a tar archive
 979         return (
 980             "sudo -S nice yum -y makecache && "
 981             "sudo -S sed -i -r 's/keepcache *= *0/keepcache=1/' /etc/yum.conf && "
 982             " ( ( "
 983                 "sudo -S nice yum -y install %s ; "
 984                 "rm -f ${BUILD}/packages.tar ; "
 985                 "tar -C /var/cache/yum -rf ${BUILD}/packages.tar $(cd /var/cache/yum ; find -iname '*.rpm')"
 986             " ) || /bin/true ) && "
 987             "sudo -S sed -i -r 's/keepcache *= *1/keepcache=0/' /etc/yum.conf && "
 988             "sudo -S nice yum -y clean packages "
 989         ) % ( depends, )
 990     def _build_set(self, value):
 991         # ignore
 992         return
 993     build = property(_build_get, _build_set)
 994
 995     def _install_get(self):
 996         # canonical representation of dependencies
 997         depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
 998
 999         # unpack cached rpms into yum cache, install, and cleanup
1000         return (
1001             "sudo -S tar -k --keep-newer-files -C /var/cache/yum -xf packages.tar && "
1002             "sudo -S nice yum -y install %s && "
1003             "sudo -S nice yum -y clean packages "
1004         ) % ( depends, )
1005     def _install_set(self, value):
1006         # ignore
1007         return
1008     install = property(_install_get, _install_set)
1009
1010