src/nepi/testbeds/planetlab/application.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from constants import TESTBED_ID
   5 import plcapi
   6 import operator
   7 import os
   8 import os.path
   9 import sys
  10 import nepi.util.server as server
  11 import cStringIO
  12 import subprocess
  13 import rspawn
  14 import random
  15 import time
  16 import socket
  17 import threading
  18 import logging
  19
  20 from nepi.util.constants import ApplicationStatus as AS
  21
  22 class Dependency(object):
  23     """
  24     A Dependency is in every respect like an application.
  25
  26     It depends on some packages, it may require building binaries, it must deploy
  27     them...
  28
  29     But it has no command. Dependencies aren't ever started, or stopped, and have
  30     no status.
  31     """
  32
  33     TRACES = ('buildlog')
  34
  35     def __init__(self, api=None):
  36         if not api:
  37             api = plcapi.PLCAPI()
  38         self._api = api
  39
  40         # Attributes
  41         self.command = None
  42         self.sudo = False
  43
  44         self.build = None
  45         self.install = None
  46         self.depends = None
  47         self.buildDepends = None
  48         self.sources = None
  49         self.rpmFusion = False
  50         self.env = {}
  51
  52         self.stdin = None
  53         self.stdout = None
  54         self.stderr = None
  55         self.buildlog = None
  56
  57         self.add_to_path = True
  58
  59         # Those are filled when the app is configured
  60         self.home_path = None
  61
  62         # Those are filled when an actual node is connected
  63         self.node = None
  64
  65         # Those are filled when the app is started
  66         #   Having both pid and ppid makes it harder
  67         #   for pid rollover to induce tracking mistakes
  68         self._started = False
  69         self._setup = False
  70         self._setuper = None
  71         self._pid = None
  72         self._ppid = None
  73
  74         # Spanning tree deployment
  75         self._master = None
  76         self._master_passphrase = None
  77         self._master_prk = None
  78         self._master_puk = None
  79         self._master_token = ''.join(map(chr,[rng.randint(0,255)
  80                                       for rng in (random.SystemRandom(),)
  81                                       for i in xrange(8)] )).encode("hex")
  82         self._build_pid = None
  83         self._build_ppid = None
  84
  85         # Logging
  86         self._logger = logging.getLogger('nepi.testbeds.planetlab')
  87
  88
  89     def __str__(self):
  90         return "%s<%s>" % (
  91             self.__class__.__name__,
  92             ' '.join(filter(bool,(self.depends, self.sources)))
  93         )
  94
  95     def validate(self):
  96         if self.home_path is None:
  97             raise AssertionError, "Misconfigured application: missing home path"
  98         if self.node.ident_path is None or not os.access(self.node.ident_path, os.R_OK):
  99             raise AssertionError, "Misconfigured application: missing slice SSH key"
 100         if self.node is None:
 101             raise AssertionError, "Misconfigured application: unconnected node"
 102         if self.node.hostname is None:
 103             raise AssertionError, "Misconfigured application: misconfigured node"
 104         if self.node.slicename is None:
 105             raise AssertionError, "Misconfigured application: unspecified slice"
 106
 107     def remote_trace_path(self, whichtrace):
 108         if whichtrace in self.TRACES:
 109             tracefile = os.path.join(self.home_path, whichtrace)
 110         else:
 111             tracefile = None
 112
 113         return tracefile
 114
 115     def remote_trace_name(self, whichtrace):
 116         if whichtrace in self.TRACES:
 117             return whichtrace
 118         return None
 119
 120     def sync_trace(self, local_dir, whichtrace):
 121         tracefile = self.remote_trace_path(whichtrace)
 122         if not tracefile:
 123             return None
 124
 125         local_path = os.path.join(local_dir, tracefile)
 126
 127         # create parent local folders
 128         proc = subprocess.Popen(
 129             ["mkdir", "-p", os.path.dirname(local_path)],
 130             stdout = open("/dev/null","w"),
 131             stdin = open("/dev/null","r"))
 132
 133         if proc.wait():
 134             raise RuntimeError, "Failed to synchronize trace"
 135
 136         # sync files
 137         try:
 138             self._popen_scp(
 139                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 140                     tracefile),
 141                 local_path
 142                 )
 143         except RuntimeError, e:
 144             raise RuntimeError, "Failed to synchronize trace: %s %s" \
 145                     % (e.args[0], e.args[1],)
 146
 147         return local_path
 148
 149     def recover(self):
 150         # We assume a correct deployment, so recovery only
 151         # means we mark this dependency as deployed
 152         self._setup = True
 153
 154     def setup(self):
 155         self._logger.info("Setting up %s", self)
 156         self._make_home()
 157         self._launch_build()
 158         self._finish_build()
 159         self._setup = True
 160
 161     def async_setup(self):
 162         if not self._setuper:
 163             def setuper():
 164                 try:
 165                     self.setup()
 166                 except:
 167                     self._setuper._exc.append(sys.exc_info())
 168             self._setuper = threading.Thread(
 169                 target = setuper)
 170             self._setuper._exc = []
 171             self._setuper.start()
 172
 173     def async_setup_wait(self):
 174         if not self._setup:
 175             self._logger.info("Waiting for %s to be setup", self)
 176             if self._setuper:
 177                 self._setuper.join()
 178                 if not self._setup:
 179                     if self._setuper._exc:
 180                         exctyp,exval,exctrace = self._setuper._exc[0]
 181                         raise exctyp,exval,exctrace
 182                     else:
 183                         raise RuntimeError, "Failed to setup application"
 184                 else:
 185                     self._logger.info("Setup ready: %s", self)
 186             else:
 187                 self.setup()
 188
 189     def _make_home(self):
 190         # Make sure all the paths are created where
 191         # they have to be created for deployment
 192         # sync files
 193         try:
 194             self._popen_ssh_command(
 195                 "mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" \
 196                     % { 'home' : server.shell_escape(self.home_path) },
 197                 timeout = 120,
 198                 retry = 3
 199                 )
 200         except RuntimeError, e:
 201             raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, e.args[0], e.args[1],)
 202
 203         if self.stdin:
 204             # Write program input
 205             try:
 206                 self._popen_scp(
 207                     cStringIO.StringIO(self.stdin),
 208                     '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 209                         os.path.join(self.home_path, 'stdin') ),
 210                     )
 211             except RuntimeError, e:
 212                 raise RuntimeError, "Failed to set up application %s: %s %s" \
 213                         % (self.home_path, e.args[0], e.args[1],)
 214
 215     def _replace_paths(self, command):
 216         """
 217         Replace all special path tags with shell-escaped actual paths.
 218         """
 219         # need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
 220         root = '' if self.home_path.startswith('/') else "${HOME}/"
 221         return ( command
 222             .replace("${SOURCES}", root+server.shell_escape(self.home_path))
 223             .replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
 224
 225     def _launch_build(self):
 226         if self._master is not None:
 227             self._do_install_keys()
 228             buildscript = self._do_build_slave()
 229         else:
 230             buildscript = self._do_build_master()
 231
 232         if buildscript is not None:
 233             self._logger.info("Building %s", self)
 234
 235             # upload build script
 236             try:
 237                 self._popen_scp(
 238                     buildscript,
 239                     '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 240                         os.path.join(self.home_path, 'nepi-build.sh') )
 241                     )
 242             except RuntimeError, e:
 243                 raise RuntimeError, "Failed to set up application %s: %s %s" \
 244                         % (self.home_path, e.args[0], e.args[1],)
 245
 246             # launch build
 247             self._do_launch_build()
 248
 249     def _finish_build(self):
 250         self._do_wait_build()
 251         self._do_install()
 252
 253     def _do_build_slave(self):
 254         if not self.sources and not self.build:
 255             return None
 256
 257         # Create build script
 258         files = set()
 259
 260         if self.sources:
 261             sources = self.sources.split(' ')
 262             files.update(
 263                 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
 264                     os.path.join(self._master.home_path, os.path.basename(source)),)
 265                 for source in sources
 266             )
 267
 268         if self.build:
 269             files.add(
 270                 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
 271                     os.path.join(self._master.home_path, 'build.tar.gz'),)
 272             )
 273
 274         launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
 275                         " && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
 276                         " && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" %  \
 277         {
 278             'prk' : server.shell_escape(self._master_prk_name),
 279             'puk' : server.shell_escape(self._master_puk_name),
 280         }
 281
 282         kill_agent = "kill $SSH_AGENT_PID"
 283
 284         waitmaster = "{ . ./.ssh-agent.sh ; while [[ $(ssh -q -o UserKnownHostsFile=%(hostkey)s %(master)s cat %(token_path)s) != %(token)s ]] ; do sleep 5 ; done ; }" % {
 285             'hostkey' : 'master_known_hosts',
 286             'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostname),
 287             'token_path' : os.path.join(self._master.home_path, 'build.token'),
 288             'token' : server.shell_escape(self._master._master_token),
 289         }
 290
 291         syncfiles = "scp -p -o UserKnownHostsFile=%(hostkey)s %(files)s ." % {
 292             'hostkey' : 'master_known_hosts',
 293             'files' : ' '.join(files),
 294         }
 295         if self.build:
 296             syncfiles += " && tar xzf build.tar.gz"
 297         syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
 298         syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
 299
 300         cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
 301             'prk' : server.shell_escape(self._master_prk_name),
 302             'puk' : server.shell_escape(self._master_puk_name),
 303         }
 304
 305         slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s )" % {
 306             'waitmaster' : waitmaster,
 307             'syncfiles' : syncfiles,
 308             'cleanup' : cleanup,
 309             'kill_agent' : kill_agent,
 310             'launch_agent' : launch_agent,
 311             'home' : server.shell_escape(self.home_path),
 312         }
 313
 314         return cStringIO.StringIO(slavescript)
 315
 316     def _do_launch_build(self):
 317         script = "bash ./nepi-build.sh"
 318         if self._master_passphrase:
 319             script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
 320                 server.shell_escape(self._master_passphrase),
 321                 script
 322             )
 323         (out,err),proc = rspawn.remote_spawn(
 324             script,
 325             pidfile = 'build-pid',
 326             home = self.home_path,
 327             stdin = '/dev/null',
 328             stdout = 'buildlog',
 329             stderr = rspawn.STDOUT,
 330
 331             host = self.node.hostname,
 332             port = None,
 333             user = self.node.slicename,
 334             agent = None,
 335             ident_key = self.node.ident_path,
 336             server_key = self.node.server_key
 337             )
 338
 339         if proc.wait():
 340             raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
 341
 342
 343         pid = ppid = None
 344         delay = 1.0
 345         for i in xrange(5):
 346             pidtuple = rspawn.remote_check_pid(
 347                 os.path.join(self.home_path,'build-pid'),
 348                 host = self.node.hostname,
 349                 port = None,
 350                 user = self.node.slicename,
 351                 agent = None,
 352                 ident_key = self.node.ident_path,
 353                 server_key = self.node.server_key
 354                 )
 355
 356             if pidtuple:
 357                 pid, ppid = pidtuple
 358                 self._build_pid, self._build_ppid = pidtuple
 359                 break
 360             else:
 361                 time.sleep(delay)
 362                 delay = min(30,delay*1.2)
 363         else:
 364             raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
 365
 366         self._logger.info("Deploying %s", self)
 367
 368     def _do_wait_build(self):
 369         pid = self._build_pid
 370         ppid = self._build_ppid
 371
 372         if pid and ppid:
 373             delay = 1.0
 374             first = True
 375             bustspin = 0
 376             while True:
 377                 status = rspawn.remote_status(
 378                     pid, ppid,
 379                     host = self.node.hostname,
 380                     port = None,
 381                     user = self.node.slicename,
 382                     agent = None,
 383                     ident_key = self.node.ident_path,
 384                     server_key = self.node.server_key
 385                     )
 386
 387                 if status is rspawn.FINISHED:
 388                     self._build_pid = self._build_ppid = None
 389                     break
 390                 elif status is not rspawn.RUNNING:
 391                     bustspin += 1
 392                     time.sleep(5)
 393                     if bustspin > 12:
 394                         self._build_pid = self._build_ppid = None
 395                         break
 396                 else:
 397                     if first:
 398                         self._logger.info("Waiting for %s to finish building %s", self,
 399                             "(build slave)" if self._master is not None else "(build master)")
 400
 401                         first = False
 402                     time.sleep(delay*(0.5+random.random()))
 403                     delay = min(30,delay*1.2)
 404
 405             # check build token
 406             slave_token = ""
 407             for i in xrange(3):
 408                 (out, err), proc = self._popen_ssh_command(
 409                     "cat %(token_path)s" % {
 410                         'token_path' : os.path.join(self.home_path, 'build.token'),
 411                     },
 412                     timeout = 120,
 413                     noerrors = True)
 414                 if not proc.wait() and out:
 415                     slave_token = out.strip()
 416
 417                 if slave_token:
 418                     break
 419                 else:
 420                     time.sleep(2)
 421
 422             if slave_token != self._master_token:
 423                 # Get buildlog for the error message
 424
 425                 (buildlog, err), proc = self._popen_ssh_command(
 426                     "cat %(buildlog)s" % {
 427                         'buildlog' : os.path.join(self.home_path, 'buildlog'),
 428                         'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
 429                     },
 430                     timeout = 120,
 431                     noerrors = True)
 432
 433                 proc.wait()
 434
 435                 raise RuntimeError, "Failed to set up application %s: "\
 436                         "build failed, got wrong token from pid %s/%s "\
 437                         "(expected %r, got %r), see buildlog: %s" % (
 438                     self.home_path, pid, ppid, self._master_token, slave_token, buildlog)
 439
 440             self._logger.info("Built %s", self)
 441
 442     def _do_kill_build(self):
 443         pid = self._build_pid
 444         ppid = self._build_ppid
 445
 446         if pid and ppid:
 447             self._logger.info("Killing build of %s", self)
 448             rspawn.remote_kill(
 449                 pid, ppid,
 450                 host = self.node.hostname,
 451                 port = None,
 452                 user = self.node.slicename,
 453                 agent = None,
 454                 ident_key = self.node.ident_path
 455                 )
 456
 457
 458     def _do_build_master(self):
 459         if not self.sources and not self.build and not self.buildDepends:
 460             return None
 461
 462         if self.sources:
 463             sources = self.sources.split(' ')
 464
 465             # Copy all sources
 466             try:
 467                 self._popen_scp(
 468                     sources,
 469                     "%s@%s:%s" % (self.node.slicename, self.node.hostname,
 470                         os.path.join(self.home_path,'.'),)
 471                     )
 472             except RuntimeError, e:
 473                 raise RuntimeError, "Failed upload source file %r: %s %s" \
 474                         % (sources, e.args[0], e.args[1],)
 475
 476         buildscript = cStringIO.StringIO()
 477
 478         if self.buildDepends:
 479             # Install build dependencies
 480             buildscript.write(
 481                 "sudo -S yum -y install %(packages)s\n" % {
 482                     'packages' : self.buildDepends
 483                 }
 484             )
 485
 486
 487         if self.build:
 488             # Build sources
 489             buildscript.write(
 490                 "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
 491                     'command' : self._replace_paths(self.build),
 492                     'home' : server.shell_escape(self.home_path),
 493                 }
 494             )
 495
 496             # Make archive
 497             buildscript.write("tar czf build.tar.gz build\n")
 498
 499         # Write token
 500         buildscript.write("echo %(master_token)s > build.token" % {
 501             'master_token' : server.shell_escape(self._master_token)
 502         })
 503
 504         buildscript.seek(0)
 505
 506         return buildscript
 507
 508     def _do_install(self):
 509         if self.install:
 510             self._logger.info("Installing %s", self)
 511
 512             # Install application
 513             try:
 514                 self._popen_ssh_command(
 515                     "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/{install,build}log >&2 && false )" % \
 516                         {
 517                         'command' : self._replace_paths(self.install),
 518                         'home' : server.shell_escape(self.home_path),
 519                         },
 520                     )
 521             except RuntimeError, e:
 522                 raise RuntimeError, "Failed install build sources: %s %s" % (e.args[0], e.args[1],)
 523
 524     def set_master(self, master):
 525         self._master = master
 526
 527     def install_keys(self, prk, puk, passphrase):
 528         # Install keys
 529         self._master_passphrase = passphrase
 530         self._master_prk = prk
 531         self._master_puk = puk
 532         self._master_prk_name = os.path.basename(prk.name)
 533         self._master_puk_name = os.path.basename(puk.name)
 534
 535     def _do_install_keys(self):
 536         prk = self._master_prk
 537         puk = self._master_puk
 538
 539         try:
 540             self._popen_scp(
 541                 [ prk.name, puk.name ],
 542                 '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path )
 543                 )
 544         except RuntimeError, e:
 545             raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
 546                     % (e.args[0], e.args[1],)
 547
 548         try:
 549             self._popen_scp(
 550                 cStringIO.StringIO('%s,%s %s\n' % (
 551                     self._master.node.hostname, socket.gethostbyname(self._master.node.hostname),
 552                     self._master.node.server_key)),
 553                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 554                     os.path.join(self.home_path,"master_known_hosts") )
 555                 )
 556         except RuntimeError, e:
 557             raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
 558                     % (e.args[0], e.args[1],)
 559
 560         # No longer need'em
 561         self._master_prk = None
 562         self._master_puk = None
 563
 564     def cleanup(self):
 565         # make sure there's no leftover build processes
 566         self._do_kill_build()
 567
 568     @server.eintr_retry
 569     def _popen_scp(self, src, dst, retry = 3):
 570         while 1:
 571             try:
 572                 (out,err),proc = server.popen_scp(
 573                     src,
 574                     dst,
 575                     port = None,
 576                     agent = None,
 577                     ident_key = self.node.ident_path,
 578                     server_key = self.node.server_key
 579                     )
 580
 581                 if server.eintr_retry(proc.wait)():
 582                     raise RuntimeError, (out, err)
 583                 return (out, err), proc
 584             except:
 585                 if retry <= 0:
 586                     raise
 587                 else:
 588                     retry -= 1
 589
 590
 591     @server.eintr_retry
 592     def _popen_ssh_command(self, command, retry = 0, noerrors=False, timeout=None):
 593         (out,err),proc = server.popen_ssh_command(
 594             command,
 595             host = self.node.hostname,
 596             port = None,
 597             user = self.node.slicename,
 598             agent = None,
 599             ident_key = self.node.ident_path,
 600             server_key = self.node.server_key,
 601             timeout = timeout,
 602             retry = retry
 603             )
 604
 605         if server.eintr_retry(proc.wait)():
 606             if not noerrors:
 607                 raise RuntimeError, (out, err)
 608         return (out, err), proc
 609
 610 class Application(Dependency):
 611     """
 612     An application also has dependencies, but also a command to be ran and monitored.
 613
 614     It adds the output of that command as traces.
 615     """
 616
 617     TRACES = ('stdout','stderr','buildlog', 'output')
 618
 619     def __init__(self, api=None):
 620         super(Application,self).__init__(api)
 621
 622         # Attributes
 623         self.command = None
 624         self.sudo = False
 625
 626         self.stdin = None
 627         self.stdout = None
 628         self.stderr = None
 629         self.output = None
 630
 631         # Those are filled when the app is started
 632         #   Having both pid and ppid makes it harder
 633         #   for pid rollover to induce tracking mistakes
 634         self._started = False
 635         self._pid = None
 636         self._ppid = None
 637
 638         # Do not add to the python path of nodes
 639         self.add_to_path = False
 640
 641     def __str__(self):
 642         return "%s<command:%s%s>" % (
 643             self.__class__.__name__,
 644             "sudo " if self.sudo else "",
 645             self.command,
 646         )
 647
 648     def start(self):
 649         self._logger.info("Starting %s", self)
 650
 651         # Create shell script with the command
 652         # This way, complex commands and scripts can be ran seamlessly
 653         # sync files
 654         command = cStringIO.StringIO()
 655         command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
 656             ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
 657         ))
 658         command.write('export PATH=$PATH:%s\n' % (
 659             ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
 660         ))
 661         if self.node.env:
 662             for envkey, envvals in self.node.env.iteritems():
 663                 for envval in envvals:
 664                     command.write('export %s=%s\n' % (envkey, envval))
 665         command.write(self.command)
 666         command.seek(0)
 667
 668         try:
 669             self._popen_scp(
 670                 command,
 671                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 672                     os.path.join(self.home_path, "app.sh"))
 673                 )
 674         except RuntimeError, e:
 675             raise RuntimeError, "Failed to set up application: %s %s" \
 676                     % (e.args[0], e.args[1],)
 677
 678         # Start process in a "daemonized" way, using nohup and heavy
 679         # stdin/out redirection to avoid connection issues
 680         (out,err),proc = rspawn.remote_spawn(
 681             self._replace_paths("bash ./app.sh"),
 682
 683             pidfile = './pid',
 684             home = self.home_path,
 685             stdin = 'stdin' if self.stdin is not None else '/dev/null',
 686             stdout = 'stdout' if self.stdout else '/dev/null',
 687             stderr = 'stderr' if self.stderr else '/dev/null',
 688             sudo = self.sudo,
 689
 690             host = self.node.hostname,
 691             port = None,
 692             user = self.node.slicename,
 693             agent = None,
 694             ident_key = self.node.ident_path,
 695             server_key = self.node.server_key
 696             )
 697
 698         if proc.wait():
 699             raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
 700
 701         self._started = True
 702
 703     def recover(self):
 704         # Assuming the application is running on PlanetLab,
 705         # proper pidfiles should be present at the app's home path.
 706         # So we mark this application as started, and check the pidfiles
 707         self._started = True
 708         self.checkpid()
 709
 710     def checkpid(self):
 711         # Get PID/PPID
 712         # NOTE: wait a bit for the pidfile to be created
 713         if self._started and not self._pid or not self._ppid:
 714             pidtuple = rspawn.remote_check_pid(
 715                 os.path.join(self.home_path,'pid'),
 716                 host = self.node.hostname,
 717                 port = None,
 718                 user = self.node.slicename,
 719                 agent = None,
 720                 ident_key = self.node.ident_path,
 721                 server_key = self.node.server_key
 722                 )
 723
 724             if pidtuple:
 725                 self._pid, self._ppid = pidtuple
 726
 727     def status(self):
 728         self.checkpid()
 729         if not self._started:
 730             return AS.STATUS_NOT_STARTED
 731         elif not self._pid or not self._ppid:
 732             return AS.STATUS_NOT_STARTED
 733         else:
 734             status = rspawn.remote_status(
 735                 self._pid, self._ppid,
 736                 host = self.node.hostname,
 737                 port = None,
 738                 user = self.node.slicename,
 739                 agent = None,
 740                 ident_key = self.node.ident_path,
 741                 server_key = self.node.server_key
 742                 )
 743
 744             if status is rspawn.NOT_STARTED:
 745                 return AS.STATUS_NOT_STARTED
 746             elif status is rspawn.RUNNING:
 747                 return AS.STATUS_RUNNING
 748             elif status is rspawn.FINISHED:
 749                 return AS.STATUS_FINISHED
 750             else:
 751                 # WTF?
 752                 return AS.STATUS_NOT_STARTED
 753
 754     def kill(self):
 755         status = self.status()
 756         if status == AS.STATUS_RUNNING:
 757             # kill by ppid+pid - SIGTERM first, then try SIGKILL
 758             rspawn.remote_kill(
 759                 self._pid, self._ppid,
 760                 host = self.node.hostname,
 761                 port = None,
 762                 user = self.node.slicename,
 763                 agent = None,
 764                 ident_key = self.node.ident_path,
 765                 server_key = self.node.server_key,
 766                 sudo = self.sudo
 767                 )
 768             self._logger.info("Killed %s", self)
 769
 770
 771 class NepiDependency(Dependency):
 772     """
 773     This dependency adds nepi itself to the python path,
 774     so that you may run testbeds within PL nodes.
 775     """
 776
 777     # Class attribute holding a *weak* reference to the shared NEPI tar file
 778     # so that they may share it. Don't operate on the file itself, it would
 779     # be a mess, just use its path.
 780     _shared_nepi_tar = None
 781
 782     def __init__(self, api = None):
 783         super(NepiDependency, self).__init__(api)
 784
 785         self._tarball = None
 786
 787         self.depends = 'python python-ipaddr python-setuptools'
 788
 789         # our sources are in our ad-hoc tarball
 790         self.sources = self.tarball.name
 791
 792         tarname = os.path.basename(self.tarball.name)
 793
 794         # it's already built - just move the tarball into place
 795         self.build = "mv -f ${SOURCES}/%s ." % (tarname,)
 796
 797         # unpack it into sources, and we're done
 798         self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,)
 799
 800     @property
 801     def tarball(self):
 802         if self._tarball is None:
 803             shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
 804             if shared_tar is not None:
 805                 self._tarball = shared_tar
 806             else:
 807                 # Build an ad-hoc tarball
 808                 # Prebuilt
 809                 import nepi
 810                 import tempfile
 811
 812                 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
 813
 814                 proc = subprocess.Popen(
 815                     ["tar", "czf", shared_tar.name,
 816                         '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
 817                         'nepi'],
 818                     stdout = open("/dev/null","w"),
 819                     stdin = open("/dev/null","r"))
 820
 821                 if proc.wait():
 822                     raise RuntimeError, "Failed to create nepi tarball"
 823
 824                 self._tarball = self._shared_nepi_tar = shared_tar
 825
 826         return self._tarball
 827
 828 class NS3Dependency(Dependency):
 829     """
 830     This dependency adds NS3 libraries to the library paths,
 831     so that you may run the NS3 testbed within PL nodes.
 832
 833     You'll also need the NepiDependency.
 834     """
 835
 836     def __init__(self, api = None):
 837         super(NS3Dependency, self).__init__(api)
 838
 839         self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip'
 840
 841         # We have to download the sources, untar, build...
 842         pybindgen_source_url = "http://yans.pl.sophia.inria.fr/trac/nepi/raw-attachment/wiki/WikiStart/pybindgen-r794.tar.gz"
 843         pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
 844         ns3_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/ns-3.11-nepi/archive/tip.tar.gz"
 845         passfd_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/python-passfd/archive/tip.tar.gz"
 846         self.build =(
 847             " ( "
 848             "  cd .. && "
 849             "  python -c 'import pygccxml, pybindgen, passfd' && "
 850             "  test -f lib/ns/_core.so && "
 851             "  test -f lib/ns/__init__.py && "
 852             "  test -f lib/ns/core.py && "
 853             "  test -f lib/libns3-core.so && "
 854             "  LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
 855             " ) || ( "
 856                 # Not working, rebuild
 857                      # Archive SHA1 sums to check
 858                      "echo '7158877faff2254e6c094bf18e6b4283cac19137  pygccxml-1.0.0.zip' > archive_sums.txt && "
 859                      "echo 'a18c2ccffd0df517bc37e2f3a2475092517c43f2  pybindgen-src.tar.gz' >> archive_sums.txt && "
 860                      " ( " # check existing files
 861                      " sha1sum -c archive_sums.txt && "
 862                      " test -f passfd-src.tar.gz && "
 863                      " test -f ns3-src.tar.gz "
 864                      " ) || ( " # nope? re-download
 865                      " rm -f pybindgen-src.zip pygccxml-1.0.0.zip passfd-src.tar.gz ns3-src.tar.gz && "
 866                      " wget -q -c -O pybindgen-src.tar.gz %(pybindgen_source_url)s && " # continue, to exploit the case when it has already been dl'ed
 867                      " wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
 868                      " wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
 869                      " wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
 870                      " sha1sum -c archive_sums.txt " # Check SHA1 sums when applicable
 871                      " ) && "
 872                      "unzip -n pygccxml-1.0.0.zip && "
 873                      "mkdir -p pybindgen-src && "
 874                      "mkdir -p ns3-src && "
 875                      "mkdir -p passfd-src && "
 876                      "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
 877                      "tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
 878                      "tar xzf pybindgen-src.tar.gz --strip-components=1 -C pybindgen-src && "
 879                      "rm -rf target && "    # mv doesn't like unclean targets
 880                      "mkdir -p target && "
 881                      "cd pygccxml-1.0.0 && "
 882                      "rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
 883                      "python setup.py build && "
 884                      "python setup.py install --install-lib ${BUILD}/target && "
 885                      "python setup.py clean && "
 886                      "cd ../pybindgen-src && "
 887                      "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
 888                      "./waf configure --prefix=${BUILD}/target -d release && "
 889                      "./waf && "
 890                      "./waf install && "
 891                      "./waf clean && "
 892                      "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
 893                      "rm -rf ${BUILD}/target/lib && "
 894                      "cd ../passfd-src && "
 895                      "python setup.py build && "
 896                      "python setup.py install --install-lib ${BUILD}/target && "
 897                      "python setup.py clean && "
 898                      "cd ../ns3-src && "
 899                      "./waf configure --prefix=${BUILD}/target --with-pybindgen=../pybindgen-src -d release --disable-examples --disable-tests && "
 900                      "./waf &&"
 901                      "./waf install && "
 902                      "rm -f ${BUILD}/target/lib/*.so && "
 903                      "cp -a ${BUILD}/ns3-src/build/release/libns3*.so ${BUILD}/target/lib && "
 904                      "cp -a ${BUILD}/ns3-src/build/release/bindings/python/ns ${BUILD}/target/lib &&"
 905                      "./waf clean "
 906              " )"
 907                      % dict(
 908                         pybindgen_source_url = server.shell_escape(pybindgen_source_url),
 909                         pygccxml_source_url = server.shell_escape(pygccxml_source_url),
 910                         ns3_source_url = server.shell_escape(ns3_source_url),
 911                         passfd_source_url = server.shell_escape(passfd_source_url),
 912                      ))
 913
 914         # Just move ${BUILD}/target
 915         self.install = (
 916             " ( "
 917             "  cd .. && "
 918             "  python -c 'import pygccxml, pybindgen, passfd' && "
 919             "  test -f lib/ns/_core.so && "
 920             "  test -f lib/ns/__init__.py && "
 921             "  test -f lib/ns/core.py && "
 922             "  test -f lib/libns3-core.so && "
 923             "  LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
 924             " ) || ( "
 925                 # Not working, reinstall
 926                     "test -d ${BUILD}/target && "
 927                     "[[ \"x\" != \"x$(find ${BUILD}/target -mindepth 1 -print -quit)\" ]] &&"
 928                     "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
 929                     "mv -f ${BUILD}/target/* ${SOURCES}"
 930             " )"
 931         )
 932
 933         # Set extra environment paths
 934         self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
 935         self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib"
 936
 937     @property
 938     def tarball(self):
 939         if self._tarball is None:
 940             shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
 941             if shared_tar is not None:
 942                 self._tarball = shared_tar
 943             else:
 944                 # Build an ad-hoc tarball
 945                 # Prebuilt
 946                 import nepi
 947                 import tempfile
 948
 949                 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
 950
 951                 proc = subprocess.Popen(
 952                     ["tar", "czf", shared_tar.name,
 953                         '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
 954                         'nepi'],
 955                     stdout = open("/dev/null","w"),
 956                     stdin = open("/dev/null","r"))
 957
 958                 if proc.wait():
 959                     raise RuntimeError, "Failed to create nepi tarball"
 960
 961                 self._tarball = self._shared_nepi_tar = shared_tar
 962
 963         return self._tarball
 964
 965 class YumDependency(Dependency):
 966     """
 967     This dependency is an internal helper class used to
 968     efficiently distribute yum-downloaded rpms.
 969
 970     It temporarily sets the yum cache as persistent in the
 971     build master, and installs all the required packages.
 972
 973     The rpm packages left in the yum cache are gathered and
 974     distributed by the underlying Dependency in an efficient
 975     manner. Build slaves will then install those rpms back in
 976     the cache before issuing the install command.
 977
 978     When packages have been installed already, nothing but an
 979     empty tar is distributed.
 980     """
 981
 982     # Class attribute holding a *weak* reference to the shared NEPI tar file
 983     # so that they may share it. Don't operate on the file itself, it would
 984     # be a mess, just use its path.
 985     _shared_nepi_tar = None
 986
 987     def _build_get(self):
 988         # canonical representation of dependencies
 989         depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
 990
 991         # download rpms and pack into a tar archive
 992         return (
 993             "sudo -S nice yum -y makecache && "
 994             "sudo -S sed -i -r 's/keepcache *= *0/keepcache=1/' /etc/yum.conf && "
 995             " ( ( "
 996                 "sudo -S nice yum -y install %s ; "
 997                 "rm -f ${BUILD}/packages.tar ; "
 998                 "tar -C /var/cache/yum -rf ${BUILD}/packages.tar $(cd /var/cache/yum ; find -iname '*.rpm')"
 999             " ) || /bin/true ) && "
1000             "sudo -S sed -i -r 's/keepcache *= *1/keepcache=0/' /etc/yum.conf && "
1001             "( sudo -S nice yum -y clean packages || /bin/true ) "
1002         ) % ( depends, )
1003     def _build_set(self, value):
1004         # ignore
1005         return
1006     build = property(_build_get, _build_set)
1007
1008     def _install_get(self):
1009         # canonical representation of dependencies
1010         depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
1011
1012         # unpack cached rpms into yum cache, install, and cleanup
1013         return (
1014             "sudo -S tar -k --keep-newer-files -C /var/cache/yum -xf packages.tar && "
1015             "sudo -S nice yum -y install %s && "
1016             "( sudo -S nice yum -y clean packages || /bin/true ) "
1017         ) % ( depends, )
1018     def _install_set(self, value):
1019         # ignore
1020         return
1021     install = property(_install_get, _install_set)
1022
1023