src/nepi/testbeds/planetlab/application.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from constants import TESTBED_ID
   5 import plcapi
   6 import operator
   7 import os
   8 import os.path
   9 import sys
  10 import nepi.util.server as server
  11 import cStringIO
  12 import subprocess
  13 import rspawn
  14 import random
  15 import time
  16 import socket
  17 import threading
  18 import logging
  19
  20 from nepi.util.constants import ApplicationStatus as AS
  21
  22 class Dependency(object):
  23     """
  24     A Dependency is in every respect like an application.
  25
  26     It depends on some packages, it may require building binaries, it must deploy
  27     them...
  28
  29     But it has no command. Dependencies aren't ever started, or stopped, and have
  30     no status.
  31     """
  32
  33     TRACES = ('buildlog')
  34
  35     def __init__(self, api=None):
  36         if not api:
  37             api = plcapi.PLCAPI()
  38         self._api = api
  39
  40         # Attributes
  41         self.command = None
  42         self.sudo = False
  43
  44         self.build = None
  45         self.install = None
  46         self.depends = None
  47         self.buildDepends = None
  48         self.sources = None
  49         self.rpmFusion = False
  50         self.env = {}
  51
  52         self.stdin = None
  53         self.stdout = None
  54         self.stderr = None
  55         self.buildlog = None
  56
  57         self.add_to_path = True
  58
  59         # Those are filled when the app is configured
  60         self.home_path = None
  61
  62         # Those are filled when an actual node is connected
  63         self.node = None
  64
  65         # Those are filled when the app is started
  66         #   Having both pid and ppid makes it harder
  67         #   for pid rollover to induce tracking mistakes
  68         self._started = False
  69         self._setup = False
  70         self._setuper = None
  71         self._pid = None
  72         self._ppid = None
  73
  74         # Spanning tree deployment
  75         self._master = None
  76         self._master_passphrase = None
  77         self._master_prk = None
  78         self._master_puk = None
  79         self._master_token = ''.join(map(chr,[rng.randint(0,255)
  80                                       for rng in (random.SystemRandom(),)
  81                                       for i in xrange(8)] )).encode("hex")
  82         self._build_pid = None
  83         self._build_ppid = None
  84
  85         # Logging
  86         self._logger = logging.getLogger('nepi.testbeds.planetlab')
  87
  88
  89     def __str__(self):
  90         return "%s<%s>" % (
  91             self.__class__.__name__,
  92             ' '.join(filter(bool,(self.depends, self.sources)))
  93         )
  94
  95     def validate(self):
  96         if self.home_path is None:
  97             raise AssertionError, "Misconfigured application: missing home path"
  98         if self.node.ident_path is None or not os.access(self.node.ident_path, os.R_OK):
  99             raise AssertionError, "Misconfigured application: missing slice SSH key"
 100         if self.node is None:
 101             raise AssertionError, "Misconfigured application: unconnected node"
 102         if self.node.hostname is None:
 103             raise AssertionError, "Misconfigured application: misconfigured node"
 104         if self.node.slicename is None:
 105             raise AssertionError, "Misconfigured application: unspecified slice"
 106
 107     def remote_trace_path(self, whichtrace):
 108         if whichtrace in self.TRACES:
 109             tracefile = os.path.join(self.home_path, whichtrace)
 110         else:
 111             tracefile = None
 112
 113         return tracefile
 114
 115     def remote_trace_name(self, whichtrace):
 116         if whichtrace in self.TRACES:
 117             return whichtrace
 118         return None
 119
 120     def sync_trace(self, local_dir, whichtrace):
 121         tracefile = self.remote_trace_path(whichtrace)
 122         if not tracefile:
 123             return None
 124
 125         local_path = os.path.join(local_dir, tracefile)
 126
 127         # create parent local folders
 128         proc = subprocess.Popen(
 129             ["mkdir", "-p", os.path.dirname(local_path)],
 130             stdout = open("/dev/null","w"),
 131             stdin = open("/dev/null","r"))
 132
 133         if proc.wait():
 134             raise RuntimeError, "Failed to synchronize trace"
 135
 136         # sync files
 137         try:
 138             self._popen_scp(
 139                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 140                     tracefile),
 141                 local_path
 142                 )
 143         except RuntimeError, e:
 144             raise RuntimeError, "Failed to synchronize trace: %s %s" \
 145                     % (e.args[0], e.args[1],)
 146
 147         return local_path
 148
 149     def recover(self):
 150         # We assume a correct deployment, so recovery only
 151         # means we mark this dependency as deployed
 152         self._setup = True
 153
 154     def setup(self):
 155         self._logger.info("Setting up %s", self)
 156         self._make_home()
 157         self._launch_build()
 158         self._finish_build()
 159         self._setup = True
 160
 161     def async_setup(self):
 162         if not self._setuper:
 163             def setuper():
 164                 try:
 165                     self.setup()
 166                 except:
 167                     self._setuper._exc.append(sys.exc_info())
 168             self._setuper = threading.Thread(
 169                 target = setuper)
 170             self._setuper._exc = []
 171             self._setuper.start()
 172
 173     def async_setup_wait(self):
 174         if not self._setup:
 175             self._logger.info("Waiting for %s to be setup", self)
 176             if self._setuper:
 177                 self._setuper.join()
 178                 if not self._setup:
 179                     if self._setuper._exc:
 180                         exctyp,exval,exctrace = self._setuper._exc[0]
 181                         raise exctyp,exval,exctrace
 182                     else:
 183                         raise RuntimeError, "Failed to setup application"
 184                 else:
 185                     self._logger.info("Setup ready: %s", self)
 186             else:
 187                 self.setup()
 188
 189     def _make_home(self):
 190         # Make sure all the paths are created where
 191         # they have to be created for deployment
 192         # sync files
 193         try:
 194             self._popen_ssh_command(
 195                 "mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" \
 196                     % { 'home' : server.shell_escape(self.home_path) },
 197                 timeout = 120,
 198                 retry = 3
 199                 )
 200         except RuntimeError, e:
 201             raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, e.args[0], e.args[1],)
 202
 203         if self.stdin:
 204             # Write program input
 205             try:
 206                 self._popen_scp(
 207                     cStringIO.StringIO(self.stdin),
 208                     '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 209                         os.path.join(self.home_path, 'stdin') ),
 210                     )
 211             except RuntimeError, e:
 212                 raise RuntimeError, "Failed to set up application %s: %s %s" \
 213                         % (self.home_path, e.args[0], e.args[1],)
 214
 215     def _replace_paths(self, command):
 216         """
 217         Replace all special path tags with shell-escaped actual paths.
 218         """
 219         # need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
 220         root = '' if self.home_path.startswith('/') else "${HOME}/"
 221         return ( command
 222             .replace("${SOURCES}", root+server.shell_escape(self.home_path))
 223             .replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
 224
 225     def _launch_build(self):
 226         if self._master is not None:
 227             self._do_install_keys()
 228             buildscript = self._do_build_slave()
 229         else:
 230             buildscript = self._do_build_master()
 231
 232         if buildscript is not None:
 233             self._logger.info("Building %s", self)
 234
 235             # upload build script
 236             try:
 237                 self._popen_scp(
 238                     buildscript,
 239                     '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 240                         os.path.join(self.home_path, 'nepi-build.sh') )
 241                     )
 242             except RuntimeError, e:
 243                 raise RuntimeError, "Failed to set up application %s: %s %s" \
 244                         % (self.home_path, e.args[0], e.args[1],)
 245
 246             # launch build
 247             self._do_launch_build()
 248
 249     def _finish_build(self):
 250         self._do_wait_build()
 251         self._do_install()
 252
 253     def _do_build_slave(self):
 254         if not self.sources and not self.build:
 255             return None
 256
 257         # Create build script
 258         files = set()
 259
 260         if self.sources:
 261             sources = self.sources.split(' ')
 262             files.update(
 263                 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
 264                     os.path.join(self._master.home_path, os.path.basename(source)),)
 265                 for source in sources
 266             )
 267
 268         if self.build:
 269             files.add(
 270                 "%s@%s:%s" % (self._master.node.slicename, self._master.node.hostname,
 271                     os.path.join(self._master.home_path, 'build.tar.gz'),)
 272             )
 273
 274         launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
 275                         " && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
 276                         " && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" %  \
 277         {
 278             'prk' : server.shell_escape(self._master_prk_name),
 279             'puk' : server.shell_escape(self._master_puk_name),
 280         }
 281
 282         kill_agent = "kill $SSH_AGENT_PID"
 283
 284         waitmaster = "{ . ./.ssh-agent.sh ; while [[ $(ssh -q -o UserKnownHostsFile=%(hostkey)s %(master)s cat %(token_path)s) != %(token)s ]] ; do sleep 5 ; done ; }" % {
 285             'hostkey' : 'master_known_hosts',
 286             'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostname),
 287             'token_path' : os.path.join(self._master.home_path, 'build.token'),
 288             'token' : server.shell_escape(self._master._master_token),
 289         }
 290
 291         syncfiles = "scp -p -o UserKnownHostsFile=%(hostkey)s %(files)s ." % {
 292             'hostkey' : 'master_known_hosts',
 293             'files' : ' '.join(files),
 294         }
 295         if self.build:
 296             syncfiles += " && tar xzf build.tar.gz"
 297         syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
 298         syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
 299
 300         cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
 301             'prk' : server.shell_escape(self._master_prk_name),
 302             'puk' : server.shell_escape(self._master_puk_name),
 303         }
 304
 305         slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s )" % {
 306             'waitmaster' : waitmaster,
 307             'syncfiles' : syncfiles,
 308             'cleanup' : cleanup,
 309             'kill_agent' : kill_agent,
 310             'launch_agent' : launch_agent,
 311             'home' : server.shell_escape(self.home_path),
 312         }
 313
 314         return cStringIO.StringIO(slavescript)
 315
 316     def _do_launch_build(self):
 317         script = "bash ./nepi-build.sh"
 318         if self._master_passphrase:
 319             script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
 320                 server.shell_escape(self._master_passphrase),
 321                 script
 322             )
 323         (out,err),proc = rspawn.remote_spawn(
 324             script,
 325             pidfile = 'build-pid',
 326             home = self.home_path,
 327             stdin = '/dev/null',
 328             stdout = 'buildlog',
 329             stderr = rspawn.STDOUT,
 330
 331             host = self.node.hostname,
 332             port = None,
 333             user = self.node.slicename,
 334             agent = None,
 335             ident_key = self.node.ident_path,
 336             server_key = self.node.server_key
 337             )
 338
 339         if proc.wait():
 340             raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
 341
 342
 343         pid = ppid = None
 344         delay = 1.0
 345         for i in xrange(5):
 346             pidtuple = rspawn.remote_check_pid(
 347                 os.path.join(self.home_path,'build-pid'),
 348                 host = self.node.hostname,
 349                 port = None,
 350                 user = self.node.slicename,
 351                 agent = None,
 352                 ident_key = self.node.ident_path,
 353                 server_key = self.node.server_key
 354                 )
 355
 356             if pidtuple:
 357                 pid, ppid = pidtuple
 358                 self._build_pid, self._build_ppid = pidtuple
 359                 break
 360             else:
 361                 time.sleep(delay)
 362                 delay = min(30,delay*1.2)
 363         else:
 364             raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
 365
 366         self._logger.info("Deploying %s", self)
 367
 368     def _do_wait_build(self):
 369         pid = self._build_pid
 370         ppid = self._build_ppid
 371
 372         if pid and ppid:
 373             delay = 1.0
 374             first = True
 375             while True:
 376                 status = rspawn.remote_status(
 377                     pid, ppid,
 378                     host = self.node.hostname,
 379                     port = None,
 380                     user = self.node.slicename,
 381                     agent = None,
 382                     ident_key = self.node.ident_path,
 383                     server_key = self.node.server_key
 384                     )
 385
 386                 if status is not rspawn.RUNNING:
 387                     self._build_pid = self._build_ppid = None
 388                     break
 389                 else:
 390                     if first:
 391                         self._logger.info("Waiting for %s to finish building %s", self,
 392                             "(build slave)" if self._master is not None else "(build master)")
 393
 394                         first = False
 395                     time.sleep(delay*(0.5+random.random()))
 396                     delay = min(30,delay*1.2)
 397
 398             # check build token
 399             slave_token = ""
 400             for i in xrange(3):
 401                 (out, err), proc = self._popen_ssh_command(
 402                     "cat %(token_path)s" % {
 403                         'token_path' : os.path.join(self.home_path, 'build.token'),
 404                     },
 405                     timeout = 120,
 406                     noerrors = True)
 407                 if not proc.wait() and out:
 408                     slave_token = out.strip()
 409
 410                 if slave_token:
 411                     break
 412                 else:
 413                     time.sleep(2)
 414
 415             if slave_token != self._master_token:
 416                 # Get buildlog for the error message
 417
 418                 (buildlog, err), proc = self._popen_ssh_command(
 419                     "cat %(buildlog)s" % {
 420                         'buildlog' : os.path.join(self.home_path, 'buildlog'),
 421                         'buildscript' : os.path.join(self.home_path, 'nepi-build.sh'),
 422                     },
 423                     timeout = 120,
 424                     noerrors = True)
 425
 426                 proc.wait()
 427
 428                 raise RuntimeError, "Failed to set up application %s: "\
 429                         "build failed, got wrong token from pid %s/%s "\
 430                         "(expected %r, got %r), see buildlog: %s" % (
 431                     self.home_path, pid, ppid, self._master_token, slave_token, buildlog)
 432
 433             self._logger.info("Built %s", self)
 434
 435     def _do_kill_build(self):
 436         pid = self._build_pid
 437         ppid = self._build_ppid
 438
 439         if pid and ppid:
 440             self._logger.info("Killing build of %s", self)
 441             rspawn.remote_kill(
 442                 pid, ppid,
 443                 host = self.node.hostname,
 444                 port = None,
 445                 user = self.node.slicename,
 446                 agent = None,
 447                 ident_key = self.node.ident_path
 448                 )
 449
 450
 451     def _do_build_master(self):
 452         if not self.sources and not self.build and not self.buildDepends:
 453             return None
 454
 455         if self.sources:
 456             sources = self.sources.split(' ')
 457
 458             # Copy all sources
 459             try:
 460                 self._popen_scp(
 461                     sources,
 462                     "%s@%s:%s" % (self.node.slicename, self.node.hostname,
 463                         os.path.join(self.home_path,'.'),)
 464                     )
 465             except RuntimeError, e:
 466                 raise RuntimeError, "Failed upload source file %r: %s %s" \
 467                         % (sources, e.args[0], e.args[1],)
 468
 469         buildscript = cStringIO.StringIO()
 470
 471         if self.buildDepends:
 472             # Install build dependencies
 473             buildscript.write(
 474                 "sudo -S yum -y install %(packages)s\n" % {
 475                     'packages' : self.buildDepends
 476                 }
 477             )
 478
 479
 480         if self.build:
 481             # Build sources
 482             buildscript.write(
 483                 "mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
 484                     'command' : self._replace_paths(self.build),
 485                     'home' : server.shell_escape(self.home_path),
 486                 }
 487             )
 488
 489             # Make archive
 490             buildscript.write("tar czf build.tar.gz build\n")
 491
 492         # Write token
 493         buildscript.write("echo %(master_token)s > build.token" % {
 494             'master_token' : server.shell_escape(self._master_token)
 495         })
 496
 497         buildscript.seek(0)
 498
 499         return buildscript
 500
 501     def _do_install(self):
 502         if self.install:
 503             self._logger.info("Installing %s", self)
 504
 505             # Install application
 506             try:
 507                 self._popen_ssh_command(
 508                     "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/{install,build}log >&2 && false )" % \
 509                         {
 510                         'command' : self._replace_paths(self.install),
 511                         'home' : server.shell_escape(self.home_path),
 512                         },
 513                     )
 514             except RuntimeError, e:
 515                 raise RuntimeError, "Failed install build sources: %s %s" % (e.args[0], e.args[1],)
 516
 517     def set_master(self, master):
 518         self._master = master
 519
 520     def install_keys(self, prk, puk, passphrase):
 521         # Install keys
 522         self._master_passphrase = passphrase
 523         self._master_prk = prk
 524         self._master_puk = puk
 525         self._master_prk_name = os.path.basename(prk.name)
 526         self._master_puk_name = os.path.basename(puk.name)
 527
 528     def _do_install_keys(self):
 529         prk = self._master_prk
 530         puk = self._master_puk
 531
 532         try:
 533             self._popen_scp(
 534                 [ prk.name, puk.name ],
 535                 '%s@%s:%s' % (self.node.slicename, self.node.hostname, self.home_path )
 536                 )
 537         except RuntimeError, e:
 538             raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
 539                     % (e.args[0], e.args[1],)
 540
 541         try:
 542             self._popen_scp(
 543                 cStringIO.StringIO('%s,%s %s\n' % (
 544                     self._master.node.hostname, socket.gethostbyname(self._master.node.hostname),
 545                     self._master.node.server_key)),
 546                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 547                     os.path.join(self.home_path,"master_known_hosts") )
 548                 )
 549         except RuntimeError, e:
 550             raise RuntimeError, "Failed to set up application deployment keys: %s %s" \
 551                     % (e.args[0], e.args[1],)
 552
 553         # No longer need'em
 554         self._master_prk = None
 555         self._master_puk = None
 556
 557     def cleanup(self):
 558         # make sure there's no leftover build processes
 559         self._do_kill_build()
 560
 561     @server.eintr_retry
 562     def _popen_scp(self, src, dst, retry = 3):
 563         while 1:
 564             try:
 565                 (out,err),proc = server.popen_scp(
 566                     src,
 567                     dst,
 568                     port = None,
 569                     agent = None,
 570                     ident_key = self.node.ident_path,
 571                     server_key = self.node.server_key
 572                     )
 573
 574                 if server.eintr_retry(proc.wait)():
 575                     raise RuntimeError, (out, err)
 576                 return (out, err), proc
 577             except:
 578                 if retry <= 0:
 579                     raise
 580                 else:
 581                     retry -= 1
 582
 583
 584     @server.eintr_retry
 585     def _popen_ssh_command(self, command, retry = 0, noerrors=False, timeout=None):
 586         (out,err),proc = server.popen_ssh_command(
 587             command,
 588             host = self.node.hostname,
 589             port = None,
 590             user = self.node.slicename,
 591             agent = None,
 592             ident_key = self.node.ident_path,
 593             server_key = self.node.server_key,
 594             timeout = timeout,
 595             retry = retry
 596             )
 597
 598         if server.eintr_retry(proc.wait)():
 599             if not noerrors:
 600                 raise RuntimeError, (out, err)
 601         return (out, err), proc
 602
 603 class Application(Dependency):
 604     """
 605     An application also has dependencies, but also a command to be ran and monitored.
 606
 607     It adds the output of that command as traces.
 608     """
 609
 610     TRACES = ('stdout','stderr','buildlog', 'output')
 611
 612     def __init__(self, api=None):
 613         super(Application,self).__init__(api)
 614
 615         # Attributes
 616         self.command = None
 617         self.sudo = False
 618
 619         self.stdin = None
 620         self.stdout = None
 621         self.stderr = None
 622         self.output = None
 623
 624         # Those are filled when the app is started
 625         #   Having both pid and ppid makes it harder
 626         #   for pid rollover to induce tracking mistakes
 627         self._started = False
 628         self._pid = None
 629         self._ppid = None
 630
 631         # Do not add to the python path of nodes
 632         self.add_to_path = False
 633
 634     def __str__(self):
 635         return "%s<command:%s%s>" % (
 636             self.__class__.__name__,
 637             "sudo " if self.sudo else "",
 638             self.command,
 639         )
 640
 641     def start(self):
 642         self._logger.info("Starting %s", self)
 643
 644         # Create shell script with the command
 645         # This way, complex commands and scripts can be ran seamlessly
 646         # sync files
 647         command = cStringIO.StringIO()
 648         command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
 649             ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
 650         ))
 651         command.write('export PATH=$PATH:%s\n' % (
 652             ':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
 653         ))
 654         if self.node.env:
 655             for envkey, envvals in self.node.env.iteritems():
 656                 for envval in envvals:
 657                     command.write('export %s=%s\n' % (envkey, envval))
 658         command.write(self.command)
 659         command.seek(0)
 660
 661         try:
 662             self._popen_scp(
 663                 command,
 664                 '%s@%s:%s' % (self.node.slicename, self.node.hostname,
 665                     os.path.join(self.home_path, "app.sh"))
 666                 )
 667         except RuntimeError, e:
 668             raise RuntimeError, "Failed to set up application: %s %s" \
 669                     % (e.args[0], e.args[1],)
 670
 671         # Start process in a "daemonized" way, using nohup and heavy
 672         # stdin/out redirection to avoid connection issues
 673         (out,err),proc = rspawn.remote_spawn(
 674             self._replace_paths("bash ./app.sh"),
 675
 676             pidfile = './pid',
 677             home = self.home_path,
 678             stdin = 'stdin' if self.stdin is not None else '/dev/null',
 679             stdout = 'stdout' if self.stdout else '/dev/null',
 680             stderr = 'stderr' if self.stderr else '/dev/null',
 681             sudo = self.sudo,
 682
 683             host = self.node.hostname,
 684             port = None,
 685             user = self.node.slicename,
 686             agent = None,
 687             ident_key = self.node.ident_path,
 688             server_key = self.node.server_key
 689             )
 690
 691         if proc.wait():
 692             raise RuntimeError, "Failed to set up application: %s %s" % (out,err,)
 693
 694         self._started = True
 695
 696     def recover(self):
 697         # Assuming the application is running on PlanetLab,
 698         # proper pidfiles should be present at the app's home path.
 699         # So we mark this application as started, and check the pidfiles
 700         self._started = True
 701         self.checkpid()
 702
 703     def checkpid(self):
 704         # Get PID/PPID
 705         # NOTE: wait a bit for the pidfile to be created
 706         if self._started and not self._pid or not self._ppid:
 707             pidtuple = rspawn.remote_check_pid(
 708                 os.path.join(self.home_path,'pid'),
 709                 host = self.node.hostname,
 710                 port = None,
 711                 user = self.node.slicename,
 712                 agent = None,
 713                 ident_key = self.node.ident_path,
 714                 server_key = self.node.server_key
 715                 )
 716
 717             if pidtuple:
 718                 self._pid, self._ppid = pidtuple
 719
 720     def status(self):
 721         self.checkpid()
 722         if not self._started:
 723             return AS.STATUS_NOT_STARTED
 724         elif not self._pid or not self._ppid:
 725             return AS.STATUS_NOT_STARTED
 726         else:
 727             status = rspawn.remote_status(
 728                 self._pid, self._ppid,
 729                 host = self.node.hostname,
 730                 port = None,
 731                 user = self.node.slicename,
 732                 agent = None,
 733                 ident_key = self.node.ident_path,
 734                 server_key = self.node.server_key
 735                 )
 736
 737             if status is rspawn.NOT_STARTED:
 738                 return AS.STATUS_NOT_STARTED
 739             elif status is rspawn.RUNNING:
 740                 return AS.STATUS_RUNNING
 741             elif status is rspawn.FINISHED:
 742                 return AS.STATUS_FINISHED
 743             else:
 744                 # WTF?
 745                 return AS.STATUS_NOT_STARTED
 746
 747     def kill(self):
 748         status = self.status()
 749         if status == AS.STATUS_RUNNING:
 750             # kill by ppid+pid - SIGTERM first, then try SIGKILL
 751             rspawn.remote_kill(
 752                 self._pid, self._ppid,
 753                 host = self.node.hostname,
 754                 port = None,
 755                 user = self.node.slicename,
 756                 agent = None,
 757                 ident_key = self.node.ident_path,
 758                 server_key = self.node.server_key,
 759                 sudo = self.sudo
 760                 )
 761             self._logger.info("Killed %s", self)
 762
 763
 764 class NepiDependency(Dependency):
 765     """
 766     This dependency adds nepi itself to the python path,
 767     so that you may run testbeds within PL nodes.
 768     """
 769
 770     # Class attribute holding a *weak* reference to the shared NEPI tar file
 771     # so that they may share it. Don't operate on the file itself, it would
 772     # be a mess, just use its path.
 773     _shared_nepi_tar = None
 774
 775     def __init__(self, api = None):
 776         super(NepiDependency, self).__init__(api)
 777
 778         self._tarball = None
 779
 780         self.depends = 'python python-ipaddr python-setuptools'
 781
 782         # our sources are in our ad-hoc tarball
 783         self.sources = self.tarball.name
 784
 785         tarname = os.path.basename(self.tarball.name)
 786
 787         # it's already built - just move the tarball into place
 788         self.build = "mv -f ${SOURCES}/%s ." % (tarname,)
 789
 790         # unpack it into sources, and we're done
 791         self.install = "tar xzf ${BUILD}/%s -C .." % (tarname,)
 792
 793     @property
 794     def tarball(self):
 795         if self._tarball is None:
 796             shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
 797             if shared_tar is not None:
 798                 self._tarball = shared_tar
 799             else:
 800                 # Build an ad-hoc tarball
 801                 # Prebuilt
 802                 import nepi
 803                 import tempfile
 804
 805                 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
 806
 807                 proc = subprocess.Popen(
 808                     ["tar", "czf", shared_tar.name,
 809                         '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
 810                         'nepi'],
 811                     stdout = open("/dev/null","w"),
 812                     stdin = open("/dev/null","r"))
 813
 814                 if proc.wait():
 815                     raise RuntimeError, "Failed to create nepi tarball"
 816
 817                 self._tarball = self._shared_nepi_tar = shared_tar
 818
 819         return self._tarball
 820
 821 class NS3Dependency(Dependency):
 822     """
 823     This dependency adds NS3 libraries to the library paths,
 824     so that you may run the NS3 testbed within PL nodes.
 825
 826     You'll also need the NepiDependency.
 827     """
 828
 829     def __init__(self, api = None):
 830         super(NS3Dependency, self).__init__(api)
 831
 832         self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip'
 833
 834         # We have to download the sources, untar, build...
 835         pybindgen_source_url = "http://yans.pl.sophia.inria.fr/trac/nepi/raw-attachment/wiki/WikiStart/pybindgen-r794.tar.gz"
 836         pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
 837         ns3_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/ns-3.11-nepi/archive/tip.tar.gz"
 838         passfd_source_url = "http://yans.pl.sophia.inria.fr/code/hgwebdir.cgi/python-passfd/archive/tip.tar.gz"
 839         self.build =(
 840             " ( "
 841             "  cd .. && "
 842             "  python -c 'import pygccxml, pybindgen, passfd' && "
 843             "  test -f lib/ns/_core.so && "
 844             "  test -f lib/ns/__init__.py && "
 845             "  test -f lib/ns/core.py && "
 846             "  test -f lib/libns3-core.so && "
 847             "  LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
 848             " ) || ( "
 849                 # Not working, rebuild
 850                      # Archive SHA1 sums to check
 851                      "echo '7158877faff2254e6c094bf18e6b4283cac19137  pygccxml-1.0.0.zip' > archive_sums.txt && "
 852                      "echo 'a18c2ccffd0df517bc37e2f3a2475092517c43f2  pybindgen-src.tar.gz' >> archive_sums.txt && "
 853                      " ( " # check existing files
 854                      " sha1sum -c archive_sums.txt && "
 855                      " test -f passfd-src.tar.gz && "
 856                      " test -f ns3-src.tar.gz "
 857                      " ) || ( " # nope? re-download
 858                      " rm -f pybindgen-src.zip pygccxml-1.0.0.zip passfd-src.tar.gz ns3-src.tar.gz && "
 859                      " wget -q -c -O pybindgen-src.tar.gz %(pybindgen_source_url)s && " # continue, to exploit the case when it has already been dl'ed
 860                      " wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
 861                      " wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
 862                      " wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
 863                      " sha1sum -c archive_sums.txt " # Check SHA1 sums when applicable
 864                      " ) && "
 865                      "unzip -n pygccxml-1.0.0.zip && "
 866                      "mkdir -p pybindgen-src && "
 867                      "mkdir -p ns3-src && "
 868                      "mkdir -p passfd-src && "
 869                      "tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
 870                      "tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
 871                      "tar xzf pybindgen-src.tar.gz --strip-components=1 -C pybindgen-src && "
 872                      "rm -rf target && "    # mv doesn't like unclean targets
 873                      "mkdir -p target && "
 874                      "cd pygccxml-1.0.0 && "
 875                      "rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
 876                      "python setup.py build && "
 877                      "python setup.py install --install-lib ${BUILD}/target && "
 878                      "python setup.py clean && "
 879                      "cd ../pybindgen-src && "
 880                      "export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
 881                      "./waf configure --prefix=${BUILD}/target -d release && "
 882                      "./waf && "
 883                      "./waf install && "
 884                      "./waf clean && "
 885                      "mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
 886                      "rm -rf ${BUILD}/target/lib && "
 887                      "cd ../passfd-src && "
 888                      "python setup.py build && "
 889                      "python setup.py install --install-lib ${BUILD}/target && "
 890                      "python setup.py clean && "
 891                      "cd ../ns3-src && "
 892                      "./waf configure --prefix=${BUILD}/target --with-pybindgen=../pybindgen-src -d release --disable-examples --disable-tests && "
 893                      "./waf &&"
 894                      "./waf install && "
 895                      "rm -f ${BUILD}/target/lib/*.so && "
 896                      "cp -a ${BUILD}/ns3-src/build/release/libns3*.so ${BUILD}/target/lib && "
 897                      "cp -a ${BUILD}/ns3-src/build/release/bindings/python/ns ${BUILD}/target/lib &&"
 898                      "./waf clean "
 899              " )"
 900                      % dict(
 901                         pybindgen_source_url = server.shell_escape(pybindgen_source_url),
 902                         pygccxml_source_url = server.shell_escape(pygccxml_source_url),
 903                         ns3_source_url = server.shell_escape(ns3_source_url),
 904                         passfd_source_url = server.shell_escape(passfd_source_url),
 905                      ))
 906
 907         # Just move ${BUILD}/target
 908         self.install = (
 909             " ( "
 910             "  cd .. && "
 911             "  python -c 'import pygccxml, pybindgen, passfd' && "
 912             "  test -f lib/ns/_core.so && "
 913             "  test -f lib/ns/__init__.py && "
 914             "  test -f lib/ns/core.py && "
 915             "  test -f lib/libns3-core.so && "
 916             "  LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
 917             " ) || ( "
 918                 # Not working, reinstall
 919                     "test -d ${BUILD}/target && "
 920                     "[[ \"x\" != \"x$(find ${BUILD}/target -mindepth 1 -print -quit)\" ]] &&"
 921                     "( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
 922                     "mv -f ${BUILD}/target/* ${SOURCES}"
 923             " )"
 924         )
 925
 926         # Set extra environment paths
 927         self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
 928         self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib"
 929
 930     @property
 931     def tarball(self):
 932         if self._tarball is None:
 933             shared_tar = self._shared_nepi_tar and self._shared_nepi_tar()
 934             if shared_tar is not None:
 935                 self._tarball = shared_tar
 936             else:
 937                 # Build an ad-hoc tarball
 938                 # Prebuilt
 939                 import nepi
 940                 import tempfile
 941
 942                 shared_tar = tempfile.NamedTemporaryFile(prefix='nepi-src-', suffix='.tar.gz')
 943
 944                 proc = subprocess.Popen(
 945                     ["tar", "czf", shared_tar.name,
 946                         '-C', os.path.join(os.path.dirname(os.path.dirname(nepi.__file__)),'.'),
 947                         'nepi'],
 948                     stdout = open("/dev/null","w"),
 949                     stdin = open("/dev/null","r"))
 950
 951                 if proc.wait():
 952                     raise RuntimeError, "Failed to create nepi tarball"
 953
 954                 self._tarball = self._shared_nepi_tar = shared_tar
 955
 956         return self._tarball
 957
 958 class YumDependency(Dependency):
 959     """
 960     This dependency is an internal helper class used to
 961     efficiently distribute yum-downloaded rpms.
 962
 963     It temporarily sets the yum cache as persistent in the
 964     build master, and installs all the required packages.
 965
 966     The rpm packages left in the yum cache are gathered and
 967     distributed by the underlying Dependency in an efficient
 968     manner. Build slaves will then install those rpms back in
 969     the cache before issuing the install command.
 970
 971     When packages have been installed already, nothing but an
 972     empty tar is distributed.
 973     """
 974
 975     # Class attribute holding a *weak* reference to the shared NEPI tar file
 976     # so that they may share it. Don't operate on the file itself, it would
 977     # be a mess, just use its path.
 978     _shared_nepi_tar = None
 979
 980     def _build_get(self):
 981         # canonical representation of dependencies
 982         depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
 983
 984         # download rpms and pack into a tar archive
 985         return (
 986             "sudo -S nice yum -y makecache && "
 987             "sudo -S sed -i -r 's/keepcache *= *0/keepcache=1/' /etc/yum.conf && "
 988             " ( ( "
 989                 "sudo -S nice yum -y install %s ; "
 990                 "rm -f ${BUILD}/packages.tar ; "
 991                 "tar -C /var/cache/yum -rf ${BUILD}/packages.tar $(cd /var/cache/yum ; find -iname '*.rpm')"
 992             " ) || /bin/true ) && "
 993             "sudo -S sed -i -r 's/keepcache *= *1/keepcache=0/' /etc/yum.conf && "
 994             "( sudo -S nice yum -y clean packages || /bin/true ) "
 995         ) % ( depends, )
 996     def _build_set(self, value):
 997         # ignore
 998         return
 999     build = property(_build_get, _build_set)
1000
1001     def _install_get(self):
1002         # canonical representation of dependencies
1003         depends = ' '.join( sorted( (self.depends or "").split(' ') ) )
1004
1005         # unpack cached rpms into yum cache, install, and cleanup
1006         return (
1007             "sudo -S tar -k --keep-newer-files -C /var/cache/yum -xf packages.tar && "
1008             "sudo -S nice yum -y install %s && "
1009             "( sudo -S nice yum -y clean packages || /bin/true ) "
1010         ) % ( depends, )
1011     def _install_set(self, value):
1012         # ignore
1013         return
1014     install = property(_install_get, _install_set)
1015
1016