test/testbeds/planetlab/execute.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import getpass
   5 from nepi.util.constants import ApplicationStatus as AS
   6 from nepi.testbeds import planetlab
   7 import os
   8 import shutil
   9 import tempfile
  10 import time
  11 import unittest
  12 import re
  13 import test_util
  14 import sys
  15
  16 class PlanetLabExecuteTestCase(unittest.TestCase):
  17     testbed_id = "planetlab"
  18     slicename = "inria_nepi"
  19     plchost = "nepiplc.pl.sophia.inria.fr"
  20
  21     host1 = "nepi1.pl.sophia.inria.fr"
  22     host2 = "nepi2.pl.sophia.inria.fr"
  23
  24     port_base = 2000 + (os.getpid() % 1000) * 13
  25
  26     def setUp(self):
  27         self.root_dir = tempfile.mkdtemp()
  28         self.__class__.port_base = self.port_base + 100
  29
  30     def tearDown(self):
  31         try:
  32             shutil.rmtree(self.root_dir)
  33         except:
  34             # retry
  35             time.sleep(0.1)
  36             shutil.rmtree(self.root_dir)
  37
  38     def make_instance(self):
  39         testbed_id = self.testbed_id
  40         slicename = self.slicename
  41         plchost = self.plchost
  42
  43         instance = planetlab.TestbedController()
  44         pl_ssh_key = os.environ.get(
  45             "PL_SSH_KEY",
  46             "%s/.ssh/id_rsa_planetlab" % (os.environ['HOME'],) )
  47         pl_user, pl_pwd = test_util.pl_auth()
  48
  49         instance.defer_configure("homeDirectory", self.root_dir)
  50         instance.defer_configure("slice", slicename)
  51         instance.defer_configure("sliceSSHKey", pl_ssh_key)
  52         instance.defer_configure("authUser", pl_user)
  53         instance.defer_configure("authPass", pl_pwd)
  54         instance.defer_configure("plcHost", plchost)
  55         instance.defer_configure("tapPortBase", self.port_base)
  56         instance.defer_configure("p2pDeployment", False) # it's interactive, we don't want it in tests
  57
  58         return instance
  59
  60     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
  61     def test_simple(self):
  62         instance = self.make_instance()
  63
  64         instance.defer_create(2, "Node")
  65         instance.defer_create_set(2, "hostname", self.host1)
  66         instance.defer_create(3, "Node")
  67         instance.defer_create_set(3, "hostname", self.host2)
  68         instance.defer_create(4, "NodeInterface")
  69         instance.defer_connect(2, "devs", 4, "node")
  70         instance.defer_create(5, "NodeInterface")
  71         instance.defer_connect(3, "devs", 5, "node")
  72         instance.defer_create(6, "Internet")
  73         instance.defer_connect(4, "inet", 6, "devs")
  74         instance.defer_connect(5, "inet", 6, "devs")
  75         instance.defer_create(7, "Application")
  76         instance.defer_create_set(7, "command", "ping -qc1 {#[GUID-5].addr[0].[Address]#}")
  77         instance.defer_add_trace(7, "stdout")
  78         instance.defer_add_trace(7, "stderr")
  79         instance.defer_connect(7, "node", 2, "apps")
  80
  81         comp_result = r"""PING .* \(.*\) \d*\(\d*\) bytes of data.
  82
  83 --- .* ping statistics ---
  84 1 packets transmitted, 1 received, 0% packet loss, time \d*ms.*
  85 """
  86
  87         try:
  88             instance.do_setup()
  89             instance.do_create()
  90             instance.do_connect_init()
  91             instance.do_connect_compl()
  92             instance.do_preconfigure()
  93
  94             # Manually replace netref
  95             instance.set(7, "command",
  96                 instance.get(7, "command")
  97                     .replace("{#[GUID-5].addr[0].[Address]#}",
  98                         instance.get_address(5, 0, "Address") )
  99             )
 100
 101             instance.do_configure()
 102
 103             instance.do_prestart()
 104             instance.start()
 105             while instance.status(7) != AS.STATUS_FINISHED:
 106                 time.sleep(0.5)
 107             ping_result = instance.trace(7, "stdout") or ""
 108             instance.stop()
 109         finally:
 110             instance.shutdown()
 111
 112         # asserts at the end, to make sure there's proper cleanup
 113         self.assertTrue(re.match(comp_result, ping_result, re.MULTILINE),
 114             "Unexpected trace:\n" + ping_result)
 115
 116     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
 117     def test_depends(self):
 118         instance = self.make_instance()
 119
 120         instance.defer_create(2, "Node")
 121         instance.defer_create_set(2, "hostname", self.host1)
 122         instance.defer_create(3, "NodeInterface")
 123         instance.defer_connect(2, "devs", 3, "node")
 124         instance.defer_create(4, "Internet")
 125         instance.defer_connect(3, "inet", 4, "devs")
 126         instance.defer_create(5, "Application")
 127         instance.defer_create_set(5, "command", "gfortran --version")
 128         instance.defer_create_set(5, "depends", "gcc-gfortran")
 129         instance.defer_add_trace(5, "stdout")
 130         instance.defer_add_trace(5, "stderr")
 131         instance.defer_connect(5, "node", 2, "apps")
 132
 133         try:
 134             instance.do_setup()
 135             instance.do_create()
 136             instance.do_connect_init()
 137             instance.do_connect_compl()
 138             instance.do_preconfigure()
 139             instance.do_configure()
 140
 141             instance.do_prestart()
 142             instance.start()
 143             while instance.status(5) != AS.STATUS_FINISHED:
 144                 time.sleep(0.5)
 145             ping_result = instance.trace(5, "stdout") or ""
 146             comp_result = r".*GNU Fortran \(GCC\).*"
 147             instance.stop()
 148         finally:
 149             instance.shutdown()
 150
 151         # asserts at the end, to make sure there's proper cleanup
 152         self.assertTrue(re.match(comp_result, ping_result, re.MULTILINE),
 153             "Unexpected trace:\n" + ping_result)
 154
 155     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
 156     def test_build(self):
 157         instance = self.make_instance()
 158
 159         instance.defer_create(2, "Node")
 160         instance.defer_create_set(2, "hostname", self.host1)
 161         instance.defer_create(3, "NodeInterface")
 162         instance.defer_connect(2, "devs", 3, "node")
 163         instance.defer_create(4, "Internet")
 164         instance.defer_connect(3, "inet", 4, "devs")
 165         instance.defer_create(10, "Application")
 166         instance.defer_create_set(10, "command", "./consts")
 167         instance.defer_create_set(10, "buildDepends", "gcc")
 168         instance.defer_create_set(10, "build", "gcc ${SOURCES}/consts.c -o consts")
 169         instance.defer_create_set(10, "install", "cp consts ${SOURCES}/consts")
 170         instance.defer_create_set(10, "sources", os.path.join(os.path.dirname(planetlab.__file__),'scripts','consts.c'))
 171         instance.defer_add_trace(10, "stdout")
 172         instance.defer_add_trace(10, "stderr")
 173         instance.defer_connect(10, "node", 2, "apps")
 174
 175         comp_result = \
 176 r""".*ETH_P_ALL = 0x[0-9a-fA-F]{8}
 177 ETH_P_IP = 0x[0-9a-fA-F]{8}
 178 TUNGETIFF = 0x[0-9a-fA-F]{8}
 179 TUNSETIFF = 0x[0-9a-fA-F]{8}
 180 IFF_NO_PI = 0x[0-9a-fA-F]{8}
 181 IFF_TAP = 0x[0-9a-fA-F]{8}
 182 IFF_TUN = 0x[0-9a-fA-F]{8}
 183 IFF_VNET_HDR = 0x[0-9a-fA-F]{8}
 184 TUN_PKT_STRIP = 0x[0-9a-fA-F]{8}
 185 IFHWADDRLEN = 0x[0-9a-fA-F]{8}
 186 IFNAMSIZ = 0x[0-9a-fA-F]{8}
 187 IFREQ_SZ = 0x[0-9a-fA-F]{8}
 188 FIONREAD = 0x[0-9a-fA-F]{8}.*
 189 """
 190
 191         try:
 192             instance.do_setup()
 193             instance.do_create()
 194             instance.do_connect_init()
 195             instance.do_connect_compl()
 196             instance.do_preconfigure()
 197             instance.do_configure()
 198
 199             instance.do_prestart()
 200             instance.start()
 201             while instance.status(10) != AS.STATUS_FINISHED:
 202                 time.sleep(0.5)
 203             ping_result = instance.trace(10, "stdout") or ""
 204             instance.stop()
 205         finally:
 206             instance.shutdown()
 207
 208         # asserts at the end, to make sure there's proper cleanup
 209         self.assertTrue(re.match(comp_result, ping_result, re.MULTILINE),
 210             "Unexpected trace:\n" + ping_result)
 211
 212     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
 213     def test_simple_vsys(self):
 214         instance = self.make_instance()
 215
 216         instance.defer_create(2, "Node")
 217         instance.defer_create_set(2, "hostname", self.host1)
 218         instance.defer_create(3, "NodeInterface")
 219         instance.defer_connect(2, "devs", 3, "node")
 220         instance.defer_create(4, "Internet")
 221         instance.defer_connect(3, "inet", 4, "devs")
 222         instance.defer_create(5, "TunInterface")
 223         instance.defer_add_address(5, "192.168.2.2", 24, False)
 224         instance.defer_connect(2, "devs", 5, "node")
 225         instance.defer_create(6, "Application")
 226         instance.defer_create_set(6, "command", """
 227 set -e
 228 netconfig help > /dev/null
 229 test -e /vsys/vif_up.in > /dev/null
 230 test -e /vsys/vif_up.out > /dev/null
 231 test -e /vsys/fd_tuntap.control > /dev/null
 232 echo 'OKIDOKI'
 233 """)
 234         instance.defer_create_set(6, "sudo", True) # only sudo has access to /vsys
 235         instance.defer_add_trace(6, "stdout")
 236         instance.defer_add_trace(6, "stderr")
 237         instance.defer_connect(6, "node", 2, "apps")
 238
 239         try:
 240             instance.do_setup()
 241             instance.do_create()
 242             instance.do_connect_init()
 243             instance.do_connect_compl()
 244             instance.do_preconfigure()
 245             instance.do_configure()
 246
 247             instance.do_prestart()
 248             instance.start()
 249             while instance.status(6) != AS.STATUS_FINISHED:
 250                 time.sleep(0.5)
 251             test_result = (instance.trace(6, "stdout") or "").strip()
 252             comp_result = "OKIDOKI"
 253             instance.stop()
 254         finally:
 255             instance.shutdown()
 256
 257         # asserts at the end, to make sure there's proper cleanup
 258         self.assertEqual(comp_result, test_result)
 259
 260     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
 261     def test_emulation(self):
 262         instance = self.make_instance()
 263
 264         instance.defer_create(2, "Node")
 265         instance.defer_create_set(2, "hostname", self.host1)
 266         instance.defer_create(3, "NodeInterface")
 267         instance.defer_connect(2, "devs", 3, "node")
 268         instance.defer_create(4, "Internet")
 269         instance.defer_connect(3, "inet", 4, "devs")
 270         instance.defer_create(7, "NetPipe")
 271         instance.defer_create_set(7, "mode", "CLIENT")
 272         instance.defer_create_set(7, "portList", "80")
 273         instance.defer_create_set(7, "bwOut", 12.0/1024.0) # 12kbps
 274         instance.defer_create_set(7, "bwIn", 64.0/1024.0) # 64kbps
 275         instance.defer_create_set(7, "plrOut", 0.01) # 1% plr outbound - high loss
 276         instance.defer_create_set(7, "plrIn", 0.001) # 0.1% plr inbound - regular loss
 277         instance.defer_create_set(7, "delayOut", int(1500 * 8 / (12.0/1024.0) / 1000)) # tx delay at 12kbps in ms
 278         instance.defer_create_set(7, "delayIn", int(1500 * 8 / (64.0/1024.0) / 1000)) # rx delay at 64kbps in ms
 279         instance.defer_add_trace(7, "netpipeStats")
 280         instance.defer_connect(2, "pipes", 7, "node")
 281         instance.defer_create(8, "Application")
 282         instance.defer_create_set(8, "command", "time wget -q -O /dev/null http://www.google.com/") # Fetch ~10kb
 283         instance.defer_add_trace(8, "stdout")
 284         instance.defer_add_trace(8, "stderr")
 285         instance.defer_connect(8, "node", 2, "apps")
 286
 287         try:
 288             instance.do_setup()
 289             instance.do_create()
 290             instance.do_connect_init()
 291             instance.do_connect_compl()
 292             instance.do_preconfigure()
 293             instance.do_configure()
 294
 295             instance.do_prestart()
 296             instance.start()
 297             while instance.status(8) != AS.STATUS_FINISHED:
 298                 time.sleep(0.5)
 299             test_result = (instance.trace(8, "stderr") or "").strip()
 300             comp_result = r".*real\s*(?P<min>[0-9]+)m(?P<sec>[0-9]+[.][0-9]+)s.*"
 301             netpipe_stats = instance.trace(7, "netpipeStats")
 302
 303             instance.stop()
 304         finally:
 305             instance.shutdown()
 306
 307         # asserts at the end, to make sure there's proper cleanup
 308         match = re.match(comp_result, test_result, re.MULTILINE)
 309         self.assertTrue(match, "Unexpected output: %s" % (test_result,))
 310
 311         minutes = int(match.group("min"))
 312         seconds = float(match.group("sec"))
 313         self.assertTrue((minutes * 60 + seconds) > 1.0, "Emulation not effective: %s" % (test_result,))
 314
 315         self.assertTrue(netpipe_stats, "Unavailable netpipe stats")
 316
 317     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
 318     def _pingtest(self, TunClass, ConnectionProto, Cipher):
 319         instance = self.make_instance()
 320
 321         instance.defer_create(2, "Node")
 322         instance.defer_create_set(2, "hostname", self.host1)
 323         instance.defer_create(3, "Node")
 324         instance.defer_create_set(3, "hostname", self.host2)
 325         instance.defer_create(4, "NodeInterface")
 326         instance.defer_connect(2, "devs", 4, "node")
 327         instance.defer_create(5, "Internet")
 328         instance.defer_connect(4, "inet", 5, "devs")
 329         instance.defer_create(6, "NodeInterface")
 330         instance.defer_connect(3, "devs", 6, "node")
 331         instance.defer_connect(6, "inet", 5, "devs")
 332         instance.defer_create(7, TunClass)
 333         instance.defer_create_set(7, "tun_cipher", Cipher)
 334         instance.defer_add_trace(7, "packets")
 335         instance.defer_add_address(7, "192.168.2.2", 24, False)
 336         instance.defer_connect(2, "devs", 7, "node")
 337         instance.defer_create(8, TunClass)
 338         instance.defer_create_set(8, "tun_cipher", Cipher)
 339         instance.defer_add_trace(8, "packets")
 340         instance.defer_add_address(8, "192.168.2.3", 24, False)
 341         instance.defer_connect(3, "devs", 8, "node")
 342         instance.defer_connect(7, ConnectionProto, 8, ConnectionProto)
 343         instance.defer_create(9, "Application")
 344         instance.defer_create_set(9, "command", "ping -qc1 {#[GUID-8].addr[0].[Address]#}")
 345         instance.defer_add_trace(9, "stdout")
 346         instance.defer_add_trace(9, "stderr")
 347         instance.defer_connect(9, "node", 2, "apps")
 348
 349         comp_result = r"""PING .* \(.*\) \d*\(\d*\) bytes of data.
 350
 351 --- .* ping statistics ---
 352 1 packets transmitted, 1 received, 0% packet loss, time \d*ms.*
 353 """
 354
 355         try:
 356             instance.do_setup()
 357             instance.do_create()
 358             instance.do_connect_init()
 359             instance.do_connect_compl()
 360             instance.do_preconfigure()
 361
 362             # Manually replace netref
 363             instance.set(9, "command",
 364                 instance.get(9, "command")
 365                     .replace("{#[GUID-8].addr[0].[Address]#}",
 366                         instance.get_address(8, 0, "Address") )
 367             )
 368
 369             instance.do_configure()
 370
 371             instance.do_prestart()
 372             instance.start()
 373             while instance.status(9) != AS.STATUS_FINISHED:
 374                 time.sleep(0.5)
 375             ping_result = instance.trace(9, "stdout") or ""
 376             packets1 = instance.trace(7, "packets") or ""
 377             packets2 = instance.trace(8, "packets") or ""
 378             instance.stop()
 379         finally:
 380             instance.shutdown()
 381
 382         # asserts at the end, to make sure there's proper cleanup
 383         self.assertTrue(re.match(comp_result, ping_result, re.MULTILINE),
 384             "Unexpected trace:\n%s\nPackets @ source:\n%s\nPackets @ target:\n%s" % (
 385                 ping_result,
 386                 packets1,
 387                 packets2))
 388
 389     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
 390     def test_tun_ping(self):
 391         self._pingtest("TunInterface", "tcp", "AES")
 392
 393     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
 394     def test_tun_ping_udp(self):
 395         self._pingtest("TunInterface", "udp", "AES")
 396
 397     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
 398     def test_tun_ping_gre(self):
 399         self._pingtest("TunInterface", "gre", "PLAIN")
 400
 401     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
 402     def test_tap_ping(self):
 403         self._pingtest("TapInterface", "tcp", "AES")
 404
 405     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
 406     def test_tap_ping_udp(self):
 407         self._pingtest("TapInterface", "udp", "AES")
 408
 409     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
 410     def test_tap_ping_gre(self):
 411         self._pingtest("TapInterface", "gre", "PLAIN")
 412
 413     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
 414     def test_nepi_depends(self):
 415         instance = self.make_instance()
 416
 417         instance.defer_create(2, "Node")
 418         instance.defer_create_set(2, "hostname", self.host1)
 419         instance.defer_create(3, "NodeInterface")
 420         instance.defer_connect(2, "devs", 3, "node")
 421         instance.defer_create(4, "Internet")
 422         instance.defer_connect(3, "inet", 4, "devs")
 423         instance.defer_create(5, "NepiDependency")
 424         instance.defer_connect(5, "node", 2, "deps")
 425         instance.defer_create(12, "Application")
 426         instance.defer_connect(12, "node", 2, "apps")
 427         instance.defer_create_set(12, "command", "python -c 'import nepi'")
 428         instance.defer_add_trace(12, "stderr")
 429
 430         try:
 431             instance.do_setup()
 432             instance.do_create()
 433             instance.do_connect_init()
 434             instance.do_connect_compl()
 435             instance.do_preconfigure()
 436             instance.do_configure()
 437
 438             instance.do_prestart()
 439             instance.start()
 440             while instance.status(12) != AS.STATUS_FINISHED:
 441                 time.sleep(0.5)
 442             ping_result = (instance.trace(12, "stderr") or "").strip()
 443             instance.stop()
 444         finally:
 445             instance.shutdown()
 446
 447         # asserts at the end, to make sure there's proper cleanup
 448         self.assertEqual(ping_result, "")
 449
 450     @test_util.skipUnless(test_util.pl_auth() is not None,
 451         "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
 452     @test_util.skipUnless(os.environ.get('NEPI_FULL_TESTS','').lower() in ('1','yes','true','on'),
 453         "Test is expensive, requires NEPI_FULL_TESTS=yes")
 454     def test_ns3_depends(self):
 455         instance = self.make_instance()
 456
 457         instance.defer_create(2, "Node")
 458         instance.defer_create_set(2, "hostname", self.host1)
 459         instance.defer_create(3, "NodeInterface")
 460         instance.defer_connect(2, "devs", 3, "node")
 461         instance.defer_create(4, "Internet")
 462         instance.defer_connect(3, "inet", 4, "devs")
 463         instance.defer_create(5, "NepiDependency")
 464         instance.defer_connect(5, "node", 2, "deps")
 465         instance.defer_create(6, "NS3Dependency")
 466         instance.defer_connect(6, "node", 2, "deps")
 467         instance.defer_create(12, "Application")
 468         instance.defer_connect(12, "node", 2, "apps")
 469         instance.defer_create_set(12, "command", "python -c 'import nepi.testbeds.ns3.execute ; tb = nepi.testbeds.ns3.execute.TestbedController() ; mod = tb._configure_ns3_module()'")
 470         instance.defer_add_trace(12, "stderr")
 471
 472         try:
 473             instance.do_setup()
 474             instance.do_create()
 475             instance.do_connect_init()
 476             instance.do_connect_compl()
 477             instance.do_preconfigure()
 478             instance.do_configure()
 479
 480             instance.do_prestart()
 481             instance.start()
 482             while instance.status(12) != AS.STATUS_FINISHED:
 483                 time.sleep(0.5)
 484             ping_result = (instance.trace(12, "stderr") or "").strip()
 485             instance.stop()
 486         finally:
 487             instance.shutdown()
 488
 489         # asserts at the end, to make sure there's proper cleanup
 490         self.assertEqual(ping_result, "")
 491
 492     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
 493     def test_discovery(self):
 494         instance = self.make_instance()
 495
 496         instance.defer_create(2, "Node")
 497         instance.defer_create_set(2, "operatingSystem", "f12")
 498         instance.defer_create(3, "Node")
 499         instance.defer_create_set(3, "operatingSystem", "f12")
 500         instance.defer_create(4, "NodeInterface")
 501         instance.defer_connect(2, "devs", 4, "node")
 502         instance.defer_create(5, "NodeInterface")
 503         instance.defer_connect(3, "devs", 5, "node")
 504         instance.defer_create(6, "Internet")
 505         instance.defer_connect(4, "inet", 6, "devs")
 506         instance.defer_connect(5, "inet", 6, "devs")
 507         instance.defer_create(7, "Application")
 508         instance.defer_create_set(7, "command", "ping -qc1 {#[GUID-5].addr[0].[Address]#}")
 509         instance.defer_add_trace(7, "stdout")
 510         instance.defer_add_trace(7, "stderr")
 511         instance.defer_connect(7, "node", 2, "apps")
 512
 513         comp_result = r"""PING .* \(.*\) \d*\(\d*\) bytes of data.
 514
 515 --- .* ping statistics ---
 516 1 packets transmitted, 1 received, 0% packet loss, time \d*ms.*
 517 """
 518
 519         try:
 520             instance.do_setup()
 521             instance.do_create()
 522             instance.do_connect_init()
 523             instance.do_connect_compl()
 524             instance.do_preconfigure()
 525
 526             # Manually replace netref
 527             instance.set(7, "command",
 528                 instance.get(7, "command")
 529                     .replace("{#[GUID-5].addr[0].[Address]#}",
 530                         instance.get_address(5, 0, "Address") )
 531             )
 532
 533             instance.do_configure()
 534
 535             instance.do_prestart()
 536             instance.start()
 537             while instance.status(7) != AS.STATUS_FINISHED:
 538                 time.sleep(0.5)
 539             ping_result = instance.trace(7, "stdout") or ""
 540             instance.stop()
 541         finally:
 542             instance.shutdown()
 543
 544         # asserts at the end, to make sure there's proper cleanup
 545         self.assertTrue(re.match(comp_result, ping_result, re.MULTILINE),
 546             "Unexpected trace:\n" + ping_result)
 547
 548
 549
 550 if __name__ == '__main__':
 551     unittest.main()
 552