TUN/TAP filters, initial version, with tests.
[nepi.git] / test / testbeds / planetlab / execute.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 import getpass
5 from nepi.util.constants import ApplicationStatus as AS
6 from nepi.testbeds import planetlab
7 import os
8 import shutil
9 import tempfile
10 import time
11 import unittest
12 import re
13 import test_util
14 import sys
15
16 class PlanetLabExecuteTestCase(unittest.TestCase):
17     testbed_id = "planetlab"
18     slicename = "inria_nepi"
19     plchost = "nepiplc.pl.sophia.inria.fr"
20     
21     host1 = "nepi1.pl.sophia.inria.fr"
22     host2 = "nepi2.pl.sophia.inria.fr"
23     
24     port_base = 2000 + (os.getpid() % 1000) * 13
25     
26     PLR50_PY = os.path.join(
27         os.path.dirname(planetlab.__file__), 
28         'scripts',
29         'plr50.py')
30     PLR50_C = os.path.join(
31         os.path.dirname(planetlab.__file__), 
32         'scripts',
33         'plr50.c')
34     
35     def setUp(self):
36         self.root_dir = tempfile.mkdtemp()
37         self.__class__.port_base = self.port_base + 100
38         
39     def tearDown(self):
40         try:
41             shutil.rmtree(self.root_dir)
42         except:
43             # retry
44             time.sleep(0.1)
45             shutil.rmtree(self.root_dir)
46
47     def make_instance(self):
48         testbed_id = self.testbed_id
49         slicename = self.slicename
50         plchost = self.plchost
51         
52         instance = planetlab.TestbedController()
53         pl_ssh_key = os.environ.get(
54             "PL_SSH_KEY",
55             "%s/.ssh/id_rsa_planetlab" % (os.environ['HOME'],) )
56         pl_user, pl_pwd = test_util.pl_auth()
57         
58         instance.defer_configure("homeDirectory", self.root_dir)
59         instance.defer_configure("slice", slicename)
60         instance.defer_configure("sliceSSHKey", pl_ssh_key)
61         instance.defer_configure("authUser", pl_user)
62         instance.defer_configure("authPass", pl_pwd)
63         instance.defer_configure("plcHost", plchost)
64         instance.defer_configure("tapPortBase", self.port_base)
65         instance.defer_configure("p2pDeployment", False) # it's interactive, we don't want it in tests
66         
67         return instance
68
69     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
70     def test_simple(self):
71         instance = self.make_instance()
72         
73         instance.defer_create(2, "Node")
74         instance.defer_create_set(2, "hostname", self.host1)
75         instance.defer_create(3, "Node")
76         instance.defer_create_set(3, "hostname", self.host2)
77         instance.defer_create(4, "NodeInterface")
78         instance.defer_connect(2, "devs", 4, "node")
79         instance.defer_create(5, "NodeInterface")
80         instance.defer_connect(3, "devs", 5, "node")
81         instance.defer_create(6, "Internet")
82         instance.defer_connect(4, "inet", 6, "devs")
83         instance.defer_connect(5, "inet", 6, "devs")
84         instance.defer_create(7, "Application")
85         instance.defer_create_set(7, "command", "ping -qc1 {#[GUID-5].addr[0].[Address]#}")
86         instance.defer_add_trace(7, "stdout")
87         instance.defer_add_trace(7, "stderr")
88         instance.defer_connect(7, "node", 2, "apps")
89
90         comp_result = r"""PING .* \(.*\) \d*\(\d*\) bytes of data.
91
92 --- .* ping statistics ---
93 1 packets transmitted, 1 received, 0% packet loss, time \d*ms.*
94 """
95
96         try:
97             instance.do_setup()
98             instance.do_create()
99             instance.do_connect_init()
100             instance.do_connect_compl()
101             instance.do_preconfigure()
102             
103             # Manually replace netref
104             instance.set(7, "command",
105                 instance.get(7, "command")
106                     .replace("{#[GUID-5].addr[0].[Address]#}", 
107                         instance.get_address(5, 0, "Address") )
108             )
109
110             instance.do_configure()
111             
112             instance.do_prestart()
113             instance.start()
114             while instance.status(7) != AS.STATUS_FINISHED:
115                 time.sleep(0.5)
116             ping_result = instance.trace(7, "stdout") or ""
117             instance.stop()
118         finally:
119             instance.shutdown()
120
121         # asserts at the end, to make sure there's proper cleanup
122         self.assertTrue(re.match(comp_result, ping_result, re.MULTILINE),
123             "Unexpected trace:\n" + ping_result)
124         
125     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
126     def test_depends(self):
127         instance = self.make_instance()
128         
129         instance.defer_create(2, "Node")
130         instance.defer_create_set(2, "hostname", self.host1)
131         instance.defer_create(3, "NodeInterface")
132         instance.defer_connect(2, "devs", 3, "node")
133         instance.defer_create(4, "Internet")
134         instance.defer_connect(3, "inet", 4, "devs")
135         instance.defer_create(5, "Application")
136         instance.defer_create_set(5, "command", "gfortran --version")
137         instance.defer_create_set(5, "depends", "gcc-gfortran")
138         instance.defer_add_trace(5, "stdout")
139         instance.defer_add_trace(5, "stderr")
140         instance.defer_connect(5, "node", 2, "apps")
141
142         try:
143             instance.do_setup()
144             instance.do_create()
145             instance.do_connect_init()
146             instance.do_connect_compl()
147             instance.do_preconfigure()
148             instance.do_configure()
149             
150             instance.do_prestart()
151             instance.start()
152             while instance.status(5) != AS.STATUS_FINISHED:
153                 time.sleep(0.5)
154             ping_result = instance.trace(5, "stdout") or ""
155             comp_result = r".*GNU Fortran \(GCC\).*"
156             instance.stop()
157         finally:
158             instance.shutdown()
159
160         # asserts at the end, to make sure there's proper cleanup
161         self.assertTrue(re.match(comp_result, ping_result, re.MULTILINE),
162             "Unexpected trace:\n" + ping_result)
163         
164     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
165     def test_build(self):
166         instance = self.make_instance()
167         
168         instance.defer_create(2, "Node")
169         instance.defer_create_set(2, "hostname", self.host1)
170         instance.defer_create(3, "NodeInterface")
171         instance.defer_connect(2, "devs", 3, "node")
172         instance.defer_create(4, "Internet")
173         instance.defer_connect(3, "inet", 4, "devs")
174         instance.defer_create(10, "Application")
175         instance.defer_create_set(10, "command", "./consts")
176         instance.defer_create_set(10, "buildDepends", "gcc")
177         instance.defer_create_set(10, "build", "gcc ${SOURCES}/consts.c -o consts")
178         instance.defer_create_set(10, "install", "cp consts ${SOURCES}/consts")
179         instance.defer_create_set(10, "sources", os.path.join(os.path.dirname(planetlab.__file__),'scripts','consts.c'))
180         instance.defer_add_trace(10, "stdout")
181         instance.defer_add_trace(10, "stderr")
182         instance.defer_connect(10, "node", 2, "apps")
183
184         comp_result = \
185 r""".*ETH_P_ALL = 0x[0-9a-fA-F]{8}
186 ETH_P_IP = 0x[0-9a-fA-F]{8}
187 TUNGETIFF = 0x[0-9a-fA-F]{8}
188 TUNSETIFF = 0x[0-9a-fA-F]{8}
189 IFF_NO_PI = 0x[0-9a-fA-F]{8}
190 IFF_TAP = 0x[0-9a-fA-F]{8}
191 IFF_TUN = 0x[0-9a-fA-F]{8}
192 IFF_VNET_HDR = 0x[0-9a-fA-F]{8}
193 TUN_PKT_STRIP = 0x[0-9a-fA-F]{8}
194 IFHWADDRLEN = 0x[0-9a-fA-F]{8}
195 IFNAMSIZ = 0x[0-9a-fA-F]{8}
196 IFREQ_SZ = 0x[0-9a-fA-F]{8}
197 FIONREAD = 0x[0-9a-fA-F]{8}.*
198 """
199
200         try:
201             instance.do_setup()
202             instance.do_create()
203             instance.do_connect_init()
204             instance.do_connect_compl()
205             instance.do_preconfigure()
206             instance.do_configure()
207             
208             instance.do_prestart()
209             instance.start()
210             while instance.status(10) != AS.STATUS_FINISHED:
211                 time.sleep(0.5)
212             ping_result = instance.trace(10, "stdout") or ""
213             instance.stop()
214         finally:
215             instance.shutdown()
216
217         # asserts at the end, to make sure there's proper cleanup
218         self.assertTrue(re.match(comp_result, ping_result, re.MULTILINE),
219             "Unexpected trace:\n" + ping_result)
220         
221     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
222     def test_simple_vsys(self):
223         instance = self.make_instance()
224         
225         instance.defer_create(2, "Node")
226         instance.defer_create_set(2, "hostname", self.host1)
227         instance.defer_create(3, "NodeInterface")
228         instance.defer_connect(2, "devs", 3, "node")
229         instance.defer_create(4, "Internet")
230         instance.defer_connect(3, "inet", 4, "devs")
231         instance.defer_create(5, "TunInterface")
232         instance.defer_add_address(5, "192.168.2.2", 24, False)
233         instance.defer_connect(2, "devs", 5, "node")
234         instance.defer_create(6, "Application")
235         instance.defer_create_set(6, "command", """
236 set -e
237 netconfig help > /dev/null
238 test -e /vsys/vif_up.in > /dev/null
239 test -e /vsys/vif_up.out > /dev/null
240 test -e /vsys/fd_tuntap.control > /dev/null
241 echo 'OKIDOKI'
242 """)
243         instance.defer_create_set(6, "sudo", True) # only sudo has access to /vsys
244         instance.defer_add_trace(6, "stdout")
245         instance.defer_add_trace(6, "stderr")
246         instance.defer_connect(6, "node", 2, "apps")
247
248         try:
249             instance.do_setup()
250             instance.do_create()
251             instance.do_connect_init()
252             instance.do_connect_compl()
253             instance.do_preconfigure()
254             instance.do_configure()
255             
256             instance.do_prestart()
257             instance.start()
258             while instance.status(6) != AS.STATUS_FINISHED:
259                 time.sleep(0.5)
260             test_result = (instance.trace(6, "stdout") or "").strip()
261             comp_result = "OKIDOKI"
262             instance.stop()
263         finally:
264             instance.shutdown()
265
266         # asserts at the end, to make sure there's proper cleanup
267         self.assertEqual(comp_result, test_result)
268
269     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
270     def test_emulation(self):
271         instance = self.make_instance()
272         
273         instance.defer_create(2, "Node")
274         instance.defer_create_set(2, "hostname", self.host1)
275         instance.defer_create(3, "NodeInterface")
276         instance.defer_connect(2, "devs", 3, "node")
277         instance.defer_create(4, "Internet")
278         instance.defer_connect(3, "inet", 4, "devs")
279         instance.defer_create(7, "NetPipe")
280         instance.defer_create_set(7, "mode", "CLIENT")
281         instance.defer_create_set(7, "portList", "80")
282         instance.defer_create_set(7, "bwOut", 12.0/1024.0) # 12kbps
283         instance.defer_create_set(7, "bwIn", 64.0/1024.0) # 64kbps
284         instance.defer_create_set(7, "plrOut", 0.01) # 1% plr outbound - high loss
285         instance.defer_create_set(7, "plrIn", 0.001) # 0.1% plr inbound - regular loss
286         instance.defer_create_set(7, "delayOut", int(1500 * 8 / (12.0/1024.0) / 1000)) # tx delay at 12kbps in ms
287         instance.defer_create_set(7, "delayIn", int(1500 * 8 / (64.0/1024.0) / 1000)) # rx delay at 64kbps in ms
288         instance.defer_add_trace(7, "netpipeStats")
289         instance.defer_connect(2, "pipes", 7, "node")
290         instance.defer_create(8, "Application")
291         instance.defer_create_set(8, "command", "time wget -q -O /dev/null http://www.google.com/") # Fetch ~10kb
292         instance.defer_add_trace(8, "stdout")
293         instance.defer_add_trace(8, "stderr")
294         instance.defer_connect(8, "node", 2, "apps")
295
296         try:
297             instance.do_setup()
298             instance.do_create()
299             instance.do_connect_init()
300             instance.do_connect_compl()
301             instance.do_preconfigure()
302             instance.do_configure()
303             
304             instance.do_prestart()
305             instance.start()
306             while instance.status(8) != AS.STATUS_FINISHED:
307                 time.sleep(0.5)
308             test_result = (instance.trace(8, "stderr") or "").strip()
309             comp_result = r".*real\s*(?P<min>[0-9]+)m(?P<sec>[0-9]+[.][0-9]+)s.*"
310             netpipe_stats = instance.trace(7, "netpipeStats")
311             
312             instance.stop()
313         finally:
314             instance.shutdown()
315
316         # asserts at the end, to make sure there's proper cleanup
317         match = re.match(comp_result, test_result, re.MULTILINE)
318         self.assertTrue(match, "Unexpected output: %s" % (test_result,))
319         
320         minutes = int(match.group("min"))
321         seconds = float(match.group("sec"))
322         self.assertTrue((minutes * 60 + seconds) > 1.0, "Emulation not effective: %s" % (test_result,))
323
324         self.assertTrue(netpipe_stats, "Unavailable netpipe stats")
325
326     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
327     def _pingtest(self, TunClass, ConnectionProto, Cipher, Filter1=None, Filter2=None):
328         instance = self.make_instance()
329         
330         instance.defer_create(2, "Node")
331         instance.defer_create_set(2, "hostname", self.host1)
332         instance.defer_create(3, "Node")
333         instance.defer_create_set(3, "hostname", self.host2)
334         instance.defer_create(4, "NodeInterface")
335         instance.defer_connect(2, "devs", 4, "node")
336         instance.defer_create(5, "Internet")
337         instance.defer_connect(4, "inet", 5, "devs")
338         instance.defer_create(6, "NodeInterface")
339         instance.defer_connect(3, "devs", 6, "node")
340         instance.defer_connect(6, "inet", 5, "devs")
341         instance.defer_create(7, TunClass)
342         instance.defer_create_set(7, "tun_cipher", Cipher)
343         instance.defer_add_trace(7, "packets")
344         instance.defer_add_address(7, "192.168.2.2", 24, False)
345         instance.defer_connect(2, "devs", 7, "node")
346         instance.defer_create(8, TunClass)
347         instance.defer_create_set(8, "tun_cipher", Cipher)
348         instance.defer_add_trace(8, "packets")
349         instance.defer_add_address(8, "192.168.2.3", 24, False)
350         instance.defer_connect(3, "devs", 8, "node")
351         instance.defer_create(9, "Application")
352         instance.defer_create_set(9, "command", "ping -qc10 {#[GUID-8].addr[0].[Address]#}")
353         instance.defer_add_trace(9, "stdout")
354         instance.defer_add_trace(9, "stderr")
355         instance.defer_connect(9, "node", 2, "apps")
356         
357         if Filter1:
358             instance.defer_create(10, "TunFilter")
359             instance.defer_create_set(10, "module", Filter1)
360             instance.defer_connect(7, "fd->", 10, "->fd")
361             
362         if Filter2:
363             instance.defer_create(11, "TunFilter")
364             instance.defer_create_set(11, "module", Filter2)
365             instance.defer_connect(8, "fd->", 11, "->fd")
366
367         if Filter1 and Filter2:
368             plr = "[5-9][0-9]"
369         elif Filter1 or Filter2:
370             plr = "[3-9][0-9]"
371         else:
372             plr = "0"
373        
374         instance.defer_connect(
375             (10 if Filter1 else 7), ConnectionProto, 
376             (11 if Filter2 else 8), ConnectionProto)
377
378         comp_result = r"""PING .* \(.*\) \d*\(\d*\) bytes of data.
379
380 --- .* ping statistics ---
381 10 packets transmitted, [0-9]+ received, %s%% packet loss, time \d*ms.*
382 """ % (plr,)
383
384         try:
385             instance.do_setup()
386             instance.do_create()
387             instance.do_connect_init()
388             instance.do_connect_compl()
389             instance.do_preconfigure()
390             
391             # Manually replace netref
392             instance.set(9, "command",
393                 instance.get(9, "command")
394                     .replace("{#[GUID-8].addr[0].[Address]#}", 
395                         instance.get_address(8, 0, "Address") )
396             )
397             
398             instance.do_configure()
399             
400             instance.do_prestart()
401             instance.start()
402             while instance.status(9) != AS.STATUS_FINISHED:
403                 time.sleep(0.5)
404             ping_result = instance.trace(9, "stdout") or ""
405             packets1 = instance.trace(7, "packets") or ""
406             packets2 = instance.trace(8, "packets") or ""
407             instance.stop()
408         finally:
409             instance.shutdown()
410
411         # asserts at the end, to make sure there's proper cleanup
412         self.assertTrue(re.match(comp_result, ping_result, re.MULTILINE),
413             "Unexpected trace:\n%s\nPackets @ source:\n%s\nPackets @ target:\n%s" % (
414                 ping_result,
415                 packets1,
416                 packets2))
417
418     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
419     def test_tun_ping(self):
420         self._pingtest("TunInterface", "tcp", "AES")
421
422     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
423     def test_tun_ping_udp(self):
424         self._pingtest("TunInterface", "udp", "AES")
425
426     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
427     def test_tun_ping_gre(self):
428         self._pingtest("TunInterface", "gre", "PLAIN")
429
430     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
431     def test_tap_ping(self):
432         self._pingtest("TapInterface", "tcp", "AES")
433
434     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
435     def test_tap_ping_udp(self):
436         self._pingtest("TapInterface", "udp", "AES")
437
438     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
439     def test_tap_ping_gre(self):
440         self._pingtest("TapInterface", "gre", "PLAIN")
441
442     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
443     def test_tap_ping_udp_loss1_py(self):
444         self._pingtest("TapInterface", "udp", "AES", self.PLR50_PY)
445
446     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
447     def test_tap_ping_udp_loss2_py(self):
448         self._pingtest("TapInterface", "udp", "AES", self.PLR50_PY, self.PLR50_PY)
449
450     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
451     def test_tap_ping_udp_loss1_c(self):
452         self._pingtest("TapInterface", "udp", "AES", self.PLR50_C)
453
454     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
455     def test_tap_ping_udp_loss2_c(self):
456         self._pingtest("TapInterface", "udp", "AES", self.PLR50_C, self.PLR50_C)
457
458     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
459     def test_nepi_depends(self):
460         instance = self.make_instance()
461         
462         instance.defer_create(2, "Node")
463         instance.defer_create_set(2, "hostname", self.host1)
464         instance.defer_create(3, "NodeInterface")
465         instance.defer_connect(2, "devs", 3, "node")
466         instance.defer_create(4, "Internet")
467         instance.defer_connect(3, "inet", 4, "devs")
468         instance.defer_create(5, "NepiDependency")
469         instance.defer_connect(5, "node", 2, "deps")
470         instance.defer_create(12, "Application")
471         instance.defer_connect(12, "node", 2, "apps")
472         instance.defer_create_set(12, "command", "python -c 'import nepi'")
473         instance.defer_add_trace(12, "stderr")
474
475         try:
476             instance.do_setup()
477             instance.do_create()
478             instance.do_connect_init()
479             instance.do_connect_compl()
480             instance.do_preconfigure()
481             instance.do_configure()
482             
483             instance.do_prestart()
484             instance.start()
485             while instance.status(12) != AS.STATUS_FINISHED:
486                 time.sleep(0.5)
487             ping_result = (instance.trace(12, "stderr") or "").strip()
488             instance.stop()
489         finally:
490             instance.shutdown()
491         
492         # asserts at the end, to make sure there's proper cleanup
493         self.assertEqual(ping_result, "")
494
495     @test_util.skipUnless(test_util.pl_auth() is not None, 
496         "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
497     @test_util.skipUnless(os.environ.get('NEPI_FULL_TESTS','').lower() in ('1','yes','true','on'),
498         "Test is expensive, requires NEPI_FULL_TESTS=yes")
499     def test_ns3_depends(self):
500         instance = self.make_instance()
501         
502         instance.defer_create(2, "Node")
503         instance.defer_create_set(2, "hostname", self.host1)
504         instance.defer_create(3, "NodeInterface")
505         instance.defer_connect(2, "devs", 3, "node")
506         instance.defer_create(4, "Internet")
507         instance.defer_connect(3, "inet", 4, "devs")
508         instance.defer_create(5, "NepiDependency")
509         instance.defer_connect(5, "node", 2, "deps")
510         instance.defer_create(6, "NS3Dependency")
511         instance.defer_connect(6, "node", 2, "deps")
512         instance.defer_create(12, "Application")
513         instance.defer_connect(12, "node", 2, "apps")
514         instance.defer_create_set(12, "command", "python -c 'import nepi.testbeds.ns3.execute ; tb = nepi.testbeds.ns3.execute.TestbedController() ; mod = tb._configure_ns3_module()'")
515         instance.defer_add_trace(12, "stderr")
516
517         try:
518             instance.do_setup()
519             instance.do_create()
520             instance.do_connect_init()
521             instance.do_connect_compl()
522             instance.do_preconfigure()
523             instance.do_configure()
524             
525             instance.do_prestart()
526             instance.start()
527             while instance.status(12) != AS.STATUS_FINISHED:
528                 time.sleep(0.5)
529             ping_result = (instance.trace(12, "stderr") or "").strip()
530             instance.stop()
531         finally:
532             instance.shutdown()
533         
534         # asserts at the end, to make sure there's proper cleanup
535         self.assertEqual(ping_result, "")
536
537     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
538     def test_discovery(self):
539         instance = self.make_instance()
540         
541         instance.defer_create(2, "Node")
542         instance.defer_create_set(2, "operatingSystem", "f12")
543         instance.defer_create(3, "Node")
544         instance.defer_create_set(3, "operatingSystem", "f12")
545         instance.defer_create(4, "NodeInterface")
546         instance.defer_connect(2, "devs", 4, "node")
547         instance.defer_create(5, "NodeInterface")
548         instance.defer_connect(3, "devs", 5, "node")
549         instance.defer_create(6, "Internet")
550         instance.defer_connect(4, "inet", 6, "devs")
551         instance.defer_connect(5, "inet", 6, "devs")
552         instance.defer_create(7, "Application")
553         instance.defer_create_set(7, "command", "ping -qc1 {#[GUID-5].addr[0].[Address]#}")
554         instance.defer_add_trace(7, "stdout")
555         instance.defer_add_trace(7, "stderr")
556         instance.defer_connect(7, "node", 2, "apps")
557
558         comp_result = r"""PING .* \(.*\) \d*\(\d*\) bytes of data.
559
560 --- .* ping statistics ---
561 1 packets transmitted, 1 received, 0% packet loss, time \d*ms.*
562 """
563
564         try:
565             instance.do_setup()
566             instance.do_create()
567             instance.do_connect_init()
568             instance.do_connect_compl()
569             instance.do_preconfigure()
570             
571             # Manually replace netref
572             instance.set(7, "command",
573                 instance.get(7, "command")
574                     .replace("{#[GUID-5].addr[0].[Address]#}", 
575                         instance.get_address(5, 0, "Address") )
576             )
577
578             instance.do_configure()
579             
580             instance.do_prestart()
581             instance.start()
582             while instance.status(7) != AS.STATUS_FINISHED:
583                 time.sleep(0.5)
584             ping_result = instance.trace(7, "stdout") or ""
585             instance.stop()
586         finally:
587             instance.shutdown()
588
589         # asserts at the end, to make sure there's proper cleanup
590         self.assertTrue(re.match(comp_result, ping_result, re.MULTILINE),
591             "Unexpected trace:\n" + ping_result)
592         
593         
594
595 if __name__ == '__main__':
596     unittest.main()
597