PL execute test
[nepi.git] / test / testbeds / planetlab / execute.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 import getpass
5 from nepi.util.constants import ApplicationStatus as AS
6 from nepi.testbeds import planetlab
7 import os
8 import shutil
9 import tempfile
10 import time
11 import unittest
12 import re
13 import test_util
14 import sys
15
16 class PlanetLabExecuteTestCase(unittest.TestCase):
17     testbed_id = "planetlab"
18     slicename = "inria_nepi"
19     plchost = "nepiplc.pl.sophia.inria.fr"
20     
21     host1 = "nepi1.pl.sophia.inria.fr"
22     host2 = "nepi2.pl.sophia.inria.fr"
23     
24     port_base = 2000 + (os.getpid() % 1000) * 13
25     
26     PLR50_PY = os.path.join(
27         os.path.dirname(planetlab.__file__), 
28         'scripts',
29         'plr50.py')
30     PLR50_C = os.path.join(
31         os.path.dirname(planetlab.__file__), 
32         'scripts',
33         'plr50.c')
34     
35     def setUp(self):
36         self.root_dir = tempfile.mkdtemp()
37         self.__class__.port_base = self.port_base + 100
38         
39     def tearDown(self):
40         try:
41             shutil.rmtree(self.root_dir)
42         except:
43             # retry
44             time.sleep(0.1)
45             shutil.rmtree(self.root_dir)
46
47     def make_instance(self):
48         testbed_id = self.testbed_id
49         slicename = self.slicename
50         plchost = self.plchost
51         
52         instance = planetlab.TestbedController()
53         pl_ssh_key = os.environ.get(
54             "PL_SSH_KEY",
55             "%s/.ssh/id_rsa_planetlab" % (os.environ['HOME'],) )
56         slicename = os.environ.get(
57             "PL_SLICE",
58             slicename)
59         pl_user, pl_pwd = test_util.pl_auth()
60         
61         instance.defer_configure("homeDirectory", self.root_dir)
62         instance.defer_configure("slice", slicename)
63         instance.defer_configure("sliceSSHKey", pl_ssh_key)
64         instance.defer_configure("authUser", pl_user)
65         instance.defer_configure("authPass", pl_pwd)
66         instance.defer_configure("plcHost", plchost)
67         instance.defer_configure("tapPortBase", self.port_base)
68         instance.defer_configure("p2pDeployment", False) # it's interactive, we don't want it in tests
69         instance.defer_configure("dedicatedSlice", True)
70         
71         # Hack, but we need vsys_vnet
72         instance.do_setup()
73         vnet = instance.vsys_vnet
74         self.net_prefix = vnet.rsplit('.',1)[0]
75         
76         return instance
77
78     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
79     def test_simple(self):
80         instance = self.make_instance()
81         
82         instance.defer_create(2, "Node")
83         instance.defer_create_set(2, "hostname", self.host1)
84         instance.defer_create(3, "Node")
85         instance.defer_create_set(3, "hostname", self.host2)
86         instance.defer_create(4, "NodeInterface")
87         instance.defer_connect(2, "devs", 4, "node")
88         instance.defer_create(5, "NodeInterface")
89         instance.defer_connect(3, "devs", 5, "node")
90         instance.defer_create(6, "Internet")
91         instance.defer_connect(4, "inet", 6, "devs")
92         instance.defer_connect(5, "inet", 6, "devs")
93         instance.defer_create(7, "Application")
94         instance.defer_create_set(7, "command", "ping -qc1 {#[GUID-5].addr[0].[Address]#}")
95         instance.defer_add_trace(7, "stdout")
96         instance.defer_add_trace(7, "stderr")
97         instance.defer_connect(7, "node", 2, "apps")
98
99         comp_result = r"""PING .* \(.*\) \d*\(\d*\) bytes of data.
100
101 --- .* ping statistics ---
102 1 packets transmitted, 1 received, 0% packet loss, time \d*ms.*
103 """
104
105         try:
106             instance.do_setup()
107             instance.do_create()
108             instance.do_connect_init()
109             instance.do_connect_compl()
110             instance.do_preconfigure()
111             
112             # Manually replace netref
113             instance.set(7, "command",
114                 instance.get(7, "command")
115                     .replace("{#[GUID-5].addr[0].[Address]#}", 
116                         instance.get_address(5, 0, "Address") )
117             )
118
119             instance.do_configure()
120             
121             instance.do_prestart()
122             instance.start()
123             while instance.status(7) != AS.STATUS_FINISHED:
124                 time.sleep(0.5)
125             ping_result = instance.trace(7, "stdout") or ""
126             instance.stop()
127         finally:
128             try:
129                 instance.shutdown()
130             except:
131                 pass
132
133         # asserts at the end, to make sure there's proper cleanup
134         self.assertTrue(re.match(comp_result, ping_result, re.MULTILINE),
135             "Unexpected trace:\n" + ping_result)
136         
137     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
138     def test_depends(self):
139         instance = self.make_instance()
140         
141         instance.defer_create(2, "Node")
142         instance.defer_create_set(2, "hostname", self.host1)
143         instance.defer_create(3, "NodeInterface")
144         instance.defer_connect(2, "devs", 3, "node")
145         instance.defer_create(4, "Internet")
146         instance.defer_connect(3, "inet", 4, "devs")
147         instance.defer_create(5, "Application")
148         instance.defer_create_set(5, "command", "gfortran --version")
149         instance.defer_create_set(5, "depends", "gcc-gfortran")
150         instance.defer_add_trace(5, "stdout")
151         instance.defer_add_trace(5, "stderr")
152         instance.defer_connect(5, "node", 2, "apps")
153
154         try:
155             instance.do_setup()
156             instance.do_create()
157             instance.do_connect_init()
158             instance.do_connect_compl()
159             instance.do_preconfigure()
160             instance.do_configure()
161             
162             instance.do_prestart()
163             instance.start()
164             while instance.status(5) != AS.STATUS_FINISHED:
165                 time.sleep(0.5)
166             ping_result = instance.trace(5, "stdout") or ""
167             comp_result = r".*GNU Fortran \(GCC\).*"
168             instance.stop()
169         finally:
170             try:
171                 instance.shutdown()
172             except:
173                 pass
174
175         # asserts at the end, to make sure there's proper cleanup
176         self.assertTrue(re.match(comp_result, ping_result, re.MULTILINE),
177             "Unexpected trace:\n" + ping_result)
178         
179     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
180     def test_build(self):
181         instance = self.make_instance()
182         
183         instance.defer_create(2, "Node")
184         instance.defer_create_set(2, "hostname", self.host1)
185         instance.defer_create(3, "NodeInterface")
186         instance.defer_connect(2, "devs", 3, "node")
187         instance.defer_create(4, "Internet")
188         instance.defer_connect(3, "inet", 4, "devs")
189         instance.defer_create(10, "Application")
190         instance.defer_create_set(10, "command", "./consts")
191         instance.defer_create_set(10, "buildDepends", "gcc")
192         instance.defer_create_set(10, "build", "gcc ${SOURCES}/consts.c -o consts")
193         instance.defer_create_set(10, "install", "cp consts ${SOURCES}/consts")
194         instance.defer_create_set(10, "sources", os.path.join(os.path.dirname(planetlab.__file__),'scripts','consts.c'))
195         instance.defer_add_trace(10, "stdout")
196         instance.defer_add_trace(10, "stderr")
197         instance.defer_connect(10, "node", 2, "apps")
198
199         comp_result = \
200 r""".*ETH_P_ALL = 0x[0-9a-fA-F]{8}
201 ETH_P_IP = 0x[0-9a-fA-F]{8}
202 TUNGETIFF = 0x[0-9a-fA-F]{8}
203 TUNSETIFF = 0x[0-9a-fA-F]{8}
204 IFF_NO_PI = 0x[0-9a-fA-F]{8}
205 IFF_TAP = 0x[0-9a-fA-F]{8}
206 IFF_TUN = 0x[0-9a-fA-F]{8}
207 IFF_VNET_HDR = 0x[0-9a-fA-F]{8}
208 TUN_PKT_STRIP = 0x[0-9a-fA-F]{8}
209 IFHWADDRLEN = 0x[0-9a-fA-F]{8}
210 IFNAMSIZ = 0x[0-9a-fA-F]{8}
211 IFREQ_SZ = 0x[0-9a-fA-F]{8}
212 FIONREAD = 0x[0-9a-fA-F]{8}.*
213 """
214
215         try:
216             instance.do_setup()
217             instance.do_create()
218             instance.do_connect_init()
219             instance.do_connect_compl()
220             instance.do_preconfigure()
221             instance.do_configure()
222             
223             instance.do_prestart()
224             instance.start()
225             while instance.status(10) != AS.STATUS_FINISHED:
226                 time.sleep(0.5)
227             ping_result = instance.trace(10, "stdout") or ""
228             instance.stop()
229         finally:
230             try:
231                 instance.shutdown()
232             except:
233                 pass
234
235         # asserts at the end, to make sure there's proper cleanup
236         self.assertTrue(re.match(comp_result, ping_result, re.MULTILINE),
237             "Unexpected trace:\n" + ping_result)
238         
239     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
240     def test_simple_vsys(self):
241         instance = self.make_instance()
242         
243         instance.defer_create(2, "Node")
244         instance.defer_create_set(2, "hostname", self.host1)
245         instance.defer_create(3, "NodeInterface")
246         instance.defer_connect(2, "devs", 3, "node")
247         instance.defer_create(4, "Internet")
248         instance.defer_connect(3, "inet", 4, "devs")
249         instance.defer_create(5, "TunInterface")
250         instance.defer_add_address(5, self.net_prefix+".2", 24, False)
251         instance.defer_connect(2, "devs", 5, "node")
252         instance.defer_create(6, "Application")
253         instance.defer_create_set(6, "command", """
254 set -e
255 netconfig help > /dev/null
256 test -e /vsys/vif_up.in > /dev/null
257 test -e /vsys/vif_up.out > /dev/null
258 test -e /vsys/fd_tuntap.control > /dev/null
259 echo 'OKIDOKI'
260 """)
261         instance.defer_create_set(6, "sudo", True) # only sudo has access to /vsys
262         instance.defer_add_trace(6, "stdout")
263         instance.defer_add_trace(6, "stderr")
264         instance.defer_connect(6, "node", 2, "apps")
265
266         try:
267             instance.do_setup()
268             instance.do_create()
269             instance.do_connect_init()
270             instance.do_connect_compl()
271             instance.do_preconfigure()
272             instance.do_configure()
273             
274             instance.do_prestart()
275             instance.start()
276             while instance.status(6) != AS.STATUS_FINISHED:
277                 time.sleep(0.5)
278             test_result = (instance.trace(6, "stdout") or "").strip()
279             comp_result = "OKIDOKI"
280             instance.stop()
281         finally:
282             try:
283                 instance.shutdown()
284             except:
285                 pass
286
287         # asserts at the end, to make sure there's proper cleanup
288         self.assertEqual(comp_result, test_result)
289
290     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
291     def test_emulation(self):
292         instance = self.make_instance()
293         
294         instance.defer_create(2, "Node")
295         instance.defer_create_set(2, "hostname", self.host1)
296         instance.defer_create(3, "NodeInterface")
297         instance.defer_connect(2, "devs", 3, "node")
298         instance.defer_create(4, "Internet")
299         instance.defer_connect(3, "inet", 4, "devs")
300         instance.defer_create(7, "NetPipe")
301         instance.defer_create_set(7, "mode", "CLIENT")
302         instance.defer_create_set(7, "portList", "80")
303         instance.defer_create_set(7, "bwOut", 12.0/1024.0) # 12kbps
304         instance.defer_create_set(7, "bwIn", 64.0/1024.0) # 64kbps
305         instance.defer_create_set(7, "plrOut", 0.01) # 1% plr outbound - high loss
306         instance.defer_create_set(7, "plrIn", 0.001) # 0.1% plr inbound - regular loss
307         instance.defer_create_set(7, "delayOut", int(1500 * 8 / (12.0/1024.0) / 1000)) # tx delay at 12kbps in ms
308         instance.defer_create_set(7, "delayIn", int(1500 * 8 / (64.0/1024.0) / 1000)) # rx delay at 64kbps in ms
309         instance.defer_add_trace(7, "netpipeStats")
310         instance.defer_connect(2, "pipes", 7, "node")
311         instance.defer_create(8, "Application")
312         instance.defer_create_set(8, "command", "time wget -q -O /dev/null http://www.google.com/") # Fetch ~10kb
313         instance.defer_add_trace(8, "stdout")
314         instance.defer_add_trace(8, "stderr")
315         instance.defer_connect(8, "node", 2, "apps")
316
317         try:
318             instance.do_setup()
319             instance.do_create()
320             instance.do_connect_init()
321             instance.do_connect_compl()
322             instance.do_preconfigure()
323             instance.do_configure()
324             
325             instance.do_prestart()
326             instance.start()
327             while instance.status(8) != AS.STATUS_FINISHED:
328                 time.sleep(0.5)
329             test_result = (instance.trace(8, "stderr") or "").strip()
330             comp_result = r".*real\s*(?P<min>[0-9]+)m(?P<sec>[0-9]+[.][0-9]+)s.*"
331             netpipe_stats = instance.trace(7, "netpipeStats")
332             
333             instance.stop()
334         finally:
335             try:
336                 instance.shutdown()
337             except:
338                 pass
339
340         # asserts at the end, to make sure there's proper cleanup
341         match = re.match(comp_result, test_result, re.MULTILINE)
342         self.assertTrue(match, "Unexpected output: %s" % (test_result,))
343         
344         minutes = int(match.group("min"))
345         seconds = float(match.group("sec"))
346         self.assertTrue((minutes * 60 + seconds) > 1.0, "Emulation not effective: %s" % (test_result,))
347
348         self.assertTrue(netpipe_stats, "Unavailable netpipe stats")
349
350     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
351     def _pingtest(self, TunClass, ConnectionProto, Cipher, Filter1=None, Filter2=None):
352         instance = self.make_instance()
353         
354         instance.defer_create(2, "Node")
355         instance.defer_create_set(2, "hostname", self.host1)
356         instance.defer_create(3, "Node")
357         instance.defer_create_set(3, "hostname", self.host2)
358         instance.defer_create(4, "NodeInterface")
359         instance.defer_connect(2, "devs", 4, "node")
360         instance.defer_create(5, "Internet")
361         instance.defer_connect(4, "inet", 5, "devs")
362         instance.defer_create(6, "NodeInterface")
363         instance.defer_connect(3, "devs", 6, "node")
364         instance.defer_connect(6, "inet", 5, "devs")
365         instance.defer_create(7, TunClass)
366         instance.defer_create_set(7, "tun_cipher", Cipher)
367         instance.defer_add_trace(7, "packets")
368         instance.defer_add_address(7, self.net_prefix+".2", 24, False)
369         instance.defer_connect(2, "devs", 7, "node")
370         instance.defer_create(8, TunClass)
371         instance.defer_create_set(8, "tun_cipher", Cipher)
372         instance.defer_add_trace(8, "packets")
373         instance.defer_add_address(8, self.net_prefix+".3", 24, False)
374         instance.defer_connect(3, "devs", 8, "node")
375         instance.defer_create(9, "Application")
376         instance.defer_create_set(9, "command", "ping -qc10 {#[GUID-8].addr[0].[Address]#}")
377         instance.defer_add_trace(9, "stdout")
378         instance.defer_add_trace(9, "stderr")
379         instance.defer_connect(9, "node", 2, "apps")
380         
381         if Filter1:
382             instance.defer_create(10, "TunFilter")
383             instance.defer_create_set(10, "module", Filter1)
384             instance.defer_connect(7, "fd->", 10, "->fd")
385             
386         if Filter2:
387             instance.defer_create(11, "TunFilter")
388             instance.defer_create_set(11, "module", Filter2)
389             instance.defer_connect(8, "fd->", 11, "->fd")
390
391         if Filter1 and Filter2:
392             plr = "[5-9][0-9]"
393         elif Filter1 or Filter2:
394             plr = "[3-9][0-9]"
395         else:
396             plr = "0"
397        
398         instance.defer_connect(
399             (10 if Filter1 else 7), ConnectionProto, 
400             (11 if Filter2 else 8), ConnectionProto)
401
402         comp_result = r"""PING .* \(.*\) \d*\(\d*\) bytes of data.
403
404 --- .* ping statistics ---
405 10 packets transmitted, [0-9]+ received,.* %s%% packet loss, time \d*ms.*
406 """ % (plr,)
407
408         try:
409             instance.do_setup()
410             instance.do_create()
411             instance.do_connect_init()
412             instance.do_connect_compl()
413             instance.do_preconfigure()
414             
415             # Manually replace netref
416             instance.set(9, "command",
417                 instance.get(9, "command")
418                     .replace("{#[GUID-8].addr[0].[Address]#}", 
419                         instance.get_address(8, 0, "Address") )
420             )
421             
422             instance.do_configure()
423             
424             instance.do_prestart()
425             instance.start()
426             while instance.status(9) != AS.STATUS_FINISHED:
427                 time.sleep(0.5)
428             ping_result = instance.trace(9, "stdout") or ""
429             packets1 = instance.trace(7, "packets") or ""
430             packets2 = instance.trace(8, "packets") or ""
431             instance.stop()
432         finally:
433             try:
434                 instance.shutdown()
435             except:
436                 pass
437
438         # asserts at the end, to make sure there's proper cleanup
439         self.assertTrue(re.match(comp_result, ping_result, re.MULTILINE),
440             "Unexpected trace:\n%s\nPackets @ source:\n%s\nPackets @ target:\n%s" % (
441                 ping_result,
442                 packets1,
443                 packets2))
444
445     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
446     def test_tun_ping(self):
447         self._pingtest("TunInterface", "tcp", "AES")
448
449     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
450     def test_tun_ping_udp(self):
451         self._pingtest("TunInterface", "udp", "AES")
452
453     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
454     def test_tun_ping_gre(self):
455         self._pingtest("TunInterface", "gre", "PLAIN")
456
457     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
458     def test_tap_ping(self):
459         self._pingtest("TapInterface", "tcp", "AES")
460
461     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
462     def test_tap_ping_udp(self):
463         self._pingtest("TapInterface", "udp", "AES")
464
465     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
466     def test_tap_ping_gre(self):
467         self._pingtest("TapInterface", "gre", "PLAIN")
468
469     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
470     def test_tap_ping_udp_loss1_py(self):
471         self._pingtest("TapInterface", "udp", "AES", self.PLR50_PY)
472
473     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
474     def test_tap_ping_udp_loss2_py(self):
475         self._pingtest("TapInterface", "udp", "AES", self.PLR50_PY, self.PLR50_PY)
476
477     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
478     def test_tap_ping_udp_loss1_c(self):
479         self._pingtest("TapInterface", "udp", "AES", self.PLR50_C)
480
481     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
482     def test_tap_ping_udp_loss2_c(self):
483         self._pingtest("TapInterface", "udp", "AES", self.PLR50_C, self.PLR50_C)
484
485     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
486     def test_nepi_depends(self):
487         instance = self.make_instance()
488         
489         instance.defer_create(2, "Node")
490         instance.defer_create_set(2, "hostname", self.host1)
491         instance.defer_create(3, "NodeInterface")
492         instance.defer_connect(2, "devs", 3, "node")
493         instance.defer_create(4, "Internet")
494         instance.defer_connect(3, "inet", 4, "devs")
495         instance.defer_create(5, "NepiDependency")
496         instance.defer_connect(5, "node", 2, "deps")
497         instance.defer_create(12, "Application")
498         instance.defer_connect(12, "node", 2, "apps")
499         instance.defer_create_set(12, "command", "python -c 'import nepi'")
500         instance.defer_add_trace(12, "stderr")
501
502         try:
503             instance.do_setup()
504             instance.do_create()
505             instance.do_connect_init()
506             instance.do_connect_compl()
507             instance.do_preconfigure()
508             instance.do_configure()
509             
510             instance.do_prestart()
511             instance.start()
512             while instance.status(12) != AS.STATUS_FINISHED:
513                 time.sleep(0.5)
514             ping_result = (instance.trace(12, "stderr") or "").strip()
515             instance.stop()
516         finally:
517             try:
518                 instance.shutdown()
519             except:
520                 pass
521         
522         # asserts at the end, to make sure there's proper cleanup
523         self.assertEqual(ping_result, "")
524
525     @test_util.skipUnless(test_util.pl_auth() is not None, 
526         "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
527     @test_util.skipUnless(os.environ.get('NEPI_FULL_TESTS','').lower() in ('1','yes','true','on'),
528         "Test is expensive, requires NEPI_FULL_TESTS=yes")
529     def test_ns3_depends(self):
530         instance = self.make_instance()
531         
532         instance.defer_create(2, "Node")
533         instance.defer_create_set(2, "hostname", self.host1)
534         instance.defer_create(3, "NodeInterface")
535         instance.defer_connect(2, "devs", 3, "node")
536         instance.defer_create(4, "Internet")
537         instance.defer_connect(3, "inet", 4, "devs")
538         instance.defer_create(5, "NepiDependency")
539         instance.defer_connect(5, "node", 2, "deps")
540         instance.defer_create(6, "NS3Dependency")
541         instance.defer_connect(6, "node", 2, "deps")
542         instance.defer_create(12, "Application")
543         instance.defer_connect(12, "node", 2, "apps")
544         instance.defer_create_set(12, "command", "python -c 'import nepi.testbeds.ns3.execute ; tb = nepi.testbeds.ns3.execute.TestbedController() ; mod = tb._configure_ns3_module()'")
545         instance.defer_add_trace(12, "stderr")
546
547         try:
548             instance.do_setup()
549             instance.do_create()
550             instance.do_connect_init()
551             instance.do_connect_compl()
552             instance.do_preconfigure()
553             instance.do_configure()
554             
555             instance.do_prestart()
556             instance.start()
557             while instance.status(12) != AS.STATUS_FINISHED:
558                 time.sleep(0.5)
559             ping_result = (instance.trace(12, "stderr") or "").strip()
560             instance.stop()
561         finally:
562             try:
563                 instance.shutdown()
564             except:
565                 pass
566         
567         # asserts at the end, to make sure there's proper cleanup
568         self.assertEqual(ping_result, "")
569
570     @test_util.skipUnless(test_util.pl_auth() is not None, "Test requires PlanetLab authentication info (PL_USER and PL_PASS environment variables)")
571     def test_discovery(self):
572         instance = self.make_instance()
573         
574         instance.defer_create(2, "Node")
575         instance.defer_create_set(2, "operatingSystem", "f12")
576         instance.defer_create(3, "Node")
577         instance.defer_create_set(3, "operatingSystem", "f12")
578         instance.defer_create(4, "NodeInterface")
579         instance.defer_connect(2, "devs", 4, "node")
580         instance.defer_create(5, "NodeInterface")
581         instance.defer_connect(3, "devs", 5, "node")
582         instance.defer_create(6, "Internet")
583         instance.defer_connect(4, "inet", 6, "devs")
584         instance.defer_connect(5, "inet", 6, "devs")
585         instance.defer_create(7, "Application")
586         instance.defer_create_set(7, "command", "ping -qc1 {#[GUID-5].addr[0].[Address]#}")
587         instance.defer_add_trace(7, "stdout")
588         instance.defer_add_trace(7, "stderr")
589         instance.defer_connect(7, "node", 2, "apps")
590
591         comp_result = r"""PING .* \(.*\) \d*\(\d*\) bytes of data.
592
593 --- .* ping statistics ---
594 1 packets transmitted, 1 received, 0% packet loss, time \d*ms.*
595 """
596
597         try:
598             instance.do_setup()
599             instance.do_create()
600             instance.do_connect_init()
601             instance.do_connect_compl()
602             instance.do_preconfigure()
603             
604             # Manually replace netref
605             instance.set(7, "command",
606                 instance.get(7, "command")
607                     .replace("{#[GUID-5].addr[0].[Address]#}", 
608                         instance.get_address(5, 0, "Address") )
609             )
610
611             instance.do_configure()
612             
613             instance.do_prestart()
614             instance.start()
615             while instance.status(7) != AS.STATUS_FINISHED:
616                 time.sleep(0.5)
617             ping_result = instance.trace(7, "stdout") or ""
618             instance.stop()
619         finally:
620             try:
621                 instance.shutdown()
622             except:
623                 pass
624
625         # asserts at the end, to make sure there's proper cleanup
626         self.assertTrue(re.match(comp_result, ping_result, re.MULTILINE),
627             "Unexpected trace:\n" + ping_result)
628         
629         
630
631 if __name__ == '__main__':
632     unittest.main()
633