Sometimes when controller process cleanup happens concurrently with testbed teardown...

Just catch them and retry, sidestepping the issue. Proper synchronization is not easy, and not worth the amount of work for mere testing.

diff --git a/test/core/integration.py b/test/core/integration.py
index d58dc86..3618c87 100755 (executable)
--- a/test/core/integration.py
+++ b/test/core/integration.py
@@ -25,7 +25,12 @@ class ExecuteTestCase(unittest.TestCase):
         self.root_dir = tempfile.mkdtemp()
 
     def tearDown(self):
-        shutil.rmtree(self.root_dir)
+        try:
+            shutil.rmtree(self.root_dir)
+        except:
+            # retry
+            time.sleep(0.1)
+            shutil.rmtree(self.root_dir)
 
     def make_testbed(self, exp_desc, testbed_id, testbed_version):
         provider = FactoriesProvider(testbed_id, testbed_version)

diff --git a/test/testbeds/netns/execute.py b/test/testbeds/netns/execute.py
index 319de37..d37d84f 100755 (executable)
--- a/test/testbeds/netns/execute.py
+++ b/test/testbeds/netns/execute.py
@@ -164,7 +164,12 @@ class NetnsExecuteTestCase(unittest.TestCase):
         instance.shutdown()
         
     def tearDown(self):
-        shutil.rmtree(self.root_dir)
+        try:
+            shutil.rmtree(self.root_dir)
+        except:
+            # retry
+            time.sleep(0.1)
+            shutil.rmtree(self.root_dir)
 
 if __name__ == '__main__':
     unittest.main()

diff --git a/test/testbeds/netns/integration.py b/test/testbeds/netns/integration.py
index 16f336c..a18cdb0 100755 (executable)
--- a/test/testbeds/netns/integration.py
+++ b/test/testbeds/netns/integration.py
@@ -187,7 +187,12 @@ class NetnsIntegrationTestCase(unittest.TestCase):
         controller.shutdown()
 
     def tearDown(self):
-        shutil.rmtree(self.root_dir)
+        try:
+            shutil.rmtree(self.root_dir)
+        except:
+            # retry
+            time.sleep(0.1)
+            shutil.rmtree(self.root_dir)
 
 if __name__ == '__main__':
     unittest.main()

diff --git a/test/testbeds/ns3/execute.py b/test/testbeds/ns3/execute.py
index ffefbfc..be7a432 100755 (executable)
--- a/test/testbeds/ns3/execute.py
+++ b/test/testbeds/ns3/execute.py
@@ -78,7 +78,12 @@ class Ns3ExecuteTestCase(unittest.TestCase):
         instance.shutdown()
 
     def tearDown(self):
-        shutil.rmtree(self.root_dir)
+        try:
+            shutil.rmtree(self.root_dir)
+        except:
+            # retry
+            time.sleep(0.1)
+            shutil.rmtree(self.root_dir)
 
 if __name__ == '__main__':
     unittest.main()

diff --git a/test/testbeds/ns3/execute2.py b/test/testbeds/ns3/execute2.py
index 1a24e70..52ac526 100755 (executable)
--- a/test/testbeds/ns3/execute2.py
+++ b/test/testbeds/ns3/execute2.py
@@ -114,7 +114,12 @@ class Ns3ExecuteTestCase(unittest.TestCase):
         instance.shutdown()
 
     def tearDown(self):
-        shutil.rmtree(self.root_dir)
+        try:
+            shutil.rmtree(self.root_dir)
+        except:
+            # retry
+            time.sleep(0.1)
+            shutil.rmtree(self.root_dir)
 
 if __name__ == '__main__':
     unittest.main()

diff --git a/test/testbeds/ns3/integration.py b/test/testbeds/ns3/integration.py
index 61df6ae..f0926da 100755 (executable)
--- a/test/testbeds/ns3/integration.py
+++ b/test/testbeds/ns3/integration.py
@@ -152,7 +152,12 @@ class Ns3IntegrationTestCase(unittest.TestCase):
         controller.shutdown()
 
     def tearDown(self):
-        shutil.rmtree(self.root_dir)
+        try:
+            shutil.rmtree(self.root_dir)
+        except:
+            # retry
+            time.sleep(0.1)
+            shutil.rmtree(self.root_dir)
 
 if __name__ == '__main__':
     unittest.main()

diff --git a/test/testbeds/planetlab/execute.py b/test/testbeds/planetlab/execute.py
index 4d0fa75..627e256 100755 (executable)
--- a/test/testbeds/planetlab/execute.py
+++ b/test/testbeds/planetlab/execute.py
@@ -17,7 +17,12 @@ class PlanetLabExecuteTestCase(unittest.TestCase):
         self.root_dir = tempfile.mkdtemp()
         
     def tearDown(self):
-        shutil.rmtree(self.root_dir)
+        try:
+            shutil.rmtree(self.root_dir)
+        except:
+            # retry
+            time.sleep(0.1)
+            shutil.rmtree(self.root_dir)
 
     def make_instance(self):
         testbed_version = "01"

diff --git a/test/testbeds/planetlab/integration.py b/test/testbeds/planetlab/integration.py
index e7e34d3..0edcb1e 100755 (executable)
--- a/test/testbeds/planetlab/integration.py
+++ b/test/testbeds/planetlab/integration.py
@@ -18,7 +18,12 @@ class PlanetLabIntegrationTestCase(unittest.TestCase):
         self.root_dir = tempfile.mkdtemp()
 
     def tearDown(self):
-        shutil.rmtree(self.root_dir)
+        try:
+            shutil.rmtree(self.root_dir)
+        except:
+            # retry
+            time.sleep(0.1)
+            shutil.rmtree(self.root_dir)
 
     def make_experiment_desc(self):
         testbed_id = "planetlab"

diff --git a/test/util/server.py b/test/util/server.py
index f1bdd1c..7e4c6f1 100755 (executable)
--- a/test/util/server.py
+++ b/test/util/server.py
@@ -9,6 +9,7 @@ import sys
 import tempfile
 import test_util
 import unittest
+import time
 
 class ServerTestCase(unittest.TestCase):
     def setUp(self):
@@ -161,7 +162,12 @@ class ServerTestCase(unittest.TestCase):
         self.assertEqual(reply, "Stopping server")
 
     def tearDown(self):
-        shutil.rmtree(self.root_dir)
+        try:
+            shutil.rmtree(self.root_dir)
+        except:
+            # retry
+            time.sleep(0.1)
+            shutil.rmtree(self.root_dir)
 
 if __name__ == '__main__':
     unittest.main()