test/core/integration.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import getpass
   5 from nepi.core.design import ExperimentDescription, FactoriesProvider
   6 from nepi.util import proxy
   7 from nepi.util.constants import DeploymentConfiguration as DC
   8 import mock
   9 import mock.metadata
  10 import mock2
  11 import mock2.metadata
  12 import os
  13 import shutil
  14 import sys
  15 import tempfile
  16 import test_util
  17 import time
  18 import unittest
  19
  20 class ExecuteTestCase(unittest.TestCase):
  21     def setUp(self):
  22         sys.modules["nepi.testbeds.mock.metadata"] = mock.metadata
  23         sys.modules["nepi.testbeds.mock"] = mock
  24         sys.modules["nepi.testbeds.mock2.metadata"] = mock2.metadata
  25         sys.modules["nepi.testbeds.mock2"] = mock2
  26         self.root_dir = tempfile.mkdtemp()
  27
  28     def tearDown(self):
  29         try:
  30             shutil.rmtree(self.root_dir)
  31         except:
  32             # retry
  33             time.sleep(0.1)
  34             shutil.rmtree(self.root_dir)
  35
  36     def make_testbed(self, exp_desc, testbed_id):
  37         provider = FactoriesProvider(testbed_id)
  38         desc = exp_desc.add_testbed_description(provider)
  39         desc.set_attribute_value("fake", True)
  40         node1 = desc.create("Node")
  41         node2 = desc.create("Node")
  42         iface1 = desc.create("Interface")
  43         iface1.set_attribute_value("fake", True)
  44         node1.connector("devs").connect(iface1.connector("node"))
  45         iface2 = desc.create("Interface")
  46         iface2.set_attribute_value("fake", True)
  47         node2.connector("devs").connect(iface2.connector("node"))
  48         iface1.connector("iface").connect(iface2.connector("iface"))
  49         app = desc.create("Application")
  50         app.connector("node").connect(node1.connector("apps"))
  51         app.enable_trace("fake")
  52
  53         return exp_desc, desc, app, node1, node2, iface1, iface2
  54
  55     def make_test_experiment(self):
  56         exp_desc = ExperimentDescription()
  57         testbed_id = "mock"
  58         return self.make_testbed(exp_desc, testbed_id)
  59
  60     def make_cross_test_experiment(self):
  61         exp_desc = ExperimentDescription()
  62         testbed_id1 = "mock"
  63         testbed_id2 = "mock2"
  64         exp_desc, desc1, app1, node11, node12, iface11, iface12 = \
  65                 self.make_testbed(exp_desc, testbed_id1)
  66         exp_desc, desc2, app2, node21, node22, iface21, iface22 = \
  67                  self.make_testbed(exp_desc, testbed_id2)
  68         iface12.connector("cross").connect(iface21.connector("cross"))
  69
  70         return exp_desc, desc1, desc2, iface12, iface21
  71
  72     def test_single_process_cross_integration(self):
  73         exp_desc, desc1, desc2, iface12, iface21 = \
  74                 self.make_cross_test_experiment()
  75         xml = exp_desc.to_xml()
  76         access_config = None
  77         controller = proxy.create_experiment_controller(xml, access_config)
  78
  79         controller.start()
  80         cross1 = controller.get(iface12.guid, "cross")
  81         cross2 = controller.get(iface21.guid, "cross")
  82         self.assertTrue(cross1 == cross2 == True)
  83         controller.stop()
  84         controller.shutdown()
  85
  86     def test_single_process_integration(self):
  87         exp_desc, desc, app, node1, node2, iface1, iface2 = self.make_test_experiment()
  88         xml = exp_desc.to_xml()
  89         access_config = None
  90         controller = proxy.create_experiment_controller(xml, access_config)
  91
  92         controller.start()
  93         started_time = controller.started_time
  94         self.assertTrue(started_time < time.time())
  95         while not controller.is_finished(app.guid):
  96             time.sleep(0.5)
  97         fake_result = controller.trace(app.guid, "fake")
  98         comp_result = """PING 10.0.0.2 (10.0.0.2) 56(84) bytes of data.
  99
 100 --- 10.0.0.2 ping statistics ---
 101 1 packets transmitted, 1 received, 0% packet loss, time 0ms
 102 """
 103         self.assertTrue(fake_result.startswith(comp_result))
 104
 105         self.assertEquals(controller.get_testbed_id(node1.guid), "mock")
 106         self.assertEquals(controller.get_testbed_version(node1.guid), "0.1")
 107         self.assertEquals(controller.get_factory_id(node1.guid), "Node")
 108
 109         controller.stop()
 110         stopped_time = controller.stopped_time
 111         self.assertTrue(stopped_time < time.time())
 112         controller.shutdown()
 113
 114     def test_daemonized_controller_integration(self):
 115         exp_desc, desc, app, node1, node2, iface1, iface2 = self.make_test_experiment()
 116         xml = exp_desc.to_xml()
 117         access_config = proxy.AccessConfiguration()
 118         access_config.set_attribute_value(DC.DEPLOYMENT_MODE, DC.MODE_DAEMON)
 119         access_config.set_attribute_value(DC.ROOT_DIRECTORY, self.root_dir)
 120         access_config.set_attribute_value(DC.DEPLOYMENT_ENVIRONMENT_SETUP,
 121             "export PYTHONPATH=%r:%r:$PYTHONPATH "
 122             "export NEPI_TESTBEDS='mock:mock mock2:mock2' " % (
 123                 os.path.dirname(os.path.dirname(mock.__file__)),
 124                 os.path.dirname(os.path.dirname(mock2.__file__)),))
 125
 126         controller = proxy.create_experiment_controller(xml, access_config)
 127
 128         controller.start()
 129         started_time = controller.started_time
 130         self.assertTrue(started_time < time.time())
 131         while not controller.is_finished(app.guid):
 132             time.sleep(0.5)
 133         fake_result = controller.trace(app.guid, "fake")
 134         comp_result = """PING 10.0.0.2 (10.0.0.2) 56(84) bytes of data.
 135
 136 --- 10.0.0.2 ping statistics ---
 137 1 packets transmitted, 1 received, 0% packet loss, time 0ms
 138 """
 139         self.assertTrue(fake_result.startswith(comp_result))
 140
 141         self.assertEquals(controller.get_testbed_id(node1.guid), "mock")
 142         self.assertEquals(controller.get_testbed_version(node1.guid), "0.1")
 143         self.assertEquals(controller.get_factory_id(node1.guid), "Node")
 144
 145         controller.stop()
 146         stopped_time = controller.stopped_time
 147         self.assertTrue(stopped_time < time.time())
 148         controller.shutdown()
 149
 150     def test_daemonized_testbed_integration(self):
 151         exp_desc, desc, app, node1, node2, iface1, iface2 = self.make_test_experiment()
 152
 153         desc.set_attribute_value(DC.DEPLOYMENT_MODE, DC.MODE_DAEMON)
 154         desc.set_attribute_value(DC.ROOT_DIRECTORY, self.root_dir)
 155         desc.set_attribute_value(DC.DEPLOYMENT_ENVIRONMENT_SETUP,
 156             "export PYTHONPATH=%r:%r:$PYTHONPATH "
 157             "export NEPI_TESTBEDS='mock:mock mock2:mock2' " % (
 158                 os.path.dirname(os.path.dirname(mock.__file__)),
 159                 os.path.dirname(os.path.dirname(mock2.__file__)),))
 160
 161         xml = exp_desc.to_xml()
 162
 163         controller = proxy.create_experiment_controller(xml, access_config = None)
 164
 165         controller.start()
 166         while not controller.is_finished(app.guid):
 167             time.sleep(0.5)
 168         fake_result = controller.trace(app.guid, "fake")
 169         comp_result = """PING 10.0.0.2 (10.0.0.2) 56(84) bytes of data.
 170
 171 --- 10.0.0.2 ping statistics ---
 172 1 packets transmitted, 1 received, 0% packet loss, time 0ms
 173 """
 174         self.assertTrue(fake_result.startswith(comp_result))
 175
 176         self.assertEquals(controller.get_testbed_id(node1.guid), "mock")
 177         self.assertEquals(controller.get_testbed_version(node1.guid), "0.1")
 178         self.assertEquals(controller.get_factory_id(node1.guid), "Node")
 179
 180         controller.stop()
 181         controller.shutdown()
 182
 183     def test_daemonized_all_integration(self):
 184         exp_desc, desc, app, node1, node2, iface1, iface2 = self.make_test_experiment()
 185
 186         desc.set_attribute_value(DC.DEPLOYMENT_MODE, DC.MODE_DAEMON)
 187         inst_root_dir = os.path.join(self.root_dir, "instance")
 188         os.mkdir(inst_root_dir)
 189         desc.set_attribute_value(DC.ROOT_DIRECTORY, inst_root_dir)
 190
 191         xml = exp_desc.to_xml()
 192
 193         access_config = proxy.AccessConfiguration()
 194         access_config.set_attribute_value(DC.DEPLOYMENT_MODE, DC.MODE_DAEMON)
 195         access_config.set_attribute_value(DC.ROOT_DIRECTORY, self.root_dir)
 196         access_config.set_attribute_value(DC.DEPLOYMENT_ENVIRONMENT_SETUP,
 197             "export PYTHONPATH=%r:%r:$PYTHONPATH "
 198             "export NEPI_TESTBEDS='mock:mock mock2:mock2' " % (
 199                 os.path.dirname(os.path.dirname(mock.__file__)),
 200                 os.path.dirname(os.path.dirname(mock2.__file__)),))
 201         controller = proxy.create_experiment_controller(xml, access_config)
 202
 203         controller.start()
 204         while not controller.is_finished(app.guid):
 205             time.sleep(0.5)
 206         fake_result = controller.trace(app.guid, "fake")
 207         comp_result = """PING 10.0.0.2 (10.0.0.2) 56(84) bytes of data.
 208
 209 --- 10.0.0.2 ping statistics ---
 210 1 packets transmitted, 1 received, 0% packet loss, time 0ms
 211 """
 212         self.assertTrue(fake_result.startswith(comp_result))
 213
 214         self.assertEquals(controller.get_testbed_id(node1.guid), "mock")
 215         self.assertEquals(controller.get_testbed_version(node1.guid), "0.1")
 216         self.assertEquals(controller.get_factory_id(node1.guid), "Node")
 217
 218         traces_info = controller.traces_info()
 219         expected_traces_info = dict({
 220             1: dict({ # testbed guid
 221                 6: dict({ # element guid
 222                     'fake': dict({ # trace_id
 223                         'host': 'localhost',
 224                         'user': getpass.getuser(),
 225                         'filepath': '<test>'
 226                         })
 227                     })
 228                 })
 229             })
 230         self.assertEquals(traces_info, expected_traces_info)
 231
 232         controller.stop()
 233         controller.shutdown()
 234
 235     def test_daemonized_all_integration_recovery(self):
 236         exp_desc, desc, app, node1, node2, iface1, iface2 = self.make_test_experiment()
 237
 238         desc.set_attribute_value(DC.DEPLOYMENT_MODE, DC.MODE_DAEMON)
 239         inst_root_dir = os.path.join(self.root_dir, "instance")
 240         os.mkdir(inst_root_dir)
 241         desc.set_attribute_value(DC.ROOT_DIRECTORY, inst_root_dir)
 242
 243         xml = exp_desc.to_xml()
 244
 245         access_config = proxy.AccessConfiguration()
 246         access_config.set_attribute_value(DC.DEPLOYMENT_MODE, DC.MODE_DAEMON)
 247         access_config.set_attribute_value(DC.ROOT_DIRECTORY, self.root_dir)
 248         access_config.set_attribute_value(DC.DEPLOYMENT_ENVIRONMENT_SETUP,
 249             "export PYTHONPATH=%r:%r:$PYTHONPATH "
 250             "export NEPI_TESTBEDS='mock:mock mock2:mock2' " % (
 251                 os.path.dirname(os.path.dirname(mock.__file__)),
 252                 os.path.dirname(os.path.dirname(mock2.__file__)),))
 253         controller = proxy.create_experiment_controller(xml, access_config)
 254
 255         controller.start()
 256         while not controller.is_finished(app.guid):
 257             time.sleep(0.5)
 258         fake_result = controller.trace(app.guid, "fake")
 259         comp_result = """PING 10.0.0.2 (10.0.0.2) 56(84) bytes of data.
 260
 261 --- 10.0.0.2 ping statistics ---
 262 1 packets transmitted, 1 received, 0% packet loss, time 0ms
 263 """
 264         self.assertTrue(fake_result.startswith(comp_result))
 265
 266         self.assertEquals(controller.get_testbed_id(node1.guid), "mock")
 267         self.assertEquals(controller.get_testbed_version(node1.guid), "0.1")
 268         self.assertEquals(controller.get_factory_id(node1.guid), "Node")
 269
 270         # controller dies
 271         del controller
 272
 273         # recover
 274         access_config.set_attribute_value(DC.RECOVER,True)
 275         controller = proxy.create_experiment_controller(xml, access_config)
 276
 277         # test recovery
 278         self.assertTrue(controller.is_finished(app.guid))
 279         fake_result = controller.trace(app.guid, "fake")
 280         self.assertTrue(fake_result.startswith(comp_result))
 281
 282         controller.stop()
 283         controller.shutdown()
 284
 285     def test_reference_expressions(self):
 286         exp_desc, desc, app, node1, node2, iface1, iface2 = self.make_test_experiment()
 287
 288         iface1.set_attribute_value("label", "some")
 289         addr = iface1.add_address()
 290         addr.set_attribute_value("Address", "10.0.0.2")
 291         iface2.set_attribute_value("test", "{#[some].addr[0].[Address]#}")
 292
 293         xml = exp_desc.to_xml()
 294         access_config = None
 295         controller = proxy.create_experiment_controller(xml, access_config)
 296         controller.start()
 297         while not controller.is_finished(app.guid):
 298             time.sleep(0.5)
 299         fake_result = controller.trace(app.guid, "fake")
 300         comp_result = """PING 10.0.0.2 (10.0.0.2) 56(84) bytes of data.
 301
 302 --- 10.0.0.2 ping statistics ---
 303 1 packets transmitted, 1 received, 0% packet loss, time 0ms
 304 """
 305
 306         self.assertTrue(fake_result.startswith(comp_result))
 307
 308         self.assertEqual(
 309             controller._testbeds[desc.guid].get(iface2.guid, "test"),
 310             addr.get_attribute_value("Address") )
 311
 312         controller.stop()
 313         controller.shutdown()
 314
 315     def test_testbed_reference_expressions(self):
 316         exp_desc, desc, app, node1, node2, iface1, iface2 = self.make_test_experiment()
 317
 318         iface1.set_attribute_value("label", "some")
 319         addr = iface1.add_address()
 320         addr.set_attribute_value("Address", "10.0.0.2")
 321
 322         desc2 = exp_desc.add_testbed_description(
 323             FactoriesProvider("mock2") )
 324         desc2.set_attribute_value(DC.DEPLOYMENT_HOST, "{#[some].addr[0].[Address]#}")
 325         # DC.DEPLOYMENT_HOST should be ignored if DC.DEPLOYMENT_CONNECTION is not set
 326         # But it should be resolved anyway
 327
 328         xml = exp_desc.to_xml()
 329         access_config = None
 330         controller = proxy.create_experiment_controller(xml, access_config)
 331         controller.start()
 332         while not controller.is_finished(app.guid):
 333             time.sleep(0.5)
 334         fake_result = controller.trace(app.guid, "fake")
 335         comp_result = """PING 10.0.0.2 (10.0.0.2) 56(84) bytes of data.
 336
 337 --- 10.0.0.2 ping statistics ---
 338 1 packets transmitted, 1 received, 0% packet loss, time 0ms
 339 """
 340         self.assertTrue(fake_result.startswith(comp_result))
 341
 342         self.assertEqual(
 343             controller._deployment_config[desc2.guid]
 344                 .get_attribute_value(DC.DEPLOYMENT_HOST),
 345             addr.get_attribute_value("Address") )
 346
 347         controller.stop()
 348         controller.shutdown()
 349
 350     def test_ssh_daemonized_integration(self):
 351         exp_desc, desc, app, node1, node2, iface1, iface2 = self.make_test_experiment()
 352         env = test_util.test_environment()
 353
 354         desc.set_attribute_value(DC.DEPLOYMENT_MODE, DC.MODE_DAEMON)
 355         inst_root_dir = os.path.join(self.root_dir, "instance")
 356         os.mkdir(inst_root_dir)
 357         desc.set_attribute_value(DC.ROOT_DIRECTORY, inst_root_dir)
 358         xml = exp_desc.to_xml()
 359
 360         access_config = proxy.AccessConfiguration()
 361         access_config.set_attribute_value(DC.DEPLOYMENT_MODE, DC.MODE_DAEMON)
 362         access_config.set_attribute_value(DC.ROOT_DIRECTORY, self.root_dir)
 363         access_config.set_attribute_value(DC.DEPLOYMENT_COMMUNICATION, DC.ACCESS_SSH)
 364         access_config.set_attribute_value(DC.DEPLOYMENT_PORT, env.port)
 365         access_config.set_attribute_value(DC.USE_AGENT, True)
 366         access_config.set_attribute_value(DC.DEPLOYMENT_ENVIRONMENT_SETUP,
 367             "export PYTHONPATH=%r:%r:$PYTHONPATH "
 368             "export NEPI_TESTBEDS='mock:mock mock2:mock2' " % (
 369                 os.path.dirname(os.path.dirname(mock.__file__)),
 370                 os.path.dirname(os.path.dirname(mock2.__file__)),))
 371         controller = proxy.create_experiment_controller(xml, access_config)
 372
 373         try:
 374             controller.start()
 375             while not controller.is_finished(app.guid):
 376                 time.sleep(0.5)
 377             fake_result = controller.trace(app.guid, "fake")
 378             comp_result = """PING 10.0.0.2 (10.0.0.2) 56(84) bytes of data.
 379
 380 --- 10.0.0.2 ping statistics ---
 381 1 packets transmitted, 1 received, 0% packet loss, time 0ms
 382 """
 383             self.assertTrue(fake_result.startswith(comp_result))
 384         finally:
 385             controller.stop()
 386             controller.shutdown()
 387
 388     def ptest_experiment_suite(self):
 389         exp_desc, desc, app, node1, node2, iface1, iface2 = self.make_test_experiment()
 390
 391         xml = exp_desc.to_xml()
 392
 393         access_config = proxy.AccessConfiguration()
 394         access_config.set_attribute_value(DC.LOG_LEVEL, DC.DEBUG_LEVEL)
 395         access_config.set_attribute_value(DC.DEPLOYMENT_MODE, DC.MODE_DAEMON)
 396         access_config.set_attribute_value(DC.ROOT_DIRECTORY, self.root_dir)
 397         access_config.set_attribute_value(DC.DEPLOYMENT_ENVIRONMENT_SETUP,
 398             "export PYTHONPATH=%r:%r:$PYTHONPATH "
 399             "export NEPI_TESTBEDS='mock:mock mock2:mock2' " % (
 400                 os.path.dirname(os.path.dirname(mock.__file__)),
 401                 os.path.dirname(os.path.dirname(mock2.__file__)),))
 402
 403         print self.root_dir
 404         exp_suite = proxy.create_experiment_suite(xml, access_config, repetitions = 4)
 405         exp_suite.start()
 406         while not exp_suite.is_finished():
 407             time.sleep(0.5)
 408
 409         for access_config in exp_suite.get_access_configurations():
 410             access_config.set_attribute_value(DC.RECOVER, True)
 411             controller = proxy.create_experiment_controller(None, access_config)
 412
 413             fake_result = controller.trace(app.guid, "fake")
 414             comp_result = """PING 10.0.0.2 (10.0.0.2) 56(84) bytes of data.
 415
 416 --- 10.0.0.2 ping statistics ---
 417 1 packets transmitted, 1 received, 0% packet loss, time 0ms
 418 """
 419             self.assertTrue(fake_result.startswith(comp_result))
 420
 421             self.assertEquals(controller.get_testbed_id(node1.guid), "mock")
 422             self.assertEquals(controller.get_testbed_version(node1.guid), "0.1")
 423             self.assertEquals(controller.get_factory_id(node1.guid), "Node")
 424
 425         exp_suite.shutdown()
 426
 427 if __name__ == '__main__':
 428     unittest.main()
 429