Adding NS3 FDNetDevice RM

[nepi.git] / src / nepi / execution / runner.py

diff --git a/src/nepi/execution/runner.py b/src/nepi/execution/runner.py
index 21a20cb..064c2e2 100644 (file)
--- a/src/nepi/execution/runner.py
+++ b/src/nepi/execution/runner.py
@@ -25,9 +25,9 @@ import os
 import time
 
 class ExperimentRunner(object):
-    """ The ExperimentRunner entity is reponsible of
+    """ The ExperimentRunner entity is responsible of
     re-running an experiment described by an ExperimentController 
-    multiple time.
+    multiple time
 
     """
     def __init__(self):
@@ -36,39 +36,49 @@ class ExperimentRunner(object):
     def run(self, ec, min_runs = 1, max_runs = -1, wait_time = 0, 
             wait_guids = [], compute_metric_callback = None, 
             evaluate_convergence_callback = None ):
-        """ Re-runs a same experiment multiple times
+        """ Run a same experiment independently multiple times, until the 
+        evaluate_convergence_callback function returns True
 
-        :param ec: Experiment description of experiment to run
+        :param ec: Description of experiment to replicate. 
+            The runner takes care of deploying the EC, so ec.deploy()
+            must not be invoked directly before or after invoking
+            runner.run().
         :type ec: ExperimentController
 
-        :param min_runs: Minimum number of repetitions for experiment
+        :param min_runs: Minimum number of times the experiment must be 
+            replicated
         :type min_runs: int
 
-        :param max_runs: Maximum number of repetitions for experiment
+        :param max_runs: Maximum number of times the experiment can be 
+            replicated
         :type max_runs: int
 
-        :param wait_time: Time to wait in seconds between invoking
-            ec.deploy() and ec.release()
+        :param wait_time: Time to wait in seconds on each run between invoking
+            ec.deploy() and ec.release().
         :type wait_time: float
 
-        :param wait_guids: List of guids to pass to ec.wait_finished
-            after invoking ec.deploy()
+        :param wait_guids: List of guids wait for finalization on each run.
+            This list is passed to ec.wait_finished()
         :type wait_guids: list 
 
-        :param compute_metric_callback: function to invoke after each 
-            experiment run, to compute an experiment metric. 
-            It will be invoked with the ec and the run count as arguments,
-            and it must return the metric value(s) computed for the run
+        :param compute_metric_callback: User defined function invoked after 
+            each experiment run to compute a metric. The metric is usually
+            a network measurement obtained from the data collected 
+            during experiment execution.
+            The function is invoked passing the ec and the run number as arguments. 
+            It must return the value for the computed metric(s) (usually a single 
+            numerical value, but it can be several).
 
                 metric = compute_metric_callback(ec, run)
             
         :type compute_metric_callback: function 
 
-        :param evaluate_convergence_callback: function to evaluate whether the 
-            collected metric samples have converged and the experiment runner
-            can stop. It will be invoked with the ec, the run count and the
-            list of collected metric samples as argument, and it must return
-            either True or False
+        :param evaluate_convergence_callback: User defined function invoked after
+            computing the metric on each run, to evaluate the experiment was
+            run enough times. It takes the list of cumulated metrics produced by 
+            the compute_metric_callback up to the current run, and decided 
+            whether the metrics have statistically converged to a meaningful value
+            or not. It must return either True or False. 
 
                 stop = evaluate_convergence_callback(ec, run, metrics)
 
@@ -121,19 +131,24 @@ class ExperimentRunner(object):
 
         return run
 
-    def evaluate_normal_convergence(self, ec, run, samples):
-        if len(samples) == 0:
+    def evaluate_normal_convergence(self, ec, run, metrics):
+        """ Returns True when the confidence interval of the sample mean is
+        less than 5% of the mean value, for a 95% confidence level,
+        assuming normal distribution of the data
+        """
+
+        if len(metrics) == 0:
             msg = "0 samples collected"
             raise RuntimeError, msg
 
-        x = numpy.array(samples)
-        n = len(samples)
+        x = numpy.array(metrics)
+        n = len(metrics)
         std = x.std()
         se = std / math.sqrt(n)
         m = x.mean()
 
-        # confidence interval for 95% confidence level.
-        # Asuming samples are normally distributed
+        # Confidence interval for 95% confidence level, 
+        # assuming normally distributed data.
         ci95 = se * 2
         
         ec.logger.info(" RUN %d - SAMPLES %d MEAN %.2f STD %.2f CI (95%%) %.2f \n" % (
@@ -141,7 +156,11 @@ class ExperimentRunner(object):
 
         return m * 0.05 >= ci95
 
-    def run_experiment(self, filepath, wait_time, wait_guids): 
+    def run_experiment(self, filepath, wait_time, wait_guids):
+        """ Run an experiment based on the description stored
+        in filepath.
+
+        """
         ec = ExperimentController.load(filepath)
 
         ec.deploy()