2 # NEPI, a framework to manage network experiments
3 # Copyright (C) 2013 INRIA
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation, either version 3 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
18 # Author: Alina Quereilhac <alina.quereilhac@inria.fr>
20 from nepi.execution.ec import ExperimentController, ECState
27 class ExperimentRunner(object):
28 """ The ExperimentRunner entity is responsible of
29 re-running an experiment described by an ExperimentController
34 super(ExperimentRunner, self).__init__()
36 def run(self, ec, min_runs = 1, max_runs = -1, wait_time = 0,
37 wait_guids = [], compute_metric_callback = None,
38 evaluate_convergence_callback = None ):
39 """ Run a same experiment independently multiple times, until the
40 evaluate_convergence_callback function returns True
42 :param ec: Description of experiment to replicate.
43 The runner takes care of deploying the EC, so ec.deploy()
44 must not be invoked directly before or after invoking
46 :type ec: ExperimentController
48 :param min_runs: Minimum number of times the experiment must be
52 :param max_runs: Maximum number of times the experiment can be
56 :param wait_time: Time to wait in seconds on each run between invoking
57 ec.deploy() and ec.release().
58 :type wait_time: float
60 :param wait_guids: List of guids wait for finalization on each run.
61 This list is passed to ec.wait_finished()
62 :type wait_guids: list
64 :param compute_metric_callback: User defined function invoked after
65 each experiment run to compute a metric. The metric is usually
66 a network measurement obtained from the data collected
67 during experiment execution.
68 The function is invoked passing the ec and the run number as arguments.
69 It must return the value for the computed metric(s) (usually a single
70 numerical value, but it can be several).
72 metric = compute_metric_callback(ec, run)
74 :type compute_metric_callback: function
76 :param evaluate_convergence_callback: User defined function invoked after
77 computing the metric on each run, to evaluate the experiment was
78 run enough times. It takes the list of cumulated metrics produced by
79 the compute_metric_callback up to the current run, and decided
80 whether the metrics have statistically converged to a meaningful value
81 or not. It must return either True or False.
83 stop = evaluate_convergence_callback(ec, run, metrics)
85 If stop is True, then the runner will exit.
87 :type evaluate_convergence_callback: function
91 if (not max_runs or max_runs < 0) and not compute_metric_callback:
92 msg = "Undefined STOP condition, set stop_callback or max_runs"
93 raise RuntimeError, msg
95 if compute_metric_callback and not evaluate_convergence_callback:
96 evaluate_convergence_callback = self.evaluate_normal_convergence
97 ec.logger.info(" Treating data as normal to evaluate convergence. "
98 "Experiment will stop when the standard error with 95% "
99 "confidence interval is >= 5% of the mean of the collected samples ")
101 # Force persistence of experiment controller
104 filepath = ec.save(dirpath = ec.exp_dir)
113 ec = self.run_experiment(filepath, wait_time, wait_guids)
115 ec.logger.info(" RUN %d \n" % run)
117 if compute_metric_callback:
118 metric = compute_metric_callback(ec, run)
119 if metric is not None:
120 samples.append(metric)
122 if run >= min_runs and evaluate_convergence_callback:
123 if evaluate_convergence_callback(ec, run, samples):
126 if run >= min_runs and max_runs > -1 and run >= max_runs :
134 def evaluate_normal_convergence(self, ec, run, metrics):
135 """ Returns True when the confidence interval of the sample mean is
136 less than 5% of the mean value, for a 95% confidence level,
137 assuming normal distribution of the data
140 if len(metrics) == 0:
141 msg = "0 samples collected"
142 raise RuntimeError, msg
144 x = numpy.array(metrics)
147 se = std / math.sqrt(n)
150 # Confidence interval for 95% confidence level,
151 # assuming normally distributed data.
154 ec.logger.info(" RUN %d - SAMPLES %d MEAN %.2f STD %.2f CI (95%%) %.2f \n" % (
155 run, n, m, std, ci95 ) )
157 return m * 0.05 >= ci95
159 def run_experiment(self, filepath, wait_time, wait_guids):
160 """ Run an experiment based on the description stored
164 ec = ExperimentController.load(filepath)
168 ec.wait_finished(wait_guids)
169 time.sleep(wait_time)
173 if ec.state == ECState.FAILED:
174 raise RuntimeError, "Experiment failed"