1 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
2 "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5 <html xmlns="http://www.w3.org/1999/xhtml">
7 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
9 <title>nepi.execution.runner — NEPI 3.0 documentation</title>
11 <link rel="stylesheet" href="../../../_static/sphinxdoc.css" type="text/css" />
12 <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
14 <script type="text/javascript">
15 var DOCUMENTATION_OPTIONS = {
16 URL_ROOT: '../../../',
18 COLLAPSE_INDEX: false,
23 <script type="text/javascript" src="../../../_static/jquery.js"></script>
24 <script type="text/javascript" src="../../../_static/underscore.js"></script>
25 <script type="text/javascript" src="../../../_static/doctools.js"></script>
26 <link rel="top" title="NEPI 3.0 documentation" href="../../../index.html" />
27 <link rel="up" title="Module code" href="../../index.html" />
33 <li class="right" style="margin-right: 10px">
34 <a href="../../../genindex.html" title="General Index"
35 accesskey="I">index</a></li>
37 <a href="../../../py-modindex.html" title="Python Module Index"
39 <li><a href="../../../index.html">NEPI 3.0 documentation</a> »</li>
40 <li><a href="../../index.html" accesskey="U">Module code</a> »</li>
43 <div class="sphinxsidebar">
44 <div class="sphinxsidebarwrapper">
45 <div id="searchbox" style="display: none">
47 <form class="search" action="../../../search.html" method="get">
48 <input type="text" name="q" />
49 <input type="submit" value="Go" />
50 <input type="hidden" name="check_keywords" value="yes" />
51 <input type="hidden" name="area" value="default" />
53 <p class="searchtip" style="font-size: 90%">
54 Enter search terms or a module, class or function name.
57 <script type="text/javascript">$('#searchbox').show(0);</script>
61 <div class="document">
62 <div class="documentwrapper">
63 <div class="bodywrapper">
66 <h1>Source code for nepi.execution.runner</h1><div class="highlight"><pre>
67 <span class="c">#</span>
68 <span class="c"># NEPI, a framework to manage network experiments</span>
69 <span class="c"># Copyright (C) 2013 INRIA</span>
70 <span class="c">#</span>
71 <span class="c"># This program is free software: you can redistribute it and/or modify</span>
72 <span class="c"># it under the terms of the GNU General Public License version 2 as</span>
73 <span class="c"># published by the Free Software Foundation;</span>
74 <span class="c">#</span>
75 <span class="c"># This program is distributed in the hope that it will be useful,</span>
76 <span class="c"># but WITHOUT ANY WARRANTY; without even the implied warranty of</span>
77 <span class="c"># MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the</span>
78 <span class="c"># GNU General Public License for more details.</span>
79 <span class="c">#</span>
80 <span class="c"># You should have received a copy of the GNU General Public License</span>
81 <span class="c"># along with this program. If not, see <http://www.gnu.org/licenses/>.</span>
82 <span class="c">#</span>
83 <span class="c"># Author: Alina Quereilhac <alina.quereilhac@inria.fr></span>
85 <span class="kn">from</span> <span class="nn">nepi.execution.ec</span> <span class="kn">import</span> <span class="n">ExperimentController</span><span class="p">,</span> <span class="n">ECState</span>
87 <span class="kn">import</span> <span class="nn">math</span>
88 <span class="kn">import</span> <span class="nn">numpy</span>
89 <span class="kn">import</span> <span class="nn">os</span>
90 <span class="kn">import</span> <span class="nn">time</span>
92 <div class="viewcode-block" id="ExperimentRunner"><a class="viewcode-back" href="../../../_layout/nepi.execution.html#nepi.execution.runner.ExperimentRunner">[docs]</a><span class="k">class</span> <span class="nc">ExperimentRunner</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
93 <span class="sd">""" The ExperimentRunner entity is responsible of</span>
94 <span class="sd"> re-running an experiment described by an ExperimentController </span>
95 <span class="sd"> multiple time</span>
97 <span class="sd"> """</span>
98 <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
99 <span class="nb">super</span><span class="p">(</span><span class="n">ExperimentRunner</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
101 <div class="viewcode-block" id="ExperimentRunner.run"><a class="viewcode-back" href="../../../_layout/nepi.execution.html#nepi.execution.runner.ExperimentRunner.run">[docs]</a> <span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ec</span><span class="p">,</span> <span class="n">min_runs</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span> <span class="n">max_runs</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">wait_time</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
102 <span class="n">wait_guids</span> <span class="o">=</span> <span class="p">[],</span> <span class="n">compute_metric_callback</span> <span class="o">=</span> <span class="bp">None</span><span class="p">,</span>
103 <span class="n">evaluate_convergence_callback</span> <span class="o">=</span> <span class="bp">None</span> <span class="p">):</span>
104 <span class="sd">""" Run a same experiment independently multiple times, until the </span>
105 <span class="sd"> evaluate_convergence_callback function returns True</span>
107 <span class="sd"> :param ec: Description of experiment to replicate. </span>
108 <span class="sd"> The runner takes care of deploying the EC, so ec.deploy()</span>
109 <span class="sd"> must not be invoked directly before or after invoking</span>
110 <span class="sd"> runner.run().</span>
111 <span class="sd"> :type ec: ExperimentController</span>
113 <span class="sd"> :param min_runs: Minimum number of times the experiment must be </span>
114 <span class="sd"> replicated</span>
115 <span class="sd"> :type min_runs: int</span>
117 <span class="sd"> :param max_runs: Maximum number of times the experiment can be </span>
118 <span class="sd"> replicated</span>
119 <span class="sd"> :type max_runs: int</span>
121 <span class="sd"> :param wait_time: Time to wait in seconds on each run between invoking</span>
122 <span class="sd"> ec.deploy() and ec.release().</span>
123 <span class="sd"> :type wait_time: float</span>
125 <span class="sd"> :param wait_guids: List of guids wait for finalization on each run.</span>
126 <span class="sd"> This list is passed to ec.wait_finished()</span>
127 <span class="sd"> :type wait_guids: list </span>
129 <span class="sd"> :param compute_metric_callback: User defined function invoked after </span>
130 <span class="sd"> each experiment run to compute a metric. The metric is usually</span>
131 <span class="sd"> a network measurement obtained from the data collected </span>
132 <span class="sd"> during experiment execution.</span>
133 <span class="sd"> The function is invoked passing the ec and the run number as arguments. </span>
134 <span class="sd"> It must return the value for the computed metric(s) (usually a single </span>
135 <span class="sd"> numerical value, but it can be several).</span>
137 <span class="sd"> metric = compute_metric_callback(ec, run)</span>
138 <span class="sd"> </span>
139 <span class="sd"> :type compute_metric_callback: function </span>
141 <span class="sd"> :param evaluate_convergence_callback: User defined function invoked after</span>
142 <span class="sd"> computing the metric on each run, to evaluate the experiment was</span>
143 <span class="sd"> run enough times. It takes the list of cumulated metrics produced by </span>
144 <span class="sd"> the compute_metric_callback up to the current run, and decided </span>
145 <span class="sd"> whether the metrics have statistically converged to a meaningful value</span>
146 <span class="sd"> or not. It must return either True or False. </span>
148 <span class="sd"> stop = evaluate_convergence_callback(ec, run, metrics)</span>
150 <span class="sd"> If stop is True, then the runner will exit.</span>
151 <span class="sd"> </span>
152 <span class="sd"> :type evaluate_convergence_callback: function </span>
154 <span class="sd"> """</span>
156 <span class="k">if</span> <span class="p">(</span><span class="ow">not</span> <span class="n">max_runs</span> <span class="ow">or</span> <span class="n">max_runs</span> <span class="o"><</span> <span class="mi">0</span><span class="p">)</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">compute_metric_callback</span><span class="p">:</span>
157 <span class="n">msg</span> <span class="o">=</span> <span class="s">"Undefined STOP condition, set stop_callback or max_runs"</span>
158 <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">,</span> <span class="n">msg</span>
160 <span class="k">if</span> <span class="n">compute_metric_callback</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">evaluate_convergence_callback</span><span class="p">:</span>
161 <span class="n">evaluate_convergence_callback</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">evaluate_normal_convergence</span>
162 <span class="n">ec</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s">" Treating data as normal to evaluate convergence. "</span>
163 <span class="s">"Experiment will stop when the standard error with 95% "</span>
164 <span class="s">"confidence interval is >= 5</span><span class="si">% o</span><span class="s">f the mean of the collected samples "</span><span class="p">)</span>
166 <span class="c"># Force persistence of experiment controller</span>
167 <span class="n">ec</span><span class="o">.</span><span class="n">_persist</span> <span class="o">=</span> <span class="bp">True</span>
169 <span class="n">filepath</span> <span class="o">=</span> <span class="n">ec</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">dirpath</span> <span class="o">=</span> <span class="n">ec</span><span class="o">.</span><span class="n">exp_dir</span><span class="p">)</span>
171 <span class="n">samples</span> <span class="o">=</span> <span class="p">[]</span>
172 <span class="n">run</span> <span class="o">=</span> <span class="mi">0</span>
173 <span class="n">stop</span> <span class="o">=</span> <span class="bp">False</span>
175 <span class="k">while</span> <span class="ow">not</span> <span class="n">stop</span><span class="p">:</span>
176 <span class="n">run</span> <span class="o">+=</span> <span class="mi">1</span>
178 <span class="n">ec</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_experiment</span><span class="p">(</span><span class="n">filepath</span><span class="p">,</span> <span class="n">wait_time</span><span class="p">,</span> <span class="n">wait_guids</span><span class="p">)</span>
180 <span class="n">ec</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s">" RUN </span><span class="si">%d</span><span class="s"> </span><span class="se">\n</span><span class="s">"</span> <span class="o">%</span> <span class="n">run</span><span class="p">)</span>
182 <span class="k">if</span> <span class="n">compute_metric_callback</span><span class="p">:</span>
183 <span class="n">metric</span> <span class="o">=</span> <span class="n">compute_metric_callback</span><span class="p">(</span><span class="n">ec</span><span class="p">,</span> <span class="n">run</span><span class="p">)</span>
184 <span class="k">if</span> <span class="n">metric</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
185 <span class="n">samples</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">metric</span><span class="p">)</span>
187 <span class="k">if</span> <span class="n">run</span> <span class="o">>=</span> <span class="n">min_runs</span> <span class="ow">and</span> <span class="n">evaluate_convergence_callback</span><span class="p">:</span>
188 <span class="k">if</span> <span class="n">evaluate_convergence_callback</span><span class="p">(</span><span class="n">ec</span><span class="p">,</span> <span class="n">run</span><span class="p">,</span> <span class="n">samples</span><span class="p">):</span>
189 <span class="n">stop</span> <span class="o">=</span> <span class="bp">True</span>
191 <span class="k">if</span> <span class="n">run</span> <span class="o">>=</span> <span class="n">min_runs</span> <span class="ow">and</span> <span class="n">max_runs</span> <span class="o">></span> <span class="o">-</span><span class="mi">1</span> <span class="ow">and</span> <span class="n">run</span> <span class="o">>=</span> <span class="n">max_runs</span> <span class="p">:</span>
192 <span class="n">stop</span> <span class="o">=</span> <span class="bp">True</span>
194 <span class="n">ec</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span>
195 <span class="k">del</span> <span class="n">ec</span>
197 <span class="k">return</span> <span class="n">run</span>
199 <div class="viewcode-block" id="ExperimentRunner.evaluate_normal_convergence"><a class="viewcode-back" href="../../../_layout/nepi.execution.html#nepi.execution.runner.ExperimentRunner.evaluate_normal_convergence">[docs]</a> <span class="k">def</span> <span class="nf">evaluate_normal_convergence</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ec</span><span class="p">,</span> <span class="n">run</span><span class="p">,</span> <span class="n">metrics</span><span class="p">):</span>
200 <span class="sd">""" Returns True when the confidence interval of the sample mean is</span>
201 <span class="sd"> less than 5% of the mean value, for a 95% confidence level,</span>
202 <span class="sd"> assuming normal distribution of the data</span>
203 <span class="sd"> """</span>
205 <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">metrics</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
206 <span class="n">msg</span> <span class="o">=</span> <span class="s">"0 samples collected"</span>
207 <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">,</span> <span class="n">msg</span>
209 <span class="n">x</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">metrics</span><span class="p">)</span>
210 <span class="n">n</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">metrics</span><span class="p">)</span>
211 <span class="n">std</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">std</span><span class="p">()</span>
212 <span class="n">se</span> <span class="o">=</span> <span class="n">std</span> <span class="o">/</span> <span class="n">math</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">n</span><span class="p">)</span>
213 <span class="n">m</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
215 <span class="c"># Confidence interval for 95% confidence level, </span>
216 <span class="c"># assuming normally distributed data.</span>
217 <span class="n">ci95</span> <span class="o">=</span> <span class="n">se</span> <span class="o">*</span> <span class="mi">2</span>
219 <span class="n">ec</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s">" RUN </span><span class="si">%d</span><span class="s"> - SAMPLES </span><span class="si">%d</span><span class="s"> MEAN </span><span class="si">%.2f</span><span class="s"> STD </span><span class="si">%.2f</span><span class="s"> CI (95</span><span class="si">%%</span><span class="s">) </span><span class="si">%.2f</span><span class="s"> </span><span class="se">\n</span><span class="s">"</span> <span class="o">%</span> <span class="p">(</span>
220 <span class="n">run</span><span class="p">,</span> <span class="n">n</span><span class="p">,</span> <span class="n">m</span><span class="p">,</span> <span class="n">std</span><span class="p">,</span> <span class="n">ci95</span> <span class="p">)</span> <span class="p">)</span>
222 <span class="k">return</span> <span class="n">m</span> <span class="o">*</span> <span class="mf">0.05</span> <span class="o">>=</span> <span class="n">ci95</span>
224 <div class="viewcode-block" id="ExperimentRunner.run_experiment"><a class="viewcode-back" href="../../../_layout/nepi.execution.html#nepi.execution.runner.ExperimentRunner.run_experiment">[docs]</a> <span class="k">def</span> <span class="nf">run_experiment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filepath</span><span class="p">,</span> <span class="n">wait_time</span><span class="p">,</span> <span class="n">wait_guids</span><span class="p">):</span>
225 <span class="sd">""" Run an experiment based on the description stored</span>
226 <span class="sd"> in filepath.</span>
228 <span class="sd"> """</span>
229 <span class="n">ec</span> <span class="o">=</span> <span class="n">ExperimentController</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">filepath</span><span class="p">)</span>
231 <span class="n">ec</span><span class="o">.</span><span class="n">deploy</span><span class="p">()</span>
233 <span class="n">ec</span><span class="o">.</span><span class="n">wait_finished</span><span class="p">(</span><span class="n">wait_guids</span><span class="p">)</span>
234 <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">wait_time</span><span class="p">)</span>
236 <span class="n">ec</span><span class="o">.</span><span class="n">release</span><span class="p">()</span>
238 <span class="k">if</span> <span class="n">ec</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">ECState</span><span class="o">.</span><span class="n">FAILED</span><span class="p">:</span>
239 <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">,</span> <span class="s">"Experiment failed"</span>
241 <span class="k">return</span> <span class="n">ec</span>
242 </pre></div></div></div>
247 <div class="clearer"></div>
249 <div class="related">
252 <li class="right" style="margin-right: 10px">
253 <a href="../../../genindex.html" title="General Index"
256 <a href="../../../py-modindex.html" title="Python Module Index"
258 <li><a href="../../../index.html">NEPI 3.0 documentation</a> »</li>
259 <li><a href="../../index.html" >Module code</a> »</li>
263 © Copyright 2014, Alina Quereilhac, Lucia Guevgeozian Odizzio, Julien Tribino.
264 Created using <a href="http://sphinx-doc.org/">Sphinx</a> 1.2.3.