--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+
+<html xmlns="http://www.w3.org/1999/xhtml">
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+
+ <title>nepi.execution.runner — NEPI 3.0 documentation</title>
+
+ <link rel="stylesheet" href="../../../_static/sphinxdoc.css" type="text/css" />
+ <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT: '../../../',
+ VERSION: '3.0',
+ COLLAPSE_INDEX: false,
+ FILE_SUFFIX: '.html',
+ HAS_SOURCE: true
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+ <link rel="top" title="NEPI 3.0 documentation" href="../../../index.html" />
+ <link rel="up" title="Module code" href="../../index.html" />
+ </head>
+ <body>
+ <div class="related">
+ <h3>Navigation</h3>
+ <ul>
+ <li class="right" style="margin-right: 10px">
+ <a href="../../../genindex.html" title="General Index"
+ accesskey="I">index</a></li>
+ <li class="right" >
+ <a href="../../../py-modindex.html" title="Python Module Index"
+ >modules</a> |</li>
+ <li><a href="../../../index.html">NEPI 3.0 documentation</a> »</li>
+ <li><a href="../../index.html" accesskey="U">Module code</a> »</li>
+ </ul>
+ </div>
+ <div class="sphinxsidebar">
+ <div class="sphinxsidebarwrapper">
+<div id="searchbox" style="display: none">
+ <h3>Quick search</h3>
+ <form class="search" action="../../../search.html" method="get">
+ <input type="text" name="q" />
+ <input type="submit" value="Go" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+ <p class="searchtip" style="font-size: 90%">
+ Enter search terms or a module, class or function name.
+ </p>
+</div>
+<script type="text/javascript">$('#searchbox').show(0);</script>
+ </div>
+ </div>
+
+ <div class="document">
+ <div class="documentwrapper">
+ <div class="bodywrapper">
+ <div class="body">
+
+ <h1>Source code for nepi.execution.runner</h1><div class="highlight"><pre>
+<span class="c">#</span>
+<span class="c"># NEPI, a framework to manage network experiments</span>
+<span class="c"># Copyright (C) 2013 INRIA</span>
+<span class="c">#</span>
+<span class="c"># This program is free software: you can redistribute it and/or modify</span>
+<span class="c"># it under the terms of the GNU General Public License version 2 as</span>
+<span class="c"># published by the Free Software Foundation;</span>
+<span class="c">#</span>
+<span class="c"># This program is distributed in the hope that it will be useful,</span>
+<span class="c"># but WITHOUT ANY WARRANTY; without even the implied warranty of</span>
+<span class="c"># MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the</span>
+<span class="c"># GNU General Public License for more details.</span>
+<span class="c">#</span>
+<span class="c"># You should have received a copy of the GNU General Public License</span>
+<span class="c"># along with this program. If not, see <http://www.gnu.org/licenses/>.</span>
+<span class="c">#</span>
+<span class="c"># Author: Alina Quereilhac <alina.quereilhac@inria.fr></span>
+
+<span class="kn">from</span> <span class="nn">nepi.execution.ec</span> <span class="kn">import</span> <span class="n">ExperimentController</span><span class="p">,</span> <span class="n">ECState</span>
+
+<span class="kn">import</span> <span class="nn">math</span>
+<span class="kn">import</span> <span class="nn">numpy</span>
+<span class="kn">import</span> <span class="nn">os</span>
+<span class="kn">import</span> <span class="nn">time</span>
+
+<div class="viewcode-block" id="ExperimentRunner"><a class="viewcode-back" href="../../../_layout/nepi.execution.html#nepi.execution.runner.ExperimentRunner">[docs]</a><span class="k">class</span> <span class="nc">ExperimentRunner</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+ <span class="sd">""" The ExperimentRunner entity is responsible of</span>
+<span class="sd"> re-running an experiment described by an ExperimentController </span>
+<span class="sd"> multiple time</span>
+
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">ExperimentRunner</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
+
+<div class="viewcode-block" id="ExperimentRunner.run"><a class="viewcode-back" href="../../../_layout/nepi.execution.html#nepi.execution.runner.ExperimentRunner.run">[docs]</a> <span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ec</span><span class="p">,</span> <span class="n">min_runs</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span> <span class="n">max_runs</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">wait_time</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
+ <span class="n">wait_guids</span> <span class="o">=</span> <span class="p">[],</span> <span class="n">compute_metric_callback</span> <span class="o">=</span> <span class="bp">None</span><span class="p">,</span>
+ <span class="n">evaluate_convergence_callback</span> <span class="o">=</span> <span class="bp">None</span> <span class="p">):</span>
+ <span class="sd">""" Run a same experiment independently multiple times, until the </span>
+<span class="sd"> evaluate_convergence_callback function returns True</span>
+
+<span class="sd"> :param ec: Description of experiment to replicate. </span>
+<span class="sd"> The runner takes care of deploying the EC, so ec.deploy()</span>
+<span class="sd"> must not be invoked directly before or after invoking</span>
+<span class="sd"> runner.run().</span>
+<span class="sd"> :type ec: ExperimentController</span>
+
+<span class="sd"> :param min_runs: Minimum number of times the experiment must be </span>
+<span class="sd"> replicated</span>
+<span class="sd"> :type min_runs: int</span>
+
+<span class="sd"> :param max_runs: Maximum number of times the experiment can be </span>
+<span class="sd"> replicated</span>
+<span class="sd"> :type max_runs: int</span>
+
+<span class="sd"> :param wait_time: Time to wait in seconds on each run between invoking</span>
+<span class="sd"> ec.deploy() and ec.release().</span>
+<span class="sd"> :type wait_time: float</span>
+
+<span class="sd"> :param wait_guids: List of guids wait for finalization on each run.</span>
+<span class="sd"> This list is passed to ec.wait_finished()</span>
+<span class="sd"> :type wait_guids: list </span>
+
+<span class="sd"> :param compute_metric_callback: User defined function invoked after </span>
+<span class="sd"> each experiment run to compute a metric. The metric is usually</span>
+<span class="sd"> a network measurement obtained from the data collected </span>
+<span class="sd"> during experiment execution.</span>
+<span class="sd"> The function is invoked passing the ec and the run number as arguments. </span>
+<span class="sd"> It must return the value for the computed metric(s) (usually a single </span>
+<span class="sd"> numerical value, but it can be several).</span>
+
+<span class="sd"> metric = compute_metric_callback(ec, run)</span>
+<span class="sd"> </span>
+<span class="sd"> :type compute_metric_callback: function </span>
+
+<span class="sd"> :param evaluate_convergence_callback: User defined function invoked after</span>
+<span class="sd"> computing the metric on each run, to evaluate the experiment was</span>
+<span class="sd"> run enough times. It takes the list of cumulated metrics produced by </span>
+<span class="sd"> the compute_metric_callback up to the current run, and decided </span>
+<span class="sd"> whether the metrics have statistically converged to a meaningful value</span>
+<span class="sd"> or not. It must return either True or False. </span>
+
+<span class="sd"> stop = evaluate_convergence_callback(ec, run, metrics)</span>
+
+<span class="sd"> If stop is True, then the runner will exit.</span>
+<span class="sd"> </span>
+<span class="sd"> :type evaluate_convergence_callback: function </span>
+
+<span class="sd"> """</span>
+
+ <span class="k">if</span> <span class="p">(</span><span class="ow">not</span> <span class="n">max_runs</span> <span class="ow">or</span> <span class="n">max_runs</span> <span class="o"><</span> <span class="mi">0</span><span class="p">)</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">compute_metric_callback</span><span class="p">:</span>
+ <span class="n">msg</span> <span class="o">=</span> <span class="s">"Undefined STOP condition, set stop_callback or max_runs"</span>
+ <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">,</span> <span class="n">msg</span>
+
+ <span class="k">if</span> <span class="n">compute_metric_callback</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">evaluate_convergence_callback</span><span class="p">:</span>
+ <span class="n">evaluate_convergence_callback</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">evaluate_normal_convergence</span>
+ <span class="n">ec</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s">" Treating data as normal to evaluate convergence. "</span>
+ <span class="s">"Experiment will stop when the standard error with 95% "</span>
+ <span class="s">"confidence interval is >= 5</span><span class="si">% o</span><span class="s">f the mean of the collected samples "</span><span class="p">)</span>
+
+ <span class="c"># Force persistence of experiment controller</span>
+ <span class="n">ec</span><span class="o">.</span><span class="n">_persist</span> <span class="o">=</span> <span class="bp">True</span>
+
+ <span class="n">filepath</span> <span class="o">=</span> <span class="n">ec</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">dirpath</span> <span class="o">=</span> <span class="n">ec</span><span class="o">.</span><span class="n">exp_dir</span><span class="p">)</span>
+
+ <span class="n">samples</span> <span class="o">=</span> <span class="p">[]</span>
+ <span class="n">run</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="n">stop</span> <span class="o">=</span> <span class="bp">False</span>
+
+ <span class="k">while</span> <span class="ow">not</span> <span class="n">stop</span><span class="p">:</span>
+ <span class="n">run</span> <span class="o">+=</span> <span class="mi">1</span>
+
+ <span class="n">ec</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_experiment</span><span class="p">(</span><span class="n">filepath</span><span class="p">,</span> <span class="n">wait_time</span><span class="p">,</span> <span class="n">wait_guids</span><span class="p">)</span>
+
+ <span class="n">ec</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s">" RUN </span><span class="si">%d</span><span class="s"> </span><span class="se">\n</span><span class="s">"</span> <span class="o">%</span> <span class="n">run</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">compute_metric_callback</span><span class="p">:</span>
+ <span class="n">metric</span> <span class="o">=</span> <span class="n">compute_metric_callback</span><span class="p">(</span><span class="n">ec</span><span class="p">,</span> <span class="n">run</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">metric</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
+ <span class="n">samples</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">metric</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">run</span> <span class="o">>=</span> <span class="n">min_runs</span> <span class="ow">and</span> <span class="n">evaluate_convergence_callback</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">evaluate_convergence_callback</span><span class="p">(</span><span class="n">ec</span><span class="p">,</span> <span class="n">run</span><span class="p">,</span> <span class="n">samples</span><span class="p">):</span>
+ <span class="n">stop</span> <span class="o">=</span> <span class="bp">True</span>
+
+ <span class="k">if</span> <span class="n">run</span> <span class="o">>=</span> <span class="n">min_runs</span> <span class="ow">and</span> <span class="n">max_runs</span> <span class="o">></span> <span class="o">-</span><span class="mi">1</span> <span class="ow">and</span> <span class="n">run</span> <span class="o">>=</span> <span class="n">max_runs</span> <span class="p">:</span>
+ <span class="n">stop</span> <span class="o">=</span> <span class="bp">True</span>
+
+ <span class="n">ec</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span>
+ <span class="k">del</span> <span class="n">ec</span>
+
+ <span class="k">return</span> <span class="n">run</span>
+</div>
+<div class="viewcode-block" id="ExperimentRunner.evaluate_normal_convergence"><a class="viewcode-back" href="../../../_layout/nepi.execution.html#nepi.execution.runner.ExperimentRunner.evaluate_normal_convergence">[docs]</a> <span class="k">def</span> <span class="nf">evaluate_normal_convergence</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ec</span><span class="p">,</span> <span class="n">run</span><span class="p">,</span> <span class="n">metrics</span><span class="p">):</span>
+ <span class="sd">""" Returns True when the confidence interval of the sample mean is</span>
+<span class="sd"> less than 5% of the mean value, for a 95% confidence level,</span>
+<span class="sd"> assuming normal distribution of the data</span>
+<span class="sd"> """</span>
+
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">metrics</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="n">msg</span> <span class="o">=</span> <span class="s">"0 samples collected"</span>
+ <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">,</span> <span class="n">msg</span>
+
+ <span class="n">x</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">metrics</span><span class="p">)</span>
+ <span class="n">n</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">metrics</span><span class="p">)</span>
+ <span class="n">std</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">std</span><span class="p">()</span>
+ <span class="n">se</span> <span class="o">=</span> <span class="n">std</span> <span class="o">/</span> <span class="n">math</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">n</span><span class="p">)</span>
+ <span class="n">m</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
+
+ <span class="c"># Confidence interval for 95% confidence level, </span>
+ <span class="c"># assuming normally distributed data.</span>
+ <span class="n">ci95</span> <span class="o">=</span> <span class="n">se</span> <span class="o">*</span> <span class="mi">2</span>
+
+ <span class="n">ec</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s">" RUN </span><span class="si">%d</span><span class="s"> - SAMPLES </span><span class="si">%d</span><span class="s"> MEAN </span><span class="si">%.2f</span><span class="s"> STD </span><span class="si">%.2f</span><span class="s"> CI (95</span><span class="si">%%</span><span class="s">) </span><span class="si">%.2f</span><span class="s"> </span><span class="se">\n</span><span class="s">"</span> <span class="o">%</span> <span class="p">(</span>
+ <span class="n">run</span><span class="p">,</span> <span class="n">n</span><span class="p">,</span> <span class="n">m</span><span class="p">,</span> <span class="n">std</span><span class="p">,</span> <span class="n">ci95</span> <span class="p">)</span> <span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">m</span> <span class="o">*</span> <span class="mf">0.05</span> <span class="o">>=</span> <span class="n">ci95</span>
+</div>
+<div class="viewcode-block" id="ExperimentRunner.run_experiment"><a class="viewcode-back" href="../../../_layout/nepi.execution.html#nepi.execution.runner.ExperimentRunner.run_experiment">[docs]</a> <span class="k">def</span> <span class="nf">run_experiment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filepath</span><span class="p">,</span> <span class="n">wait_time</span><span class="p">,</span> <span class="n">wait_guids</span><span class="p">):</span>
+ <span class="sd">""" Run an experiment based on the description stored</span>
+<span class="sd"> in filepath.</span>
+
+<span class="sd"> """</span>
+ <span class="n">ec</span> <span class="o">=</span> <span class="n">ExperimentController</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">filepath</span><span class="p">)</span>
+
+ <span class="n">ec</span><span class="o">.</span><span class="n">deploy</span><span class="p">()</span>
+
+ <span class="n">ec</span><span class="o">.</span><span class="n">wait_finished</span><span class="p">(</span><span class="n">wait_guids</span><span class="p">)</span>
+ <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">wait_time</span><span class="p">)</span>
+
+ <span class="n">ec</span><span class="o">.</span><span class="n">release</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="n">ec</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">ECState</span><span class="o">.</span><span class="n">FAILED</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">,</span> <span class="s">"Experiment failed"</span>
+
+ <span class="k">return</span> <span class="n">ec</span>
+</pre></div></div></div>
+
+ </div>
+ </div>
+ </div>
+ <div class="clearer"></div>
+ </div>
+ <div class="related">
+ <h3>Navigation</h3>
+ <ul>
+ <li class="right" style="margin-right: 10px">
+ <a href="../../../genindex.html" title="General Index"
+ >index</a></li>
+ <li class="right" >
+ <a href="../../../py-modindex.html" title="Python Module Index"
+ >modules</a> |</li>
+ <li><a href="../../../index.html">NEPI 3.0 documentation</a> »</li>
+ <li><a href="../../index.html" >Module code</a> »</li>
+ </ul>
+ </div>
+ <div class="footer">
+ © Copyright 2014, Alina Quereilhac, Lucia Guevgeozian Odizzio, Julien Tribino.
+ Created using <a href="http://sphinx-doc.org/">Sphinx</a> 1.2.3.
+ </div>
+ </body>
+</html>
\ No newline at end of file
git://git.onelab.eu / nepi.git / blobdiff