doc/sphinx/_build/html/_modules/nepi/execution/runner.html

   1 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
   2   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
   3
   4
   5 <html xmlns="http://www.w3.org/1999/xhtml">
   6   <head>
   7     <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
   8
   9     <title>nepi.execution.runner &mdash; NEPI 3.2 documentation</title>
  10
  11     <link rel="stylesheet" href="../../../_static/sphinxdoc.css" type="text/css" />
  12     <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
  13
  14     <script type="text/javascript">
  15       var DOCUMENTATION_OPTIONS = {
  16         URL_ROOT:    '../../../',
  17         VERSION:     '3.2',
  18         COLLAPSE_INDEX: false,
  19         FILE_SUFFIX: '.html',
  20         HAS_SOURCE:  true
  21       };
  22     </script>
  23     <script type="text/javascript" src="../../../_static/jquery.js"></script>
  24     <script type="text/javascript" src="../../../_static/underscore.js"></script>
  25     <script type="text/javascript" src="../../../_static/doctools.js"></script>
  26     <link rel="top" title="NEPI 3.2 documentation" href="../../../index.html" />
  27     <link rel="up" title="Module code" href="../../index.html" />
  28   </head>
  29   <body>
  30     <div class="related">
  31       <h3>Navigation</h3>
  32       <ul>
  33         <li class="right" style="margin-right: 10px">
  34           <a href="../../../genindex.html" title="General Index"
  35              accesskey="I">index</a></li>
  36         <li class="right" >
  37           <a href="../../../py-modindex.html" title="Python Module Index"
  38              >modules</a> |</li>
  39         <li><a href="../../../index.html">NEPI 3.2 documentation</a> &raquo;</li>
  40           <li><a href="../../index.html" accesskey="U">Module code</a> &raquo;</li>
  41       </ul>
  42     </div>
  43       <div class="sphinxsidebar">
  44         <div class="sphinxsidebarwrapper">
  45 <div id="searchbox" style="display: none">
  46   <h3>Quick search</h3>
  47     <form class="search" action="../../../search.html" method="get">
  48       <input type="text" name="q" />
  49       <input type="submit" value="Go" />
  50       <input type="hidden" name="check_keywords" value="yes" />
  51       <input type="hidden" name="area" value="default" />
  52     </form>
  53     <p class="searchtip" style="font-size: 90%">
  54     Enter search terms or a module, class or function name.
  55     </p>
  56 </div>
  57 <script type="text/javascript">$('#searchbox').show(0);</script>
  58         </div>
  59       </div>
  60
  61     <div class="document">
  62       <div class="documentwrapper">
  63         <div class="bodywrapper">
  64           <div class="body">
  65
  66   <h1>Source code for nepi.execution.runner</h1><div class="highlight"><pre>
  67 <span class="c">#</span>
  68 <span class="c">#    NEPI, a framework to manage network experiments</span>
  69 <span class="c">#    Copyright (C) 2013 INRIA</span>
  70 <span class="c">#</span>
  71 <span class="c">#    This program is free software: you can redistribute it and/or modify</span>
  72 <span class="c">#    it under the terms of the GNU General Public License version 2 as</span>
  73 <span class="c">#    published by the Free Software Foundation;</span>
  74 <span class="c">#</span>
  75 <span class="c">#    This program is distributed in the hope that it will be useful,</span>
  76 <span class="c">#    but WITHOUT ANY WARRANTY; without even the implied warranty of</span>
  77 <span class="c">#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the</span>
  78 <span class="c">#    GNU General Public License for more details.</span>
  79 <span class="c">#</span>
  80 <span class="c">#    You should have received a copy of the GNU General Public License</span>
  81 <span class="c">#    along with this program.  If not, see &lt;http://www.gnu.org/licenses/&gt;.</span>
  82 <span class="c">#</span>
  83 <span class="c"># Author: Alina Quereilhac &lt;alina.quereilhac@inria.fr&gt;</span>
  84
  85 <span class="kn">from</span> <span class="nn">nepi.execution.ec</span> <span class="kn">import</span> <span class="n">ExperimentController</span><span class="p">,</span> <span class="n">ECState</span>
  86
  87 <span class="kn">import</span> <span class="nn">math</span>
  88 <span class="kn">import</span> <span class="nn">numpy</span>
  89 <span class="kn">import</span> <span class="nn">os</span>
  90 <span class="kn">import</span> <span class="nn">time</span>
  91
  92 <div class="viewcode-block" id="ExperimentRunner"><a class="viewcode-back" href="../../../_layout/nepi.execution.html#nepi.execution.runner.ExperimentRunner">[docs]</a><span class="k">class</span> <span class="nc">ExperimentRunner</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
  93     <span class="sd">&quot;&quot;&quot; The ExperimentRunner entity is responsible of</span>
  94 <span class="sd">    re-running an experiment described by an ExperimentController </span>
  95 <span class="sd">    multiple time</span>
  96
  97 <span class="sd">    &quot;&quot;&quot;</span>
  98     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
  99         <span class="nb">super</span><span class="p">(</span><span class="n">ExperimentRunner</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
 100
 101 <div class="viewcode-block" id="ExperimentRunner.run"><a class="viewcode-back" href="../../../_layout/nepi.execution.html#nepi.execution.runner.ExperimentRunner.run">[docs]</a>    <span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ec</span><span class="p">,</span> <span class="n">min_runs</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span> <span class="n">max_runs</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">wait_time</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
 102             <span class="n">wait_guids</span> <span class="o">=</span> <span class="p">[],</span> <span class="n">compute_metric_callback</span> <span class="o">=</span> <span class="bp">None</span><span class="p">,</span>
 103             <span class="n">evaluate_convergence_callback</span> <span class="o">=</span> <span class="bp">None</span> <span class="p">):</span>
 104         <span class="sd">&quot;&quot;&quot; Run a same experiment independently multiple times, until the </span>
 105 <span class="sd">        evaluate_convergence_callback function returns True</span>
 106
 107 <span class="sd">        :param ec: Description of experiment to replicate. </span>
 108 <span class="sd">            The runner takes care of deploying the EC, so ec.deploy()</span>
 109 <span class="sd">            must not be invoked directly before or after invoking</span>
 110 <span class="sd">            runner.run().</span>
 111 <span class="sd">        :type ec: ExperimentController</span>
 112
 113 <span class="sd">        :param min_runs: Minimum number of times the experiment must be </span>
 114 <span class="sd">            replicated</span>
 115 <span class="sd">        :type min_runs: int</span>
 116
 117 <span class="sd">        :param max_runs: Maximum number of times the experiment can be </span>
 118 <span class="sd">            replicated</span>
 119 <span class="sd">        :type max_runs: int</span>
 120
 121 <span class="sd">        :param wait_time: Time to wait in seconds on each run between invoking</span>
 122 <span class="sd">            ec.deploy() and ec.release().</span>
 123 <span class="sd">        :type wait_time: float</span>
 124
 125 <span class="sd">        :param wait_guids: List of guids wait for finalization on each run.</span>
 126 <span class="sd">            This list is passed to ec.wait_finished()</span>
 127 <span class="sd">        :type wait_guids: list </span>
 128
 129 <span class="sd">        :param compute_metric_callback: User defined function invoked after </span>
 130 <span class="sd">            each experiment run to compute a metric. The metric is usually</span>
 131 <span class="sd">            a network measurement obtained from the data collected </span>
 132 <span class="sd">            during experiment execution.</span>
 133 <span class="sd">            The function is invoked passing the ec and the run number as arguments. </span>
 134 <span class="sd">            It must return the value for the computed metric(s) (usually a single </span>
 135 <span class="sd">            numerical value, but it can be several).</span>
 136
 137 <span class="sd">                metric = compute_metric_callback(ec, run)</span>
 138 <span class="sd">            </span>
 139 <span class="sd">        :type compute_metric_callback: function </span>
 140
 141 <span class="sd">        :param evaluate_convergence_callback: User defined function invoked after</span>
 142 <span class="sd">            computing the metric on each run, to evaluate the experiment was</span>
 143 <span class="sd">            run enough times. It takes the list of cumulated metrics produced by </span>
 144 <span class="sd">            the compute_metric_callback up to the current run, and decided </span>
 145 <span class="sd">            whether the metrics have statistically converged to a meaningful value</span>
 146 <span class="sd">            or not. It must return either True or False. </span>
 147
 148 <span class="sd">                stop = evaluate_convergence_callback(ec, run, metrics)</span>
 149
 150 <span class="sd">            If stop is True, then the runner will exit.</span>
 151 <span class="sd">            </span>
 152 <span class="sd">        :type evaluate_convergence_callback: function </span>
 153
 154 <span class="sd">        &quot;&quot;&quot;</span>
 155
 156         <span class="k">if</span> <span class="p">(</span><span class="ow">not</span> <span class="n">max_runs</span> <span class="ow">or</span> <span class="n">max_runs</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">)</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">compute_metric_callback</span><span class="p">:</span>
 157             <span class="n">msg</span> <span class="o">=</span> <span class="s">&quot;Undefined STOP condition, set stop_callback or max_runs&quot;</span>
 158             <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">,</span> <span class="n">msg</span>
 159
 160         <span class="k">if</span> <span class="n">compute_metric_callback</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">evaluate_convergence_callback</span><span class="p">:</span>
 161             <span class="n">evaluate_convergence_callback</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">evaluate_normal_convergence</span>
 162             <span class="n">ec</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s">&quot; Treating data as normal to evaluate convergence. &quot;</span>
 163                     <span class="s">&quot;Experiment will stop when the standard error with 95% &quot;</span>
 164                     <span class="s">&quot;confidence interval is &gt;= 5</span><span class="si">% o</span><span class="s">f the mean of the collected samples &quot;</span><span class="p">)</span>
 165
 166         <span class="c"># Force persistence of experiment controller</span>
 167         <span class="n">ec</span><span class="o">.</span><span class="n">_persist</span> <span class="o">=</span> <span class="bp">True</span>
 168
 169         <span class="n">filepath</span> <span class="o">=</span> <span class="n">ec</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">dirpath</span> <span class="o">=</span> <span class="n">ec</span><span class="o">.</span><span class="n">exp_dir</span><span class="p">)</span>
 170
 171         <span class="n">samples</span> <span class="o">=</span> <span class="p">[]</span>
 172         <span class="n">run</span> <span class="o">=</span> <span class="mi">0</span>
 173         <span class="n">stop</span> <span class="o">=</span> <span class="bp">False</span>
 174
 175         <span class="k">while</span> <span class="ow">not</span> <span class="n">stop</span><span class="p">:</span>
 176             <span class="n">run</span> <span class="o">+=</span> <span class="mi">1</span>
 177
 178             <span class="n">ec</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_experiment</span><span class="p">(</span><span class="n">filepath</span><span class="p">,</span> <span class="n">wait_time</span><span class="p">,</span> <span class="n">wait_guids</span><span class="p">)</span>
 179
 180             <span class="n">ec</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s">&quot; RUN </span><span class="si">%d</span><span class="s"> </span><span class="se">\n</span><span class="s">&quot;</span> <span class="o">%</span> <span class="n">run</span><span class="p">)</span>
 181
 182             <span class="k">if</span> <span class="n">compute_metric_callback</span><span class="p">:</span>
 183                 <span class="n">metric</span> <span class="o">=</span> <span class="n">compute_metric_callback</span><span class="p">(</span><span class="n">ec</span><span class="p">,</span> <span class="n">run</span><span class="p">)</span>
 184                 <span class="k">if</span> <span class="n">metric</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
 185                     <span class="n">samples</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">metric</span><span class="p">)</span>
 186
 187                     <span class="k">if</span> <span class="n">run</span> <span class="o">&gt;=</span> <span class="n">min_runs</span> <span class="ow">and</span> <span class="n">evaluate_convergence_callback</span><span class="p">:</span>
 188                         <span class="k">if</span> <span class="n">evaluate_convergence_callback</span><span class="p">(</span><span class="n">ec</span><span class="p">,</span> <span class="n">run</span><span class="p">,</span> <span class="n">samples</span><span class="p">):</span>
 189                             <span class="n">stop</span> <span class="o">=</span> <span class="bp">True</span>
 190
 191             <span class="k">if</span> <span class="n">run</span> <span class="o">&gt;=</span> <span class="n">min_runs</span> <span class="ow">and</span> <span class="n">max_runs</span> <span class="o">&gt;</span> <span class="o">-</span><span class="mi">1</span> <span class="ow">and</span> <span class="n">run</span> <span class="o">&gt;=</span> <span class="n">max_runs</span> <span class="p">:</span>
 192                 <span class="n">stop</span> <span class="o">=</span> <span class="bp">True</span>
 193
 194             <span class="n">ec</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span>
 195             <span class="k">del</span> <span class="n">ec</span>
 196
 197         <span class="k">return</span> <span class="n">run</span>
 198 </div>
 199 <div class="viewcode-block" id="ExperimentRunner.evaluate_normal_convergence"><a class="viewcode-back" href="../../../_layout/nepi.execution.html#nepi.execution.runner.ExperimentRunner.evaluate_normal_convergence">[docs]</a>    <span class="k">def</span> <span class="nf">evaluate_normal_convergence</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ec</span><span class="p">,</span> <span class="n">run</span><span class="p">,</span> <span class="n">metrics</span><span class="p">):</span>
 200         <span class="sd">&quot;&quot;&quot; Returns True when the confidence interval of the sample mean is</span>
 201 <span class="sd">        less than 5% of the mean value, for a 95% confidence level,</span>
 202 <span class="sd">        assuming normal distribution of the data</span>
 203 <span class="sd">        &quot;&quot;&quot;</span>
 204
 205         <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">metrics</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
 206             <span class="n">msg</span> <span class="o">=</span> <span class="s">&quot;0 samples collected&quot;</span>
 207             <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">,</span> <span class="n">msg</span>
 208
 209         <span class="n">x</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">metrics</span><span class="p">)</span>
 210         <span class="n">n</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">metrics</span><span class="p">)</span>
 211         <span class="n">std</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">std</span><span class="p">()</span>
 212         <span class="n">se</span> <span class="o">=</span> <span class="n">std</span> <span class="o">/</span> <span class="n">math</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">n</span><span class="p">)</span>
 213         <span class="n">m</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
 214
 215         <span class="c"># Confidence interval for 95% confidence level, </span>
 216         <span class="c"># assuming normally distributed data.</span>
 217         <span class="n">ci95</span> <span class="o">=</span> <span class="n">se</span> <span class="o">*</span> <span class="mi">2</span>
 218
 219         <span class="n">ec</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s">&quot; RUN </span><span class="si">%d</span><span class="s"> - SAMPLES </span><span class="si">%d</span><span class="s"> MEAN </span><span class="si">%.2f</span><span class="s"> STD </span><span class="si">%.2f</span><span class="s"> CI (95</span><span class="si">%%</span><span class="s">) </span><span class="si">%.2f</span><span class="s"> </span><span class="se">\n</span><span class="s">&quot;</span> <span class="o">%</span> <span class="p">(</span>
 220             <span class="n">run</span><span class="p">,</span> <span class="n">n</span><span class="p">,</span> <span class="n">m</span><span class="p">,</span> <span class="n">std</span><span class="p">,</span> <span class="n">ci95</span> <span class="p">)</span> <span class="p">)</span>
 221
 222         <span class="k">return</span> <span class="n">m</span> <span class="o">*</span> <span class="mf">0.05</span> <span class="o">&gt;=</span> <span class="n">ci95</span>
 223 </div>
 224 <div class="viewcode-block" id="ExperimentRunner.run_experiment"><a class="viewcode-back" href="../../../_layout/nepi.execution.html#nepi.execution.runner.ExperimentRunner.run_experiment">[docs]</a>    <span class="k">def</span> <span class="nf">run_experiment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filepath</span><span class="p">,</span> <span class="n">wait_time</span><span class="p">,</span> <span class="n">wait_guids</span><span class="p">):</span>
 225         <span class="sd">&quot;&quot;&quot; Run an experiment based on the description stored</span>
 226 <span class="sd">        in filepath.</span>
 227
 228 <span class="sd">        &quot;&quot;&quot;</span>
 229         <span class="n">ec</span> <span class="o">=</span> <span class="n">ExperimentController</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">filepath</span><span class="p">)</span>
 230
 231         <span class="n">ec</span><span class="o">.</span><span class="n">deploy</span><span class="p">()</span>
 232
 233         <span class="n">ec</span><span class="o">.</span><span class="n">wait_finished</span><span class="p">(</span><span class="n">wait_guids</span><span class="p">)</span>
 234         <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">wait_time</span><span class="p">)</span>
 235
 236         <span class="n">ec</span><span class="o">.</span><span class="n">release</span><span class="p">()</span>
 237
 238         <span class="k">if</span> <span class="n">ec</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">ECState</span><span class="o">.</span><span class="n">FAILED</span><span class="p">:</span>
 239             <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">,</span> <span class="s">&quot;Experiment failed&quot;</span>
 240
 241         <span class="k">return</span> <span class="n">ec</span>
 242 </pre></div></div></div>
 243
 244           </div>
 245         </div>
 246       </div>
 247       <div class="clearer"></div>
 248     </div>
 249     <div class="related">
 250       <h3>Navigation</h3>
 251       <ul>
 252         <li class="right" style="margin-right: 10px">
 253           <a href="../../../genindex.html" title="General Index"
 254              >index</a></li>
 255         <li class="right" >
 256           <a href="../../../py-modindex.html" title="Python Module Index"
 257              >modules</a> |</li>
 258         <li><a href="../../../index.html">NEPI 3.2 documentation</a> &raquo;</li>
 259           <li><a href="../../index.html" >Module code</a> &raquo;</li>
 260       </ul>
 261     </div>
 262     <div class="footer">
 263         &copy; Copyright 2014, Alina Quereilhac, Lucia Guevgeozian Odizzio, Julien Tribino.
 264       Created using <a href="http://sphinx-doc.org/">Sphinx</a> 1.2.3.
 265     </div>
 266   </body>
 267 </html>