doc/sphinx/_build/html/_modules/nepi/util/parallel.html

   1 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
   2   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
   3
   4
   5 <html xmlns="http://www.w3.org/1999/xhtml">
   6   <head>
   7     <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
   8
   9     <title>nepi.util.parallel &mdash; NEPI 3.2 documentation</title>
  10
  11     <link rel="stylesheet" href="../../../_static/sphinxdoc.css" type="text/css" />
  12     <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
  13
  14     <script type="text/javascript">
  15       var DOCUMENTATION_OPTIONS = {
  16         URL_ROOT:    '../../../',
  17         VERSION:     '3.2',
  18         COLLAPSE_INDEX: false,
  19         FILE_SUFFIX: '.html',
  20         HAS_SOURCE:  true
  21       };
  22     </script>
  23     <script type="text/javascript" src="../../../_static/jquery.js"></script>
  24     <script type="text/javascript" src="../../../_static/underscore.js"></script>
  25     <script type="text/javascript" src="../../../_static/doctools.js"></script>
  26     <link rel="top" title="NEPI 3.2 documentation" href="../../../index.html" />
  27     <link rel="up" title="Module code" href="../../index.html" />
  28   </head>
  29   <body>
  30     <div class="related">
  31       <h3>Navigation</h3>
  32       <ul>
  33         <li class="right" style="margin-right: 10px">
  34           <a href="../../../genindex.html" title="General Index"
  35              accesskey="I">index</a></li>
  36         <li class="right" >
  37           <a href="../../../py-modindex.html" title="Python Module Index"
  38              >modules</a> |</li>
  39         <li><a href="../../../index.html">NEPI 3.2 documentation</a> &raquo;</li>
  40           <li><a href="../../index.html" accesskey="U">Module code</a> &raquo;</li>
  41       </ul>
  42     </div>
  43       <div class="sphinxsidebar">
  44         <div class="sphinxsidebarwrapper">
  45 <div id="searchbox" style="display: none">
  46   <h3>Quick search</h3>
  47     <form class="search" action="../../../search.html" method="get">
  48       <input type="text" name="q" />
  49       <input type="submit" value="Go" />
  50       <input type="hidden" name="check_keywords" value="yes" />
  51       <input type="hidden" name="area" value="default" />
  52     </form>
  53     <p class="searchtip" style="font-size: 90%">
  54     Enter search terms or a module, class or function name.
  55     </p>
  56 </div>
  57 <script type="text/javascript">$('#searchbox').show(0);</script>
  58         </div>
  59       </div>
  60
  61     <div class="document">
  62       <div class="documentwrapper">
  63         <div class="bodywrapper">
  64           <div class="body">
  65
  66   <h1>Source code for nepi.util.parallel</h1><div class="highlight"><pre>
  67 <span class="c">#</span>
  68 <span class="c">#    NEPI, a framework to manage network experiments</span>
  69 <span class="c">#    Copyright (C) 2013 INRIA</span>
  70 <span class="c">#</span>
  71 <span class="c">#    This program is free software: you can redistribute it and/or modify</span>
  72 <span class="c">#    it under the terms of the GNU General Public License version 2 as</span>
  73 <span class="c">#    published by the Free Software Foundation;</span>
  74 <span class="c">#</span>
  75 <span class="c">#    This program is distributed in the hope that it will be useful,</span>
  76 <span class="c">#    but WITHOUT ANY WARRANTY; without even the implied warranty of</span>
  77 <span class="c">#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the</span>
  78 <span class="c">#    GNU General Public License for more details.</span>
  79 <span class="c">#</span>
  80 <span class="c">#    You should have received a copy of the GNU General Public License</span>
  81 <span class="c">#    along with this program.  If not, see &lt;http://www.gnu.org/licenses/&gt;.</span>
  82 <span class="c">#</span>
  83 <span class="c"># Author: Claudio Freire &lt;claudio-daniel.freire@inria.fr&gt;</span>
  84 <span class="c">#         Alina Quereilhac &lt;alina.quereilhac@inria.fr&gt;</span>
  85 <span class="c">#</span>
  86
  87 <span class="kn">import</span> <span class="nn">threading</span>
  88 <span class="kn">import</span> <span class="nn">Queue</span>
  89 <span class="kn">import</span> <span class="nn">traceback</span>
  90 <span class="kn">import</span> <span class="nn">sys</span>
  91 <span class="kn">import</span> <span class="nn">os</span>
  92
  93 <span class="n">N_PROCS</span> <span class="o">=</span> <span class="bp">None</span>
  94
  95 <div class="viewcode-block" id="WorkerThread"><a class="viewcode-back" href="../../../_layout/nepi.util.html#nepi.util.parallel.WorkerThread">[docs]</a><span class="k">class</span> <span class="nc">WorkerThread</span><span class="p">(</span><span class="n">threading</span><span class="o">.</span><span class="n">Thread</span><span class="p">):</span>
  96 <div class="viewcode-block" id="WorkerThread.QUIT"><a class="viewcode-back" href="../../../_layout/nepi.util.html#nepi.util.parallel.WorkerThread.QUIT">[docs]</a>    <span class="k">class</span> <span class="nc">QUIT</span><span class="p">:</span>
  97         <span class="k">pass</span>
  98 </div>
  99 <div class="viewcode-block" id="WorkerThread.run"><a class="viewcode-back" href="../../../_layout/nepi.util.html#nepi.util.parallel.WorkerThread.run">[docs]</a>    <span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
 100         <span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
 101             <span class="n">task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
 102
 103             <span class="k">if</span> <span class="n">task</span> <span class="ow">is</span> <span class="bp">self</span><span class="o">.</span><span class="n">QUIT</span><span class="p">:</span>
 104                 <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
 105                 <span class="k">break</span>
 106
 107             <span class="k">try</span><span class="p">:</span>
 108                 <span class="k">try</span><span class="p">:</span>
 109                     <span class="nb">callable</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="n">kwargs</span> <span class="o">=</span> <span class="n">task</span>
 110                     <span class="n">rv</span> <span class="o">=</span> <span class="nb">callable</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
 111
 112                     <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">rvqueue</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
 113                         <span class="bp">self</span><span class="o">.</span><span class="n">rvqueue</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">rv</span><span class="p">)</span>
 114                 <span class="k">finally</span><span class="p">:</span>
 115                     <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
 116             <span class="k">except</span><span class="p">:</span>
 117                 <span class="n">traceback</span><span class="o">.</span><span class="n">print_exc</span><span class="p">(</span><span class="nb">file</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">stderr</span><span class="p">)</span>
 118                 <span class="bp">self</span><span class="o">.</span><span class="n">delayed_exceptions</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">sys</span><span class="o">.</span><span class="n">exc_info</span><span class="p">())</span>
 119 </div>
 120 <div class="viewcode-block" id="WorkerThread.attach"><a class="viewcode-back" href="../../../_layout/nepi.util.html#nepi.util.parallel.WorkerThread.attach">[docs]</a>    <span class="k">def</span> <span class="nf">attach</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">queue</span><span class="p">,</span> <span class="n">rvqueue</span><span class="p">,</span> <span class="n">delayed_exceptions</span><span class="p">):</span>
 121         <span class="bp">self</span><span class="o">.</span><span class="n">queue</span> <span class="o">=</span> <span class="n">queue</span>
 122         <span class="bp">self</span><span class="o">.</span><span class="n">rvqueue</span> <span class="o">=</span> <span class="n">rvqueue</span>
 123         <span class="bp">self</span><span class="o">.</span><span class="n">delayed_exceptions</span> <span class="o">=</span> <span class="n">delayed_exceptions</span>
 124    </div>
 125 <div class="viewcode-block" id="WorkerThread.quit"><a class="viewcode-back" href="../../../_layout/nepi.util.html#nepi.util.parallel.WorkerThread.quit">[docs]</a>    <span class="k">def</span> <span class="nf">quit</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
 126         <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">QUIT</span><span class="p">)</span>
 127 </div></div>
 128 <div class="viewcode-block" id="ParallelRun"><a class="viewcode-back" href="../../../_layout/nepi.util.html#nepi.util.parallel.ParallelRun">[docs]</a><span class="k">class</span> <span class="nc">ParallelRun</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
 129     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">maxthreads</span> <span class="o">=</span> <span class="bp">None</span><span class="p">,</span> <span class="n">maxqueue</span> <span class="o">=</span> <span class="bp">None</span><span class="p">,</span> <span class="n">results</span> <span class="o">=</span> <span class="bp">True</span><span class="p">):</span>
 130         <span class="bp">self</span><span class="o">.</span><span class="n">maxqueue</span> <span class="o">=</span> <span class="n">maxqueue</span>
 131         <span class="bp">self</span><span class="o">.</span><span class="n">maxthreads</span> <span class="o">=</span> <span class="n">maxthreads</span>
 132
 133         <span class="bp">self</span><span class="o">.</span><span class="n">queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="o">.</span><span class="n">Queue</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">maxqueue</span> <span class="ow">or</span> <span class="mi">0</span><span class="p">)</span>
 134
 135         <span class="bp">self</span><span class="o">.</span><span class="n">delayed_exceptions</span> <span class="o">=</span> <span class="p">[]</span>
 136
 137         <span class="k">if</span> <span class="n">results</span><span class="p">:</span>
 138             <span class="bp">self</span><span class="o">.</span><span class="n">rvqueue</span> <span class="o">=</span> <span class="n">Queue</span><span class="o">.</span><span class="n">Queue</span><span class="p">()</span>
 139         <span class="k">else</span><span class="p">:</span>
 140             <span class="bp">self</span><span class="o">.</span><span class="n">rvqueue</span> <span class="o">=</span> <span class="bp">None</span>
 141
 142         <span class="bp">self</span><span class="o">.</span><span class="n">initialize_workers</span><span class="p">()</span>
 143
 144 <div class="viewcode-block" id="ParallelRun.initialize_workers"><a class="viewcode-back" href="../../../_layout/nepi.util.html#nepi.util.parallel.ParallelRun.initialize_workers">[docs]</a>    <span class="k">def</span> <span class="nf">initialize_workers</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
 145         <span class="k">global</span> <span class="n">N_PROCS</span>
 146
 147         <span class="n">maxthreads</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">maxthreads</span>
 148
 149         <span class="c"># Compute maximum number of threads allowed by the system</span>
 150         <span class="k">if</span> <span class="n">maxthreads</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
 151             <span class="k">if</span> <span class="n">N_PROCS</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
 152                 <span class="k">try</span><span class="p">:</span>
 153                     <span class="n">f</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="s">&quot;/proc/cpuinfo&quot;</span><span class="p">)</span>
 154                     <span class="k">try</span><span class="p">:</span>
 155                         <span class="n">N_PROCS</span> <span class="o">=</span> <span class="nb">sum</span><span class="p">(</span><span class="s">&quot;processor&quot;</span> <span class="ow">in</span> <span class="n">l</span> <span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">f</span><span class="p">)</span>
 156                     <span class="k">finally</span><span class="p">:</span>
 157                         <span class="n">f</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
 158                 <span class="k">except</span><span class="p">:</span>
 159                     <span class="k">pass</span>
 160             <span class="n">maxthreads</span> <span class="o">=</span> <span class="n">N_PROCS</span>
 161
 162         <span class="k">if</span> <span class="n">maxthreads</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
 163             <span class="n">maxthreads</span> <span class="o">=</span> <span class="mi">4</span>
 164
 165         <span class="bp">self</span><span class="o">.</span><span class="n">workers</span> <span class="o">=</span> <span class="p">[]</span>
 166
 167         <span class="c"># initialize workers</span>
 168         <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">xrange</span><span class="p">(</span><span class="n">maxthreads</span><span class="p">):</span>
 169             <span class="n">worker</span> <span class="o">=</span> <span class="n">WorkerThread</span><span class="p">()</span>
 170             <span class="n">worker</span><span class="o">.</span><span class="n">attach</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">rvqueue</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">delayed_exceptions</span><span class="p">)</span>
 171             <span class="n">worker</span><span class="o">.</span><span class="n">setDaemon</span><span class="p">(</span><span class="bp">True</span><span class="p">)</span>
 172
 173             <span class="bp">self</span><span class="o">.</span><span class="n">workers</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">worker</span><span class="p">)</span>
 174     </div>
 175     <span class="k">def</span> <span class="nf">__del__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
 176         <span class="bp">self</span><span class="o">.</span><span class="n">destroy</span><span class="p">()</span>
 177
 178 <div class="viewcode-block" id="ParallelRun.empty"><a class="viewcode-back" href="../../../_layout/nepi.util.html#nepi.util.parallel.ParallelRun.empty">[docs]</a>    <span class="k">def</span> <span class="nf">empty</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
 179         <span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
 180             <span class="k">try</span><span class="p">:</span>
 181                 <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">block</span> <span class="o">=</span> <span class="bp">False</span><span class="p">)</span>
 182                 <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
 183             <span class="k">except</span> <span class="n">Queue</span><span class="o">.</span><span class="n">Empty</span><span class="p">:</span>
 184                 <span class="k">break</span>
 185   </div>
 186 <div class="viewcode-block" id="ParallelRun.destroy"><a class="viewcode-back" href="../../../_layout/nepi.util.html#nepi.util.parallel.ParallelRun.destroy">[docs]</a>    <span class="k">def</span> <span class="nf">destroy</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
 187         <span class="bp">self</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
 188
 189         <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">workers</span><span class="p">[:]</span>
 190         </div>
 191 <div class="viewcode-block" id="ParallelRun.put"><a class="viewcode-back" href="../../../_layout/nepi.util.html#nepi.util.parallel.ParallelRun.put">[docs]</a>    <span class="k">def</span> <span class="nf">put</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">callable</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
 192         <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="nb">callable</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">))</span>
 193     </div>
 194 <div class="viewcode-block" id="ParallelRun.put_nowait"><a class="viewcode-back" href="../../../_layout/nepi.util.html#nepi.util.parallel.ParallelRun.put_nowait">[docs]</a>    <span class="k">def</span> <span class="nf">put_nowait</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">callable</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
 195         <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">put_nowait</span><span class="p">((</span><span class="nb">callable</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">))</span>
 196 </div>
 197 <div class="viewcode-block" id="ParallelRun.start"><a class="viewcode-back" href="../../../_layout/nepi.util.html#nepi.util.parallel.ParallelRun.start">[docs]</a>    <span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
 198         <span class="k">for</span> <span class="n">worker</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">workers</span><span class="p">:</span>
 199             <span class="k">if</span> <span class="ow">not</span> <span class="n">worker</span><span class="o">.</span><span class="n">isAlive</span><span class="p">():</span>
 200                 <span class="n">worker</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
 201     </div>
 202 <div class="viewcode-block" id="ParallelRun.join"><a class="viewcode-back" href="../../../_layout/nepi.util.html#nepi.util.parallel.ParallelRun.join">[docs]</a>    <span class="k">def</span> <span class="nf">join</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
 203         <span class="c"># Wait until all queued tasks have been processed</span>
 204         <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
 205
 206         <span class="k">for</span> <span class="n">worker</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">workers</span><span class="p">:</span>
 207             <span class="n">worker</span><span class="o">.</span><span class="n">quit</span><span class="p">()</span>
 208
 209         <span class="k">for</span> <span class="n">worker</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">workers</span><span class="p">:</span>
 210             <span class="n">worker</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
 211     </div>
 212 <div class="viewcode-block" id="ParallelRun.sync"><a class="viewcode-back" href="../../../_layout/nepi.util.html#nepi.util.parallel.ParallelRun.sync">[docs]</a>    <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
 213         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">delayed_exceptions</span><span class="p">:</span>
 214             <span class="n">typ</span><span class="p">,</span><span class="n">val</span><span class="p">,</span><span class="n">loc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">delayed_exceptions</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
 215             <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">delayed_exceptions</span><span class="p">[:]</span>
 216             <span class="k">raise</span> <span class="n">typ</span><span class="p">,</span><span class="n">val</span><span class="p">,</span><span class="n">loc</span>
 217         </div>
 218     <span class="k">def</span> <span class="nf">__iter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
 219         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">rvqueue</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
 220             <span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
 221                 <span class="k">try</span><span class="p">:</span>
 222                     <span class="k">yield</span> <span class="bp">self</span><span class="o">.</span><span class="n">rvqueue</span><span class="o">.</span><span class="n">get_nowait</span><span class="p">()</span>
 223                 <span class="k">except</span> <span class="n">Queue</span><span class="o">.</span><span class="n">Empty</span><span class="p">:</span>
 224                     <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
 225                     <span class="k">try</span><span class="p">:</span>
 226                         <span class="k">yield</span> <span class="bp">self</span><span class="o">.</span><span class="n">rvqueue</span><span class="o">.</span><span class="n">get_nowait</span><span class="p">()</span>
 227                     <span class="k">except</span> <span class="n">Queue</span><span class="o">.</span><span class="n">Empty</span><span class="p">:</span>
 228                         <span class="k">raise</span> <span class="ne">StopIteration</span>
 229             </div>
 230 </pre></div>
 231
 232           </div>
 233         </div>
 234       </div>
 235       <div class="clearer"></div>
 236     </div>
 237     <div class="related">
 238       <h3>Navigation</h3>
 239       <ul>
 240         <li class="right" style="margin-right: 10px">
 241           <a href="../../../genindex.html" title="General Index"
 242              >index</a></li>
 243         <li class="right" >
 244           <a href="../../../py-modindex.html" title="Python Module Index"
 245              >modules</a> |</li>
 246         <li><a href="../../../index.html">NEPI 3.2 documentation</a> &raquo;</li>
 247           <li><a href="../../index.html" >Module code</a> &raquo;</li>
 248       </ul>
 249     </div>
 250     <div class="footer">
 251         &copy; Copyright 2014, Alina Quereilhac, Lucia Guevgeozian Odizzio, Julien Tribino.
 252       Created using <a href="http://sphinx-doc.org/">Sphinx</a> 1.2.3.
 253     </div>
 254   </body>
 255 </html>