--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+
+<html xmlns="http://www.w3.org/1999/xhtml">
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+
+ <title>nepi.data.processing.ccn.parser — NEPI 3.2 documentation</title>
+
+ <link rel="stylesheet" href="../../../../../_static/sphinxdoc.css" type="text/css" />
+ <link rel="stylesheet" href="../../../../../_static/pygments.css" type="text/css" />
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT: '../../../../../',
+ VERSION: '3.2',
+ COLLAPSE_INDEX: false,
+ FILE_SUFFIX: '.html',
+ HAS_SOURCE: true
+ };
+ </script>
+ <script type="text/javascript" src="../../../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../../../_static/doctools.js"></script>
+ <link rel="top" title="NEPI 3.2 documentation" href="../../../../../index.html" />
+ <link rel="up" title="Module code" href="../../../../index.html" />
+ </head>
+ <body>
+ <div class="related">
+ <h3>Navigation</h3>
+ <ul>
+ <li class="right" style="margin-right: 10px">
+ <a href="../../../../../genindex.html" title="General Index"
+ accesskey="I">index</a></li>
+ <li class="right" >
+ <a href="../../../../../py-modindex.html" title="Python Module Index"
+ >modules</a> |</li>
+ <li><a href="../../../../../index.html">NEPI 3.2 documentation</a> »</li>
+ <li><a href="../../../../index.html" accesskey="U">Module code</a> »</li>
+ </ul>
+ </div>
+ <div class="sphinxsidebar">
+ <div class="sphinxsidebarwrapper">
+<div id="searchbox" style="display: none">
+ <h3>Quick search</h3>
+ <form class="search" action="../../../../../search.html" method="get">
+ <input type="text" name="q" />
+ <input type="submit" value="Go" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+ <p class="searchtip" style="font-size: 90%">
+ Enter search terms or a module, class or function name.
+ </p>
+</div>
+<script type="text/javascript">$('#searchbox').show(0);</script>
+ </div>
+ </div>
+
+ <div class="document">
+ <div class="documentwrapper">
+ <div class="bodywrapper">
+ <div class="body">
+
+ <h1>Source code for nepi.data.processing.ccn.parser</h1><div class="highlight"><pre>
+<span class="c">#!/usr/bin/env python</span>
+
+<span class="c">###############################################################################</span>
+<span class="c">#</span>
+<span class="c"># CCNX benchmark</span>
+<span class="c"># Copyright (C) 2014 INRIA</span>
+<span class="c">#</span>
+<span class="c"># This program is free software: you can redistribute it and/or modify</span>
+<span class="c"># it under the terms of the GNU General Public License version 2 as</span>
+<span class="c"># published by the Free Software Foundation;</span>
+<span class="c">#</span>
+<span class="c"># This program is distributed in the hope that it will be useful,</span>
+<span class="c"># but WITHOUT ANY WARRANTY; without even the implied warranty of</span>
+<span class="c"># MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the</span>
+<span class="c"># GNU General Public License for more details.</span>
+<span class="c">#</span>
+<span class="c"># You should have received a copy of the GNU General Public License</span>
+<span class="c"># along with this program. If not, see <http://www.gnu.org/licenses/>.</span>
+<span class="c">#</span>
+<span class="c">#</span>
+<span class="c"># Author: Alina Quereilhac <alina.quereilhac@inria.fr></span>
+<span class="c">#</span>
+<span class="c">###############################################################################</span>
+
+<span class="c">#</span>
+<span class="c"># This library contains functions to parse (CCNx) ccnd logs.</span>
+<span class="c">#</span>
+<span class="c"># Results from experiments must be stored in a directory</span>
+<span class="c"># named with the experiment run id.</span>
+<span class="c"># ccnd logs are stored in .log files in a subdirectory per node.</span>
+<span class="c"># The following diagram exemplifies the experiment result directory</span>
+<span class="c"># structure (nidi is the unique identifier assigned to node i):</span>
+<span class="c">#</span>
+<span class="c"># run_id</span>
+<span class="c"># \ nid1</span>
+<span class="c"># \ nid2.log</span>
+<span class="c"># \ nid2</span>
+<span class="c"># \ nid1.log</span>
+<span class="c"># \ nid3</span>
+<span class="c"># \ nid3.log</span>
+<span class="c">#</span>
+
+<span class="kn">import</span> <span class="nn">collections</span>
+<span class="kn">import</span> <span class="nn">functools</span>
+<span class="kn">import</span> <span class="nn">networkx</span>
+<span class="kn">import</span> <span class="nn">os</span>
+<span class="kn">import</span> <span class="nn">pickle</span>
+<span class="kn">import</span> <span class="nn">tempfile</span>
+
+<span class="kn">from</span> <span class="nn">nepi.util.timefuncs</span> <span class="kn">import</span> <span class="n">compute_delay_ms</span>
+<span class="kn">from</span> <span class="nn">nepi.util.statfuncs</span> <span class="kn">import</span> <span class="n">compute_mean</span>
+<span class="kn">import</span> <span class="nn">nepi.data.processing.ping.parser</span> <span class="kn">as</span> <span class="nn">ping_parser</span>
+
+<div class="viewcode-block" id="is_control"><a class="viewcode-back" href="../../../../../_layout/nepi.data.processing.ccn.html#nepi.data.processing.ccn.parser.is_control">[docs]</a><span class="k">def</span> <span class="nf">is_control</span><span class="p">(</span><span class="n">content_name</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">content_name</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s">"ccnx:/%C1"</span><span class="p">)</span> <span class="ow">or</span> \
+ <span class="n">content_name</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s">"ccnx:/ccnx"</span><span class="p">)</span> <span class="ow">or</span> \
+ <span class="n">content_name</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s">"ccnx:/..."</span><span class="p">)</span>
+
+</div>
+<div class="viewcode-block" id="parse_file"><a class="viewcode-back" href="../../../../../_layout/nepi.data.processing.ccn.html#nepi.data.processing.ccn.parser.parse_file">[docs]</a><span class="k">def</span> <span class="nf">parse_file</span><span class="p">(</span><span class="n">filename</span><span class="p">):</span>
+ <span class="sd">""" Parses message information from ccnd log files</span>
+
+<span class="sd"> filename: path to ccndlog file</span>
+
+<span class="sd"> """</span>
+
+ <span class="n">faces</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
+ <span class="n">sep</span> <span class="o">=</span> <span class="s">" "</span>
+
+ <span class="n">f</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s">"r"</span><span class="p">)</span>
+
+ <span class="n">data</span> <span class="o">=</span> <span class="p">[]</span>
+
+ <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">f</span><span class="p">:</span>
+ <span class="n">cols</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">sep</span><span class="p">)</span>
+
+ <span class="c"># CCN_PEEK</span>
+ <span class="c"># MESSAGE interest_from</span>
+ <span class="c"># 1374181938.808523 ccnd[9245]: debug.4352 interest_from 6 ccnx:/test/bunny.ts (23 bytes,sim=0CDCC1D7)</span>
+ <span class="c">#</span>
+ <span class="c"># MESSAGE interest_to</span>
+ <span class="c"># 1374181938.812750 ccnd[9245]: debug.3502 interest_to 5 ccnx:/test/bunny.ts (39 bytes,i=2844,sim=0CDCC1D7)</span>
+ <span class="c">#</span>
+ <span class="c"># MESSAGE CONTENT FROM</span>
+ <span class="c"># 1374181938.868682 ccnd[9245]: debug.4643 content_from 5 ccnx:/test/bunny.ts/%FD%05%1E%85%8FVw/%00/%9E%3D%01%D9%3Cn%95%2BvZ%8</span>
+ <span class="c">#</span>
+ <span class="c"># MESSAGE CONTENT_TO</span>
+ <span class="c"># 1374181938.868772 ccnd[9245]: debug.1619 content_to 6 ccnx:/test/bunny.ts/%FD%05%1E%85%8FVw/%00/%9E%3D%01%D9%3Cn%95%2BvZ%8</span>
+ <span class="c">#</span>
+ <span class="c"># 1375596708.222304 ccnd[9758]: debug.3692 interest_expiry ccnx:/test/bunny.ts/%FD%05%1E%86%B1GS/%00%0A%F7 (44 bytes,c=0:1,i=2819,sim=49FA8048)</span>
+
+ <span class="c"># External face creation</span>
+ <span class="c"># 1374181452.965961 ccnd[9245]: accepted datagram client id=5 (flags=0x40012) 204.85.191.10 port 9695</span>
+
+ <span class="k">if</span> <span class="n">line</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s">"accepted datagram client"</span><span class="p">)</span> <span class="o">></span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
+ <span class="n">face_id</span> <span class="o">=</span> <span class="p">(</span><span class="n">cols</span><span class="p">[</span><span class="mi">5</span><span class="p">])</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s">"id="</span><span class="p">,</span><span class="s">''</span><span class="p">)</span>
+ <span class="n">ip</span> <span class="o">=</span> <span class="n">cols</span><span class="p">[</span><span class="mi">7</span><span class="p">]</span>
+ <span class="n">port</span> <span class="o">=</span> <span class="n">cols</span><span class="p">[</span><span class="mi">9</span><span class="p">]</span>
+ <span class="n">faces</span><span class="p">[</span><span class="n">face_id</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">ip</span><span class="p">,</span> <span class="n">port</span><span class="p">)</span>
+ <span class="k">continue</span>
+
+ <span class="c"># 1374181452.985296 ccnd[9245]: releasing face id 4 (slot 4)</span>
+ <span class="k">if</span> <span class="n">line</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s">"releasing face id"</span><span class="p">)</span> <span class="o">></span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
+ <span class="n">face_id</span> <span class="o">=</span> <span class="n">cols</span><span class="p">[</span><span class="mi">5</span><span class="p">]</span>
+ <span class="k">if</span> <span class="n">face_id</span> <span class="ow">in</span> <span class="n">faces</span><span class="p">:</span>
+ <span class="k">del</span> <span class="n">faces</span><span class="p">[</span><span class="n">face_id</span><span class="p">]</span>
+ <span class="k">continue</span>
+
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">cols</span><span class="p">)</span> <span class="o"><</span> <span class="mi">6</span><span class="p">:</span>
+ <span class="k">continue</span>
+
+ <span class="n">timestamp</span> <span class="o">=</span> <span class="n">cols</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+ <span class="n">message_type</span> <span class="o">=</span> <span class="n">cols</span><span class="p">[</span><span class="mi">3</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="n">message_type</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s">"interest_from"</span><span class="p">,</span> <span class="s">"interest_to"</span><span class="p">,</span> <span class="s">"content_from"</span><span class="p">,</span>
+ <span class="s">"content_to"</span><span class="p">,</span> <span class="s">"interest_dupnonce"</span><span class="p">,</span> <span class="s">"interest_expiry"</span><span class="p">]:</span>
+ <span class="k">continue</span>
+
+ <span class="n">face_id</span> <span class="o">=</span> <span class="n">cols</span><span class="p">[</span><span class="mi">4</span><span class="p">]</span>
+ <span class="n">content_name</span> <span class="o">=</span> <span class="n">cols</span><span class="p">[</span><span class="mi">5</span><span class="p">]</span>
+
+ <span class="c"># Interest Nonce ? -> 412A74-0844-0008-50AA-F6EAD4</span>
+ <span class="n">nonce</span> <span class="o">=</span> <span class="s">""</span>
+ <span class="k">if</span> <span class="n">message_type</span> <span class="ow">in</span> <span class="p">[</span><span class="s">"interest_from"</span><span class="p">,</span> <span class="s">"interest_to"</span><span class="p">,</span> <span class="s">"interest_dupnonce"</span><span class="p">]:</span>
+ <span class="n">last</span> <span class="o">=</span> <span class="n">cols</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">last</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s">"-"</span><span class="p">))</span> <span class="o">==</span> <span class="mi">5</span><span class="p">:</span>
+ <span class="n">nonce</span> <span class="o">=</span> <span class="n">last</span>
+
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">size</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">cols</span><span class="p">[</span><span class="mi">6</span><span class="p">])</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s">'('</span><span class="p">,</span><span class="s">''</span><span class="p">))</span>
+ <span class="k">except</span><span class="p">:</span>
+ <span class="k">print</span> <span class="s">"interest_expiry without face id!"</span><span class="p">,</span> <span class="n">line</span>
+ <span class="k">continue</span>
+
+ <span class="c"># If no external IP address was identified for this face</span>
+ <span class="c"># asume it is a local face</span>
+ <span class="n">peer</span> <span class="o">=</span> <span class="s">"localhost"</span>
+
+ <span class="k">if</span> <span class="n">face_id</span> <span class="ow">in</span> <span class="n">faces</span><span class="p">:</span>
+ <span class="n">peer</span><span class="p">,</span> <span class="n">port</span> <span class="o">=</span> <span class="n">faces</span><span class="p">[</span><span class="n">face_id</span><span class="p">]</span>
+
+ <span class="n">data</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">content_name</span><span class="p">,</span> <span class="n">timestamp</span><span class="p">,</span> <span class="n">message_type</span><span class="p">,</span> <span class="n">peer</span><span class="p">,</span> <span class="n">face_id</span><span class="p">,</span>
+ <span class="n">size</span><span class="p">,</span> <span class="n">nonce</span><span class="p">,</span> <span class="n">line</span><span class="p">))</span>
+
+ <span class="n">f</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+
+ <span class="k">return</span> <span class="n">data</span>
+</div>
+<div class="viewcode-block" id="dump_content_history"><a class="viewcode-back" href="../../../../../_layout/nepi.data.processing.ccn.html#nepi.data.processing.ccn.parser.dump_content_history">[docs]</a><span class="k">def</span> <span class="nf">dump_content_history</span><span class="p">(</span><span class="n">content_history</span><span class="p">):</span>
+ <span class="n">f</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="n">delete</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
+ <span class="n">pickle</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">content_history</span><span class="p">,</span> <span class="n">f</span><span class="p">)</span>
+ <span class="n">f</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+ <span class="k">return</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span>
+</div>
+<div class="viewcode-block" id="load_content_history"><a class="viewcode-back" href="../../../../../_layout/nepi.data.processing.ccn.html#nepi.data.processing.ccn.parser.load_content_history">[docs]</a><span class="k">def</span> <span class="nf">load_content_history</span><span class="p">(</span><span class="n">fname</span><span class="p">):</span>
+ <span class="n">f</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">fname</span><span class="p">,</span> <span class="s">"r"</span><span class="p">)</span>
+ <span class="n">content_history</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
+ <span class="n">f</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+
+ <span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">fname</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">content_history</span>
+</div>
+<div class="viewcode-block" id="annotate_cn_node"><a class="viewcode-back" href="../../../../../_layout/nepi.data.processing.ccn.html#nepi.data.processing.ccn.parser.annotate_cn_node">[docs]</a><span class="k">def</span> <span class="nf">annotate_cn_node</span><span class="p">(</span><span class="n">graph</span><span class="p">,</span> <span class="n">nid</span><span class="p">,</span> <span class="n">ips2nid</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">content_history</span><span class="p">):</span>
+ <span class="k">for</span> <span class="p">(</span><span class="n">content_name</span><span class="p">,</span> <span class="n">timestamp</span><span class="p">,</span> <span class="n">message_type</span><span class="p">,</span> <span class="n">peer</span><span class="p">,</span> <span class="n">face_id</span><span class="p">,</span>
+ <span class="n">size</span><span class="p">,</span> <span class="n">nonce</span><span class="p">,</span> <span class="n">line</span><span class="p">)</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span>
+
+ <span class="c"># Ignore control messages for the time being</span>
+ <span class="k">if</span> <span class="n">is_control</span><span class="p">(</span><span class="n">content_name</span><span class="p">):</span>
+ <span class="k">continue</span>
+
+ <span class="k">if</span> <span class="n">message_type</span> <span class="o">==</span> <span class="s">"interest_from"</span> <span class="ow">and</span> \
+ <span class="n">peer</span> <span class="o">==</span> <span class="s">"localhost"</span><span class="p">:</span>
+ <span class="n">graph</span><span class="o">.</span><span class="n">node</span><span class="p">[</span><span class="n">nid</span><span class="p">][</span><span class="s">"ccn_consumer"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">True</span>
+ <span class="k">elif</span> <span class="n">message_type</span> <span class="o">==</span> <span class="s">"content_from"</span> <span class="ow">and</span> \
+ <span class="n">peer</span> <span class="o">==</span> <span class="s">"localhost"</span><span class="p">:</span>
+ <span class="n">graph</span><span class="o">.</span><span class="n">node</span><span class="p">[</span><span class="n">nid</span><span class="p">][</span><span class="s">"ccn_producer"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">True</span>
+
+ <span class="c"># Ignore local messages for the time being. </span>
+ <span class="c"># They could later be used to calculate the processing times</span>
+ <span class="c"># of messages.</span>
+ <span class="k">if</span> <span class="n">peer</span> <span class="o">==</span> <span class="s">"localhost"</span><span class="p">:</span>
+ <span class="k">continue</span>
+
+ <span class="c"># remove digest</span>
+ <span class="k">if</span> <span class="n">message_type</span> <span class="ow">in</span> <span class="p">[</span><span class="s">"content_from"</span><span class="p">,</span> <span class="s">"content_to"</span><span class="p">]:</span>
+ <span class="n">content_name</span> <span class="o">=</span> <span class="s">"/"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">content_name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s">"/"</span><span class="p">)[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
+
+ <span class="k">if</span> <span class="n">content_name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">content_history</span><span class="p">:</span>
+ <span class="n">content_history</span><span class="p">[</span><span class="n">content_name</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">()</span>
+
+ <span class="n">peernid</span> <span class="o">=</span> <span class="n">ips2nid</span><span class="p">[</span><span class="n">peer</span><span class="p">]</span>
+ <span class="n">graph</span><span class="o">.</span><span class="n">add_edge</span><span class="p">(</span><span class="n">nid</span><span class="p">,</span> <span class="n">peernid</span><span class="p">)</span>
+
+ <span class="n">content_history</span><span class="p">[</span><span class="n">content_name</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">timestamp</span><span class="p">,</span> <span class="n">message_type</span><span class="p">,</span> <span class="n">nid</span><span class="p">,</span>
+ <span class="n">peernid</span><span class="p">,</span> <span class="n">nonce</span><span class="p">,</span> <span class="n">size</span><span class="p">,</span> <span class="n">line</span><span class="p">))</span>
+</div>
+<div class="viewcode-block" id="annotate_cn_graph"><a class="viewcode-back" href="../../../../../_layout/nepi.data.processing.ccn.html#nepi.data.processing.ccn.parser.annotate_cn_graph">[docs]</a><span class="k">def</span> <span class="nf">annotate_cn_graph</span><span class="p">(</span><span class="n">logs_dir</span><span class="p">,</span> <span class="n">graph</span><span class="p">,</span> <span class="n">parse_ping_logs</span> <span class="o">=</span> <span class="bp">False</span><span class="p">):</span>
+ <span class="sd">""" Adds CCN content history for each node in the topology graph.</span>
+
+<span class="sd"> """</span>
+
+ <span class="c"># Make a copy of the graph to ensure integrity</span>
+ <span class="n">graph</span> <span class="o">=</span> <span class="n">graph</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+
+ <span class="n">ips2nid</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
+
+ <span class="k">for</span> <span class="n">nid</span> <span class="ow">in</span> <span class="n">graph</span><span class="o">.</span><span class="n">nodes</span><span class="p">():</span>
+ <span class="n">ips</span> <span class="o">=</span> <span class="n">graph</span><span class="o">.</span><span class="n">node</span><span class="p">[</span><span class="n">nid</span><span class="p">][</span><span class="s">"ips"</span><span class="p">]</span>
+ <span class="k">for</span> <span class="n">ip</span> <span class="ow">in</span> <span class="n">ips</span><span class="p">:</span>
+ <span class="n">ips2nid</span><span class="p">[</span><span class="n">ip</span><span class="p">]</span> <span class="o">=</span> <span class="n">nid</span>
+
+ <span class="n">found_files</span> <span class="o">=</span> <span class="bp">False</span>
+
+ <span class="c"># Now walk through the ccnd logs...</span>
+ <span class="k">for</span> <span class="n">dirpath</span><span class="p">,</span> <span class="n">dnames</span><span class="p">,</span> <span class="n">fnames</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">logs_dir</span><span class="p">):</span>
+ <span class="c"># continue if we are not at the leaf level (if there are subdirectories)</span>
+ <span class="k">if</span> <span class="n">dnames</span><span class="p">:</span>
+ <span class="k">continue</span>
+
+ <span class="c"># Each dirpath correspond to a different node</span>
+ <span class="n">nid</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">basename</span><span class="p">(</span><span class="n">dirpath</span><span class="p">)</span>
+
+ <span class="c"># Cast to numeric nid if necessary</span>
+ <span class="k">if</span> <span class="nb">int</span><span class="p">(</span><span class="n">nid</span><span class="p">)</span> <span class="ow">in</span> <span class="n">graph</span><span class="o">.</span><span class="n">nodes</span><span class="p">():</span>
+ <span class="n">nid</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">nid</span><span class="p">)</span>
+
+ <span class="n">content_history</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
+
+ <span class="k">for</span> <span class="n">fname</span> <span class="ow">in</span> <span class="n">fnames</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">fname</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s">".log"</span><span class="p">):</span>
+ <span class="n">found_files</span> <span class="o">=</span> <span class="bp">True</span>
+ <span class="n">filename</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">dirpath</span><span class="p">,</span> <span class="n">fname</span><span class="p">)</span>
+ <span class="n">data</span> <span class="o">=</span> <span class="n">parse_file</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span>
+ <span class="n">annotate_cn_node</span><span class="p">(</span><span class="n">graph</span><span class="p">,</span> <span class="n">nid</span><span class="p">,</span> <span class="n">ips2nid</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">content_history</span><span class="p">)</span>
+
+ <span class="c"># Avoid storing everything in memory, instead dump to a file</span>
+ <span class="c"># and reference the file</span>
+ <span class="n">fname</span> <span class="o">=</span> <span class="n">dump_content_history</span><span class="p">(</span><span class="n">content_history</span><span class="p">)</span>
+ <span class="n">graph</span><span class="o">.</span><span class="n">node</span><span class="p">[</span><span class="n">nid</span><span class="p">][</span><span class="s">"history"</span><span class="p">]</span> <span class="o">=</span> <span class="n">fname</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">found_files</span><span class="p">:</span>
+ <span class="n">msg</span> <span class="o">=</span> <span class="s">"No CCND output files were found to parse at </span><span class="si">%s</span><span class="s"> "</span> <span class="o">%</span> <span class="n">logs_dir</span>
+ <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">,</span> <span class="n">msg</span>
+
+ <span class="k">if</span> <span class="n">parse_ping_logs</span><span class="p">:</span>
+ <span class="n">ping_parser</span><span class="o">.</span><span class="n">annotate_cn_graph</span><span class="p">(</span><span class="n">logs_dir</span><span class="p">,</span> <span class="n">graph</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="n">graph</span>
+</div>
+<div class="viewcode-block" id="ccn_producers"><a class="viewcode-back" href="../../../../../_layout/nepi.data.processing.ccn.html#nepi.data.processing.ccn.parser.ccn_producers">[docs]</a><span class="k">def</span> <span class="nf">ccn_producers</span><span class="p">(</span><span class="n">graph</span><span class="p">):</span>
+ <span class="sd">""" Returns the nodes that are content providers """</span>
+ <span class="k">return</span> <span class="p">[</span><span class="n">nid</span> <span class="k">for</span> <span class="n">nid</span> <span class="ow">in</span> <span class="n">graph</span><span class="o">.</span><span class="n">nodes</span><span class="p">()</span> \
+ <span class="k">if</span> <span class="n">graph</span><span class="o">.</span><span class="n">node</span><span class="p">[</span><span class="n">nid</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s">"ccn_producer"</span><span class="p">)]</span>
+</div>
+<div class="viewcode-block" id="ccn_consumers"><a class="viewcode-back" href="../../../../../_layout/nepi.data.processing.ccn.html#nepi.data.processing.ccn.parser.ccn_consumers">[docs]</a><span class="k">def</span> <span class="nf">ccn_consumers</span><span class="p">(</span><span class="n">graph</span><span class="p">):</span>
+ <span class="sd">""" Returns the nodes that are content consumers """</span>
+ <span class="k">return</span> <span class="p">[</span><span class="n">nid</span> <span class="k">for</span> <span class="n">nid</span> <span class="ow">in</span> <span class="n">graph</span><span class="o">.</span><span class="n">nodes</span><span class="p">()</span> \
+ <span class="k">if</span> <span class="n">graph</span><span class="o">.</span><span class="n">node</span><span class="p">[</span><span class="n">nid</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s">"ccn_consumer"</span><span class="p">)]</span>
+</div>
+<div class="viewcode-block" id="process_content_history"><a class="viewcode-back" href="../../../../../_layout/nepi.data.processing.ccn.html#nepi.data.processing.ccn.parser.process_content_history">[docs]</a><span class="k">def</span> <span class="nf">process_content_history</span><span class="p">(</span><span class="n">graph</span><span class="p">):</span>
+ <span class="sd">""" Compute CCN message counts and aggregates content historical </span>
+<span class="sd"> information in the content_names dictionary </span>
+<span class="sd"> </span>
+<span class="sd"> """</span>
+
+ <span class="c">## Assume single source</span>
+ <span class="n">source</span> <span class="o">=</span> <span class="n">ccn_consumers</span><span class="p">(</span><span class="n">graph</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
+
+ <span class="n">interest_expiry_count</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="n">interest_dupnonce_count</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="n">interest_count</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="n">content_count</span> <span class="o">=</span> <span class="mi">0</span>
+ <span class="n">content_names</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
+
+ <span class="c"># Collect information about exchanged messages by content name and</span>
+ <span class="c"># link delay info.</span>
+ <span class="k">for</span> <span class="n">nid</span> <span class="ow">in</span> <span class="n">graph</span><span class="o">.</span><span class="n">nodes</span><span class="p">():</span>
+ <span class="c"># Load the data collected from the node's ccnd log</span>
+ <span class="n">fname</span> <span class="o">=</span> <span class="n">graph</span><span class="o">.</span><span class="n">node</span><span class="p">[</span><span class="n">nid</span><span class="p">][</span><span class="s">"history"</span><span class="p">]</span>
+ <span class="n">history</span> <span class="o">=</span> <span class="n">load_content_history</span><span class="p">(</span><span class="n">fname</span><span class="p">)</span>
+
+ <span class="k">for</span> <span class="n">content_name</span> <span class="ow">in</span> <span class="n">history</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
+ <span class="n">hist</span> <span class="o">=</span> <span class="n">history</span><span class="p">[</span><span class="n">content_name</span><span class="p">]</span>
+
+ <span class="k">for</span> <span class="p">(</span><span class="n">timestamp</span><span class="p">,</span> <span class="n">message_type</span><span class="p">,</span> <span class="n">nid1</span><span class="p">,</span> <span class="n">nid2</span><span class="p">,</span> <span class="n">nonce</span><span class="p">,</span> <span class="n">size</span><span class="p">,</span> <span class="n">line</span><span class="p">)</span> <span class="ow">in</span> <span class="n">hist</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">message_type</span> <span class="ow">in</span> <span class="p">[</span><span class="s">"content_from"</span><span class="p">,</span> <span class="s">"content_to"</span><span class="p">]:</span>
+ <span class="c"># The first Interest sent will not have a version or chunk number.</span>
+ <span class="c"># The first Content sent back in reply, will end in /=00 or /%00.</span>
+ <span class="c"># Make sure to map the first Content to the first Interest.</span>
+ <span class="k">if</span> <span class="n">content_name</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s">"/=00"</span><span class="p">):</span>
+ <span class="n">content_name</span> <span class="o">=</span> <span class="s">"/"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">content_name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s">"/"</span><span class="p">)[</span><span class="mi">0</span><span class="p">:</span><span class="o">-</span><span class="mi">2</span><span class="p">])</span>
+
+ <span class="c"># Add content name to dictionary</span>
+ <span class="k">if</span> <span class="n">content_name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">content_names</span><span class="p">:</span>
+ <span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
+ <span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">][</span><span class="s">"interest"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
+ <span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">][</span><span class="s">"content"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">()</span>
+
+ <span class="c"># Classify interests by replica</span>
+ <span class="k">if</span> <span class="n">message_type</span> <span class="ow">in</span> <span class="p">[</span><span class="s">"interest_from"</span><span class="p">]</span> <span class="ow">and</span> \
+ <span class="n">nonce</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">][</span><span class="s">"interest"</span><span class="p">]:</span>
+ <span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">][</span><span class="s">"interest"</span><span class="p">][</span><span class="n">nonce</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">()</span>
+
+ <span class="c"># Add consumer history</span>
+ <span class="k">if</span> <span class="n">nid</span> <span class="o">==</span> <span class="n">source</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">message_type</span> <span class="ow">in</span> <span class="p">[</span><span class="s">"interest_to"</span><span class="p">,</span> <span class="s">"content_from"</span><span class="p">]:</span>
+ <span class="c"># content name history as seen by the source</span>
+ <span class="k">if</span> <span class="s">"consumer_history"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">]:</span>
+ <span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">][</span><span class="s">"consumer_history"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">()</span>
+
+ <span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">][</span><span class="s">"consumer_history"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+ <span class="p">(</span><span class="n">timestamp</span><span class="p">,</span> <span class="n">message_type</span><span class="p">))</span>
+
+ <span class="c"># Add messages per content name and cumulate totals by message type</span>
+ <span class="k">if</span> <span class="n">message_type</span> <span class="o">==</span> <span class="s">"interest_dupnonce"</span><span class="p">:</span>
+ <span class="n">interest_dupnonce_count</span> <span class="o">+=</span> <span class="mi">1</span>
+ <span class="k">elif</span> <span class="n">message_type</span> <span class="o">==</span> <span class="s">"interest_expiry"</span><span class="p">:</span>
+ <span class="n">interest_expiry_count</span> <span class="o">+=</span> <span class="mi">1</span>
+ <span class="k">elif</span> <span class="n">message_type</span> <span class="o">==</span> <span class="s">"interest_from"</span><span class="p">:</span>
+ <span class="n">interest_count</span> <span class="o">+=</span> <span class="mi">1</span>
+ <span class="c"># Append to interest history of the content name</span>
+ <span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">][</span><span class="s">"interest"</span><span class="p">][</span><span class="n">nonce</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+ <span class="p">(</span><span class="n">timestamp</span><span class="p">,</span> <span class="n">nid2</span><span class="p">,</span> <span class="n">nid1</span><span class="p">))</span>
+ <span class="k">elif</span> <span class="n">message_type</span> <span class="o">==</span> <span class="s">"content_from"</span><span class="p">:</span>
+ <span class="n">content_count</span> <span class="o">+=</span> <span class="mi">1</span>
+ <span class="c"># Append to content history of the content name</span>
+ <span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">][</span><span class="s">"content"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">timestamp</span><span class="p">,</span> <span class="n">nid2</span><span class="p">,</span> <span class="n">nid1</span><span class="p">))</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">continue</span>
+ <span class="k">del</span> <span class="n">hist</span>
+ <span class="k">del</span> <span class="n">history</span>
+
+ <span class="c"># Compute the time elapsed between the time an interest is sent</span>
+ <span class="c"># in the consumer node and when the content is received back</span>
+ <span class="k">for</span> <span class="n">content_name</span> <span class="ow">in</span> <span class="n">content_names</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
+ <span class="c"># order content and interest messages by timestamp</span>
+ <span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">][</span><span class="s">"content"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span>
+ <span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">][</span><span class="s">"content"</span><span class="p">])</span>
+
+ <span class="k">for</span> <span class="n">nonce</span><span class="p">,</span> <span class="n">timestamps</span> <span class="ow">in</span> <span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">][</span>
+ <span class="s">"interest"</span><span class="p">]</span><span class="o">.</span><span class="n">iteritems</span><span class="p">():</span>
+ <span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">][</span><span class="s">"interest"</span><span class="p">][</span><span class="n">nonce</span><span class="p">]</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span>
+ <span class="n">timestamps</span><span class="p">)</span>
+
+ <span class="n">history</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">][</span><span class="s">"consumer_history"</span><span class="p">])</span>
+ <span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">][</span><span class="s">"consumer_history"</span><span class="p">]</span> <span class="o">=</span> <span class="n">history</span>
+
+ <span class="c"># compute the rtt time of the message</span>
+ <span class="n">rtt</span> <span class="o">=</span> <span class="bp">None</span>
+ <span class="n">waiting_content</span> <span class="o">=</span> <span class="bp">False</span>
+ <span class="n">interest_timestamp</span> <span class="o">=</span> <span class="bp">None</span>
+ <span class="n">content_timestamp</span> <span class="o">=</span> <span class="bp">None</span>
+
+ <span class="k">for</span> <span class="p">(</span><span class="n">timestamp</span><span class="p">,</span> <span class="n">message_type</span><span class="p">)</span> <span class="ow">in</span> <span class="n">history</span><span class="p">:</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">waiting_content</span> <span class="ow">and</span> <span class="n">message_type</span> <span class="o">==</span> <span class="s">"interest_to"</span><span class="p">:</span>
+ <span class="n">waiting_content</span> <span class="o">=</span> <span class="bp">True</span>
+ <span class="n">interest_timestamp</span> <span class="o">=</span> <span class="n">timestamp</span>
+ <span class="k">continue</span>
+
+ <span class="k">if</span> <span class="n">waiting_content</span> <span class="ow">and</span> <span class="n">message_type</span> <span class="o">==</span> <span class="s">"content_from"</span><span class="p">:</span>
+ <span class="n">content_timestamp</span> <span class="o">=</span> <span class="n">timestamp</span>
+ <span class="k">break</span>
+
+ <span class="c"># If we can't determine who sent the interest, discard it</span>
+ <span class="n">rtt</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
+ <span class="k">if</span> <span class="n">interest_timestamp</span> <span class="ow">and</span> <span class="n">content_timestamp</span><span class="p">:</span>
+ <span class="n">rtt</span> <span class="o">=</span> <span class="n">compute_delay_ms</span><span class="p">(</span><span class="n">content_timestamp</span><span class="p">,</span> <span class="n">interest_timestamp</span><span class="p">)</span>
+
+ <span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">][</span><span class="s">"rtt"</span><span class="p">]</span> <span class="o">=</span> <span class="n">rtt</span>
+ <span class="n">content_names</span><span class="p">[</span><span class="n">content_name</span><span class="p">][</span><span class="s">"lapse"</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">interest_timestamp</span><span class="p">,</span> <span class="n">content_timestamp</span><span class="p">)</span>
+
+ <span class="k">return</span> <span class="p">(</span><span class="n">graph</span><span class="p">,</span>
+ <span class="n">content_names</span><span class="p">,</span>
+ <span class="n">interest_expiry_count</span><span class="p">,</span>
+ <span class="n">interest_dupnonce_count</span><span class="p">,</span>
+ <span class="n">interest_count</span><span class="p">,</span>
+ <span class="n">content_count</span><span class="p">)</span>
+</div>
+<div class="viewcode-block" id="process_content_history_logs"><a class="viewcode-back" href="../../../../../_layout/nepi.data.processing.ccn.html#nepi.data.processing.ccn.parser.process_content_history_logs">[docs]</a><span class="k">def</span> <span class="nf">process_content_history_logs</span><span class="p">(</span><span class="n">logs_dir</span><span class="p">,</span> <span class="n">graph</span><span class="p">,</span> <span class="n">parse_ping_logs</span> <span class="o">=</span> <span class="bp">False</span><span class="p">):</span>
+ <span class="sd">""" Parse CCN logs and aggregate content history information in graph.</span>
+<span class="sd"> Returns annotated graph and message countn and content names history.</span>
+
+<span class="sd"> """</span>
+ <span class="c">## Process logs and analyse data</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">graph</span> <span class="o">=</span> <span class="n">annotate_cn_graph</span><span class="p">(</span><span class="n">logs_dir</span><span class="p">,</span> <span class="n">graph</span><span class="p">,</span>
+ <span class="n">parse_ping_logs</span> <span class="o">=</span> <span class="n">parse_ping_logs</span><span class="p">)</span>
+ <span class="k">except</span><span class="p">:</span>
+ <span class="k">print</span> <span class="s">"Skipping: Error parsing ccnd logs"</span><span class="p">,</span> <span class="n">logs_dir</span>
+ <span class="k">raise</span>
+
+ <span class="n">source</span> <span class="o">=</span> <span class="n">ccn_consumers</span><span class="p">(</span><span class="n">graph</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
+ <span class="n">target</span> <span class="o">=</span> <span class="n">ccn_producers</span><span class="p">(</span><span class="n">graph</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
+
+ <span class="c"># Process the data from the ccnd logs, but do not re compute</span>
+ <span class="c"># the link delay. </span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="p">(</span><span class="n">graph</span><span class="p">,</span>
+ <span class="n">content_names</span><span class="p">,</span>
+ <span class="n">interest_expiry_count</span><span class="p">,</span>
+ <span class="n">interest_dupnonce_count</span><span class="p">,</span>
+ <span class="n">interest_count</span><span class="p">,</span>
+ <span class="n">content_count</span><span class="p">)</span> <span class="o">=</span> <span class="n">process_content_history</span><span class="p">(</span><span class="n">graph</span><span class="p">)</span>
+ <span class="k">except</span><span class="p">:</span>
+ <span class="k">print</span> <span class="s">"Skipping: Error processing ccn data"</span><span class="p">,</span> <span class="n">logs_dir</span>
+ <span class="k">raise</span>
+
+ <span class="k">return</span> <span class="p">(</span><span class="n">graph</span><span class="p">,</span>
+ <span class="n">content_names</span><span class="p">,</span>
+ <span class="n">interest_expiry_count</span><span class="p">,</span>
+ <span class="n">interest_dupnonce_count</span><span class="p">,</span>
+ <span class="n">interest_count</span><span class="p">,</span>
+ <span class="n">content_count</span><span class="p">)</span> </div>
+</pre></div>
+
+ </div>
+ </div>
+ </div>
+ <div class="clearer"></div>
+ </div>
+ <div class="related">
+ <h3>Navigation</h3>
+ <ul>
+ <li class="right" style="margin-right: 10px">
+ <a href="../../../../../genindex.html" title="General Index"
+ >index</a></li>
+ <li class="right" >
+ <a href="../../../../../py-modindex.html" title="Python Module Index"
+ >modules</a> |</li>
+ <li><a href="../../../../../index.html">NEPI 3.2 documentation</a> »</li>
+ <li><a href="../../../../index.html" >Module code</a> »</li>
+ </ul>
+ </div>
+ <div class="footer">
+ © Copyright 2014, Alina Quereilhac, Lucia Guevgeozian Odizzio, Julien Tribino.
+ Created using <a href="http://sphinx-doc.org/">Sphinx</a> 1.2.3.
+ </div>
+ </body>
+</html>
\ No newline at end of file
git://git.onelab.eu / nepi.git / blobdiff