git://git.onelab.eu / nepi.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
still making both branches closer
[nepi.git] / src / nepi / data / processing / ccn / parser.py
diff --git a/src/nepi/data/processing/ccn/parser.py b/src/nepi/data/processing/ccn/parser.py
index e3c1007..64e2ada 100644 (file)
--- a/src/nepi/data/processing/ccn/parser.py
+++ b/src/nepi/data/processing/ccn/parser.py
@@ -6,9 +6,8 @@
 #    Copyright (C) 2014 INRIA
 #
 #    This program is free software: you can redistribute it and/or modify
 #    Copyright (C) 2014 INRIA
 #
 #    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
+#    it under the terms of the GNU General Public License version 2 as
+#    published by the Free Software Foundation;
 #
 #    This program is distributed in the hope that it will be useful,
 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
 #
 #    This program is distributed in the hope that it will be useful,
 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -41,6 +40,8 @@
 #                        \ nid3.log
 #
 
 #                        \ nid3.log
 #
 
+from __future__ import print_function
+
 import collections
 import functools
 import networkx
 import collections
 import functools
 import networkx
@@ -68,82 +69,80 @@ def parse_file(filename):
     faces = dict()
     sep = " "
 
     faces = dict()
     sep = " "
 
-    f = open(filename, "r")
-
-    data = []
-
-    for line in f:
-        cols =  line.strip().split(sep)
-
-        # CCN_PEEK
-        # MESSAGE interest_from
-        # 1374181938.808523 ccnd[9245]: debug.4352 interest_from 6 ccnx:/test/bunny.ts (23 bytes,sim=0CDCC1D7)
-        #
-        # MESSAGE interest_to
-        # 1374181938.812750 ccnd[9245]: debug.3502 interest_to 5 ccnx:/test/bunny.ts (39 bytes,i=2844,sim=0CDCC1D7)
-        #
-        # MESSAGE CONTENT FROM
-        # 1374181938.868682 ccnd[9245]: debug.4643 content_from 5 ccnx:/test/bunny.ts/%FD%05%1E%85%8FVw/%00/%9E%3D%01%D9%3Cn%95%2BvZ%8
-        #
-        # MESSAGE CONTENT_TO
-        # 1374181938.868772 ccnd[9245]: debug.1619 content_to 6 ccnx:/test/bunny.ts/%FD%05%1E%85%8FVw/%00/%9E%3D%01%D9%3Cn%95%2BvZ%8
-        #
-        # 1375596708.222304 ccnd[9758]: debug.3692 interest_expiry ccnx:/test/bunny.ts/%FD%05%1E%86%B1GS/%00%0A%F7 (44 bytes,c=0:1,i=2819,sim=49FA8048)
-
-        # External face creation
-        # 1374181452.965961 ccnd[9245]: accepted datagram client id=5 (flags=0x40012) 204.85.191.10 port 9695
-
-        if line.find("accepted datagram client") > -1:
-            face_id = (cols[5]).replace("id=",'')
-            ip = cols[7] 
-            port = cols[9]
-            faces[face_id] = (ip, port)
-            continue
-
-        # 1374181452.985296 ccnd[9245]: releasing face id 4 (slot 4)
-        if line.find("releasing face id") > -1:
-            face_id = cols[5]
-            if face_id in faces:
-                del faces[face_id]
-            continue
-
-        if len(cols) < 6:
-            continue
-
-        timestamp = cols[0]
-        message_type = cols[3]
+    with open(filename, "r") as f:
+
+        data = []
+
+        for line in f:
+            cols =  line.strip().split(sep)
+
+            # CCN_PEEK
+            # MESSAGE interest_from
+            # 1374181938.808523 ccnd[9245]: debug.4352 interest_from 6 ccnx:/test/bunny.ts (23 bytes,sim=0CDCC1D7)
+            #
+            # MESSAGE interest_to
+            # 1374181938.812750 ccnd[9245]: debug.3502 interest_to 5 ccnx:/test/bunny.ts (39 bytes,i=2844,sim=0CDCC1D7)
+            #
+            # MESSAGE CONTENT FROM
+            # 1374181938.868682 ccnd[9245]: debug.4643 content_from 5 ccnx:/test/bunny.ts/%FD%05%1E%85%8FVw/%00/%9E%3D%01%D9%3Cn%95%2BvZ%8
+            #
+            # MESSAGE CONTENT_TO
+            # 1374181938.868772 ccnd[9245]: debug.1619 content_to 6 ccnx:/test/bunny.ts/%FD%05%1E%85%8FVw/%00/%9E%3D%01%D9%3Cn%95%2BvZ%8
+            #
+            # 1375596708.222304 ccnd[9758]: debug.3692 interest_expiry ccnx:/test/bunny.ts/%FD%05%1E%86%B1GS/%00%0A%F7 (44 bytes,c=0:1,i=2819,sim=49FA8048)
+
+            # External face creation
+            # 1374181452.965961 ccnd[9245]: accepted datagram client id=5 (flags=0x40012) 204.85.191.10 port 9695
+
+            if line.find("accepted datagram client") > -1:
+                face_id = (cols[5]).replace("id=",'')
+                ip = cols[7] 
+                port = cols[9]
+                faces[face_id] = (ip, port)
+                continue
 
 
-        if message_type not in ["interest_from", "interest_to", "content_from", 
-                "content_to", "interest_dupnonce", "interest_expiry"]:
-            continue
+            # 1374181452.985296 ccnd[9245]: releasing face id 4 (slot 4)
+            if line.find("releasing face id") > -1:
+                face_id = cols[5]
+                if face_id in faces:
+                    del faces[face_id]
+                continue
 
 
-        face_id = cols[4] 
-        content_name = cols[5]
+            if len(cols) < 6:
+                continue
 
 
-        # Interest Nonce ? -> 412A74-0844-0008-50AA-F6EAD4
-        nonce = ""
-        if message_type in ["interest_from", "interest_to", "interest_dupnonce"]:
-            last = cols[-1]
-            if len(last.split("-")) == 5:
-                nonce = last
+            timestamp = cols[0]
+            message_type = cols[3]
 
 
-        try:
-            size = int((cols[6]).replace('(',''))
-        except:
-            print "interest_expiry without face id!", line
-            continue
+            if message_type not in ["interest_from", "interest_to", "content_from", 
+                    "content_to", "interest_dupnonce", "interest_expiry"]:
+                continue
 
 
-        # If no external IP address was identified for this face
-        # asume it is a local face
-        peer = "localhost"
+            face_id = cols[4] 
+            content_name = cols[5]
 
 
-        if face_id in faces:
-            peer, port = faces[face_id]
+            # Interest Nonce ? -> 412A74-0844-0008-50AA-F6EAD4
+            nonce = ""
+            if message_type in ["interest_from", "interest_to", "interest_dupnonce"]:
+                last = cols[-1]
+                if len(last.split("-")) == 5:
+                    nonce = last
 
 
-        data.append((content_name, timestamp, message_type, peer, face_id, 
-            size, nonce, line))
+            try:
+                size = int((cols[6]).replace('(',''))
+            except:
+                print("interest_expiry without face id!", line)
+                continue
 
 
-    f.close()
+            # If no external IP address was identified for this face
+            # asume it is a local face
+            peer = "localhost"
+    
+            if face_id in faces:
+                peer, port = faces[face_id]
+    
+            data.append((content_name, timestamp, message_type, peer, face_id, 
+                size, nonce, line))
 
     return data
 
 
     return data
 
@@ -154,9 +153,8 @@ def dump_content_history(content_history):
     return f.name
 
 def load_content_history(fname):
     return f.name
 
 def load_content_history(fname):
-    f = open(fname, "r")
-    content_history = pickle.load(f)
-    f.close()
+    with open(fname, "r") as f:
+        content_history = pickle.load(f)
 
     os.remove(fname)
     return content_history
 
     os.remove(fname)
     return content_history
@@ -199,6 +197,7 @@ def annotate_cn_graph(logs_dir, graph, parse_ping_logs = False):
     """ Adds CCN content history for each node in the topology graph.
 
     """
     """ Adds CCN content history for each node in the topology graph.
 
     """
+    
     # Make a copy of the graph to ensure integrity
     graph = graph.copy()
 
     # Make a copy of the graph to ensure integrity
     graph = graph.copy()
 
@@ -209,6 +208,8 @@ def annotate_cn_graph(logs_dir, graph, parse_ping_logs = False):
         for ip in ips:
             ips2nid[ip] = nid
 
         for ip in ips:
             ips2nid[ip] = nid
 
+    found_files = False
+
     # Now walk through the ccnd logs...
     for dirpath, dnames, fnames in os.walk(logs_dir):
         # continue if we are not at the leaf level (if there are subdirectories)
     # Now walk through the ccnd logs...
     for dirpath, dnames, fnames in os.walk(logs_dir):
         # continue if we are not at the leaf level (if there are subdirectories)
@@ -217,11 +218,16 @@ def annotate_cn_graph(logs_dir, graph, parse_ping_logs = False):
         
         # Each dirpath correspond to a different node
         nid = os.path.basename(dirpath)
         
         # Each dirpath correspond to a different node
         nid = os.path.basename(dirpath)
+
+        # Cast to numeric nid if necessary
+        if int(nid) in graph.nodes():
+            nid = int(nid)
     
         content_history = dict()
 
         for fname in fnames:
             if fname.endswith(".log"):
     
         content_history = dict()
 
         for fname in fnames:
             if fname.endswith(".log"):
+                found_files = True
                 filename = os.path.join(dirpath, fname)
                 data = parse_file(filename)
                 annotate_cn_node(graph, nid, ips2nid, data, content_history)
                 filename = os.path.join(dirpath, fname)
                 data = parse_file(filename)
                 annotate_cn_node(graph, nid, ips2nid, data, content_history)
@@ -231,6 +237,10 @@ def annotate_cn_graph(logs_dir, graph, parse_ping_logs = False):
         fname = dump_content_history(content_history)
         graph.node[nid]["history"] = fname
 
         fname = dump_content_history(content_history)
         graph.node[nid]["history"] = fname
 
+    if not found_files:
+        msg = "No CCND output files were found to parse at %s " % logs_dir
+        raise RuntimeError(msg)
+
     if parse_ping_logs:
         ping_parser.annotate_cn_graph(logs_dir, graph)
 
     if parse_ping_logs:
         ping_parser.annotate_cn_graph(logs_dir, graph)
 
@@ -268,7 +278,7 @@ def process_content_history(graph):
         fname = graph.node[nid]["history"]
         history = load_content_history(fname)
 
         fname = graph.node[nid]["history"]
         history = load_content_history(fname)
 
-        for content_name in history.keys():
+        for content_name in history:
             hist = history[content_name]
 
             for (timestamp, message_type, nid1, nid2, nonce, size, line) in hist:
             hist = history[content_name]
 
             for (timestamp, message_type, nid1, nid2, nonce, size, line) in hist:
@@ -321,13 +331,13 @@ def process_content_history(graph):
 
     # Compute the time elapsed between the time an interest is sent
     # in the consumer node and when the content is received back
 
     # Compute the time elapsed between the time an interest is sent
     # in the consumer node and when the content is received back
-    for content_name in content_names.keys():
+    for content_name in content_names:
         # order content and interest messages by timestamp
         content_names[content_name]["content"] = sorted(
               content_names[content_name]["content"])
         
         for nonce, timestamps in content_names[content_name][
         # order content and interest messages by timestamp
         content_names[content_name]["content"] = sorted(
               content_names[content_name]["content"])
         
         for nonce, timestamps in content_names[content_name][
-                    "interest"].iteritems():
+                    "interest"].items():
               content_names[content_name]["interest"][nonce] = sorted(
                         timestamps)
       
               content_names[content_name]["interest"][nonce] = sorted(
                         timestamps)
       
@@ -365,7 +375,7 @@ def process_content_history(graph):
         interest_count,
         content_count)
 
         interest_count,
         content_count)
 
-def process_content_history_logs(logs_dir, graph):
+def process_content_history_logs(logs_dir, graph, parse_ping_logs = False):
     """ Parse CCN logs and aggregate content history information in graph.
     Returns annotated graph and message countn and content names history.
 
     """ Parse CCN logs and aggregate content history information in graph.
     Returns annotated graph and message countn and content names history.
 
@@ -373,9 +383,9 @@ def process_content_history_logs(logs_dir, graph):
     ## Process logs and analyse data
     try:
         graph = annotate_cn_graph(logs_dir, graph, 
     ## Process logs and analyse data
     try:
         graph = annotate_cn_graph(logs_dir, graph, 
-                parse_ping_logs = True)
+                parse_ping_logs = parse_ping_logs)
     except:
     except:
-        print "Skipping: Error parsing ccnd logs", logs_dir
+        print("Skipping: Error parsing ccnd logs", logs_dir)
         raise
 
     source = ccn_consumers(graph)[0]
         raise
 
     source = ccn_consumers(graph)[0]
@@ -391,7 +401,7 @@ def process_content_history_logs(logs_dir, graph):
         interest_count,
         content_count) = process_content_history(graph)
     except:
         interest_count,
         content_count) = process_content_history(graph)
     except:
-        print "Skipping: Error processing ccn data", logs_dir
+        print("Skipping: Error processing ccn data", logs_dir)
         raise
 
     return (graph,
         raise
 
     return (graph,
nepi