Added commands for interacting with google's spreadsheets:

[monitor.git] / statistics / dump_db_m3_raw.py

diff --git a/statistics/dump_db_m3_raw.py b/statistics/dump_db_m3_raw.py
new file mode 100755 (executable)
index 0000000..477c269
--- /dev/null
+++ b/statistics/dump_db_m3_raw.py
@@ -0,0 +1,125 @@
+#!/usr/bin/python
+
+from monitor.database.info.model import *
+import sys
+from math import *
+from monitor.generic import *
+from datetime import datetime, timedelta
+import time
+import string
+
+def list_to_str(list):
+    ret = []
+    for l in list:
+        if type(l) == type([]):
+            ret.append(" ".join([str(i) for i in l]))
+        else:
+            s = str(l)
+            s = s.translate(string.maketrans(",\n\r\"", ";||'"))
+            ret.append(s)
+    return ret
+
+def add_if_not_present(d, add_fields=None):
+    if type(d) == type({}):
+        key_list = d.keys()
+        for k in add_fields.keys():
+            if k not in key_list:
+                d[k] = add_fields[k]
+            else:
+                add_if_not_present(d[k], add_fields[k])
+    return
+
+def dict_to_list(d, add_fields=None, ignore_fields=None):
+    """ return a list of header names from a nested dict 
+        { 'a' : 1, 'b' : { 'c':2, 'd' : 3}} 
+        would return:
+        [ 'a', 'b_c', 'b_d' ]
+    """
+    k_list = []
+    d_list = []
+    if add_fields: add_if_not_present(d, add_fields)
+    for k in d.keys():
+        if type(d[k]) == type({}):
+            (z_kl, z_dl) = dict_to_list(d[k])
+            for i,zk in enumerate(map(lambda x: "%s_%s" % (k,x), z_kl)):
+                if ignore_fields is None or zk not in ignore_fields:
+                    k_list.append(zk) 
+                    d_list.append(z_dl[i]) 
+        else:
+            if ignore_fields is None or k not in ignore_fields:
+                k_list.append(k)
+                d_list.append(d[k])
+    r = zip(k_list, list_to_str(d_list))
+    r.sort(lambda x,y: cmp(x[0], y[0]))
+    return ([ i[0] for i in r ], [ i[1] for i in r ])
+
+
+if len(sys.argv) > 1 and sys.argv[1] == "--action":
+    args = sys.argv[2:]
+    find_action = True
+else:
+    if len(sys.argv) > 1:
+        since_time = Time.str_to_dt(sys.argv[1], "%Y-%m-%d")
+        skip = int(sys.argv[2])
+        args = sys.argv[3:]
+    else:
+        args = sys.argv[1:]
+
+    find_action = False
+
+first_time = True
+index = 0
+t1=t2=0
+if find_action:
+    a = ActionRecord.query.all()
+    print >>sys.stderr, len(a)
+    for node in a:
+
+        print >>sys.stderr, index, node.hostname, t2-t1
+        index += 1
+        t1 = time.time()
+
+        d = node.__dict__
+        (k,l) = dict_to_list(d)
+        if first_time:
+            print "timestamp_unix,%s" % ",".join(k[1:])
+            first_time = False
+
+        print "%s,%s" % (Time.dt_to_ts(d['date_created']), ",".join(l[1:]))
+        t2=time.time()
+
+else:
+    ignore_fields = ['plc_node_stats_nodenetwork_ids', 'port_status_806', 'port_status_22', 'port_status_80' ]
+    add_fields = {'plc_node_stats' : { 'last_boot' : 0, 'last_pcu_confirmation' : 0, 'last_pcu_reboot' : 0, 'last_download' : 0, 'run_level': 0, }}
+    for node in FindbadNodeRecord.query.all():
+
+        print >>sys.stderr, index, node.hostname, t2-t1
+        index += 1
+        t1 = time.time()
+        if index > skip :
+            for v in node.versions:
+
+                d = v.__dict__
+                (k,l) = dict_to_list(d, add_fields=add_fields, ignore_fields=ignore_fields)
+                if not first_time:
+                    if cmp(k, k_last) != 0:
+                        print >>sys.stderr, "mismatching key lists"
+                        print >>sys.stderr, k
+                        print >>sys.stderr, k_last
+                        for i in zip(k,k_last):
+                            print >>sys.stderr, i
+                        print >>sys.stderr, set(k) - set(k_last)
+                        print >>sys.stderr, set(k_last) - set(k)
+                        #sys.exit(1)
+                        continue
+
+                if d['timestamp'] > since_time:
+                    if first_time:
+                        print "timestamp_unix,%s" % ",".join(k[1:])
+                        first_time = False
+
+
+                    print "%s,%s" % (Time.dt_to_ts(d['timestamp']), ",".join(l[1:]))
+
+                k_last = k
+        t2=time.time()