statistics/harvest_tt_resolve.py

   1 #!/usr/bin/python
   2
   3 # Collect statistics from myops db on node downtimes.
   4 # For every node that goes down we need:
   5 #       * node down time
   6 #       * node reboot time
   7 #       * node notice time
   8 #       * node up time
   9
  10 # then for each node, order events by time
  11 #       for each event sequence extract sub-sequences like:
  12 #               down xx up
  13 #       for each such sub-sequence extract
  14 #               time between down and up
  15
  16 from monitor.database.info.model import *
  17 from math import *
  18 import sys
  19 from datetime import datetime
  20
  21 def find_next(history_list, from_index, node_status):
  22         list_len = len(history_list)
  23         for i in range(min(list_len-1, from_index), 0, -1):
  24                 v = history_list[i]
  25                 if v.status == node_status:
  26                         return i
  27
  28         return 0
  29
  30 def diff_time(t1, t2):
  31         d = t1-t2
  32         return d.days*60*60*24 + d.seconds
  33
  34 times = []
  35 frequency = {}
  36 count = 0
  37 for index,node in enumerate(HistoryNodeRecord.query.all()):
  38         frequency[node.hostname] = 0
  39
  40
  41         #if index > 3: sys.exit(1)
  42
  43         if node.hostname == 'planetlab-02.kyushu.jgn2.jp':
  44                 for h in node.versions:
  45                         print h.last_checked, h.status
  46
  47         #print "--"
  48         pairs = []
  49         i = len(node.versions)
  50         ret = find_next(node.versions, i, 'online')
  51         if ret == 0:
  52                 print node.hostname
  53                 print node.status
  54                 print node.last_checked
  55                 print node.last_changed
  56                 #if count > 3: sys.exit(1)
  57                 count += 1
  58                 pairs.append((datetime.now(), node.versions[-1].last_checked))
  59         else:
  60
  61                 while i > 0:
  62                         i = find_next(node.versions, i, 'down')
  63                         i2 = find_next(node.versions, i, 'offline')
  64                         if i == 0 and i2 == 0:
  65                                 break
  66                         h1 = node.versions[i]
  67                         #print i, h1.last_checked, h1.status
  68                         h2 = node.versions[i2]
  69                         #print i2, h2.last_checked, h2.status
  70                         i = i2
  71                         pairs.append((h1.last_checked,h2.last_checked))
  72                         frequency[node.hostname] += 1
  73
  74         # list of all times
  75
  76         for p in pairs:
  77                 times.append(diff_time(p[0],p[1]))
  78
  79 ##frequency
  80 def flip_key(hash):
  81         fk = {}
  82         for key in hash.keys():
  83                 if hash[key] not in fk:
  84                         fk[hash[key]] = []
  85                 fk[hash[key]].append(key)
  86         return fk
  87
  88 freq = flip_key(frequency)
  89 freq_list = freq.keys()
  90 freq_list.sort()
  91 for f in freq_list:
  92         print f, len(freq[f]), freq[f]
  93
  94 times.sort()
  95 bins = {}
  96 for i in range(0,200,1):
  97         step = i/2.0
  98         bins[step] = []
  99
 100 for t in times:
 101         t = t/60.0/60.0/24.0
 102         b = floor(t*2)/2.0
 103         bins[b].append(t)
 104
 105 keys = bins.keys()
 106 keys.sort()
 107 total = 0
 108 for k in keys:
 109         total += len(bins[k])
 110         print k, len(bins[k]), total