statistics/harvest_tt_resolve.py

   1 #!/usr/bin/python
   2
   3 # Collect statistics from myops db on node downtimes.
   4 # For every node that goes down we need:
   5 #       * node down time
   6 #       * node reboot time
   7 #       * node notice time
   8 #       * node up time
   9
  10 # then for each node, order events by time
  11 #       for each event sequence extract sub-sequences like:
  12 #               down xx up
  13 #       for each such sub-sequence extract
  14 #               time between down and up
  15
  16 from monitor.database.info.model import *
  17 from math import *
  18 import sys
  19 from datetime import datetime
  20
  21 def find_next(history_list, from_index, node_status):
  22         list_len = len(history_list)
  23         for i in range(min(list_len-1, from_index), 0, -1):
  24                 v = history_list[i]
  25                 if v.status == node_status:
  26                         return i
  27
  28         return 0
  29
  30 def diff_time(t1, t2):
  31         d = t1-t2
  32         return d.days*60*60*24 + d.seconds
  33
  34 times = []
  35 frequency = {}
  36 count = 0
  37 for index,node in enumerate(HistoryNodeRecord.query.all()):
  38         frequency[node.hostname] = 0
  39
  40         if node.hostname == 'planetlab-02.kyushu.jgn2.jp':
  41                 for h in node.versions:
  42                         print h.last_checked, h.status
  43
  44         #print "--"
  45         pairs = []
  46         i = len(node.versions)
  47         ret = find_next(node.versions, i, 'online')
  48         if ret == 0:
  49                 print node.hostname
  50                 print node.status
  51                 print node.last_checked
  52                 print node.last_changed
  53                 #if count > 3: sys.exit(1)
  54                 count += 1
  55                 pairs.append((datetime.now(), node.versions[-1].last_checked))
  56         else:
  57                 while i > 0:
  58                         i = find_next(node.versions, i, 'down')
  59                         i2 = find_next(node.versions, i, 'offline')
  60                         if i == 0 and i2 == 0:
  61                                 break
  62                         h1 = node.versions[i]
  63                         #print i, h1.last_checked, h1.status
  64                         h2 = node.versions[i2]
  65                         #print i2, h2.last_checked, h2.status
  66                         i = i2
  67                         pairs.append((h1.last_checked,h2.last_checked))
  68                         frequency[node.hostname] += 1
  69
  70         # list of all times
  71         for p in pairs:
  72                 times.append(diff_time(p[0],p[1]))
  73
  74 ##frequency
  75 def flip_key(hash):
  76         fk = {}
  77         for key in hash.keys():
  78                 if hash[key] not in fk:
  79                         fk[hash[key]] = []
  80                 fk[hash[key]].append(key)
  81         return fk
  82
  83 freq = flip_key(frequency)
  84 freq_list = freq.keys()
  85 freq_list.sort()
  86 for f in freq_list:
  87         print f, len(freq[f]), freq[f]
  88
  89 times.sort()
  90 bins = {}
  91 for i in range(0,200,1):
  92         step = i/2.0
  93         bins[step] = []
  94
  95 for t in times:
  96         t = t/60.0/60.0/24.0
  97         b = floor(t*2)/2.0
  98         bins[b].append(t)
  99
 100 keys = bins.keys()
 101 keys.sort()
 102 total = 0
 103 for k in keys:
 104         total += len(bins[k])
 105         print k, len(bins[k]), total