statistics/comon_summary_parser.py

   1 #!/usr/bin/python
   2
   3 import sys, os, time, bz2
   4
   5 if len(sys.argv) != 2 :
   6         print 'usage: bz2comonlogfile'
   7         sys.exit()
   8
   9 filename = sys.argv[1]
  10 start_time = time.time()
  11 if not ('dump_comon_' in filename and filename.endswith('bz2')) :
  12         print 'not a comon log file:'
  13         sys.exit()
  14
  15 def str_to_ts(date_str, format="%Y-%m-%d"):
  16    ts = time.mktime(time.strptime(date_str, format))
  17    return ts
  18
  19
  20 # .../dump_cotop_20080101 -> 2008-01-01
  21 indx = filename.rfind('dump_comon_') + len('dump_comon') + 1
  22 date = filename[indx:indx+8]
  23 date = date[0:4] + '-' + date[4:6] + '-' + date[6:8]
  24 ts = str_to_ts(date)
  25
  26 # read in bz2 log file
  27 bzobj = bz2.BZ2File(filename, mode = 'r')
  28 lines = bzobj.readlines()
  29
  30 last_time = 0
  31 entry = {}
  32 processed_tags = set()
  33
  34 # keep track of malformed entries
  35 n_badTS = 0
  36 n_ambigTS = 0
  37 n_NA = {}
  38
  39 important_tags = ['Start:', 'Name:', 'RespTime:']
  40
  41 def get_field(table, *args):
  42         pos = table
  43         l = len(args)
  44         for i,v in enumerate(args[:-1]):
  45                 if v not in pos:
  46                         pos[v] = {}
  47                 pos = pos[v]
  48         v = args[-1]
  49         if args[-1] not in pos:
  50                 pos[v] = 0
  51         return pos[v]
  52
  53 def set_field(table, *args):
  54         pos = table
  55         l = len(args)
  56         #get_field(table, *args[0:-1])
  57         for i,v in enumerate(args[:-2]):
  58                 pos = pos[v]
  59         pos[args[-2]] = args[-1]
  60
  61 def isValid(entry):
  62         # check important_tags
  63         for t in important_tags:
  64                 if t not in entry:
  65                         #print "bad entry", entry
  66                         return False
  67
  68         try:
  69                 if 'Uptime:' in entry:
  70                         float(entry['Uptime:'])
  71         except:
  72                 #print "uptime fault"
  73                 return False
  74
  75         return True
  76
  77 hs = {} # HOST SUMMARY
  78
  79 # Process log
  80 for line in lines :
  81         line = line.strip()
  82         if line == '' :
  83                 #Process timestamp
  84                 try :
  85                         this_time = int(entry['Start:'][0])
  86                         fmtime = time.strftime('%D %T', time.localtime(this_time))
  87                         ambigTS = this_time < last_time
  88                         if ambigTS :
  89                                 n_ambigTS += 1
  90                         else :
  91                                 last_time = this_time
  92                         #outcsv.write('%d,%s' % (this_time, ambigTS))
  93                 except KeyError :
  94                         continue
  95                 except :
  96                         n_badTS += 1
  97                         entry = {}
  98                         processed_tags = set()
  99                         continue
 100                 #Process other fields
 101                 #try :
 102
 103
 104                 if True:
 105
 106                         if not isValid(entry):
 107                                 entry = {}
 108                                 processed_tags = set()
 109                                 continue
 110
 111                         h = entry['Name:']
 112
 113                         if h not in hs:
 114                                 get_field(hs,h,'offline')
 115                                 get_field(hs,h,'online')
 116                                 get_field(hs,h,'uptime')
 117
 118                         try:
 119                                 if len(entry['RespTime:'].split()) > 1:
 120                                         set_field(hs,h,'offline', get_field(hs,h,'offline') + 1)
 121                                 else:
 122                                         set_field(hs,h,'online', get_field(hs,h,'online') + 1)
 123                                         set_field(hs,h,'uptime', max(get_field(hs,h,'uptime'),entry['Uptime:']) )
 124                         except:
 125                                 #print "except resptime"
 126                                 continue
 127
 128
 129                 #except KeyError :
 130                 ##      print "key error! on hostname: %s" % h
 131                 #       continue
 132
 133                 entry = {}
 134                 processed_tags = set()
 135         else :
 136                 words = line.split()
 137                 tag = words[0]
 138                 if tag in processed_tags :      # start over, since a tag is repeating
 139                         entry = {}
 140                         processed_tags = set()
 141                 entry[tag] = " ".join(words[1:len(words)])
 142                 processed_tags.add(tag)
 143
 144 # finished processing log file
 145
 146 # clean up memory objs
 147 #outcsv.close()
 148 bzobj.close()
 149 lines = ''
 150
 151 online = 0
 152 offline = 0
 153 uptimes = []
 154 #print "number of hosts:%s" % len(hs)
 155 for h in hs:
 156         if hs[h]['uptime'] > 0: uptimes.append(float(hs[h]['uptime']))
 157         if hs[h]['online'] > hs[h]['offline']:
 158                 online += 1
 159         else:
 160                 offline += 1
 161
 162 l = len(uptimes)
 163 uptimes.sort()
 164 end_time = time.time()
 165 print date, ts, online+offline, online, offline, uptimes[0], uptimes[l/4], uptimes[l/2], uptimes[l/2+l/4], uptimes[-1], end_time-start_time, filename