nodecommon.py

   1
   2 import struct
   3 import reboot
   4 import time
   5 import util.file
   6 import plc
   7 from datetime import datetime
   8 from monitor import database
   9 from unified_model import PersistFlags
  10 esc = struct.pack('i', 27)
  11 RED     = esc + "[1;31m"
  12 GREEN   = esc + "[1;32m"
  13 YELLOW  = esc + "[1;33m"
  14 BLUE    = esc + "[1;34m"
  15 LIGHTBLUE       = esc + "[1;36m"
  16 NORMAL  = esc + "[0;39m"
  17
  18 def red(str):
  19         return RED + str + NORMAL
  20
  21 def yellow(str):
  22         return YELLOW + str + NORMAL
  23
  24 def green(str):
  25         return GREEN + str + NORMAL
  26
  27 def lightblue(str):
  28         return LIGHTBLUE + str + NORMAL
  29
  30 def blue(str):
  31         return BLUE + str + NORMAL
  32
  33 def get_current_state(fbnode):
  34         if 'state' in fbnode:
  35                 state = fbnode['state']
  36         else:
  37                 state = "none"
  38         l = state.lower()
  39         if l == "debug": l = 'dbg '
  40         return l
  41
  42 def color_pcu_state(fbnode):
  43
  44         if 'plcnode' in fbnode and 'pcu_ids' in fbnode['plcnode'] and len(fbnode['plcnode']['pcu_ids']) > 0 :
  45                 values = reboot.get_pcu_values(fbnode['plcnode']['pcu_ids'][0])
  46                 if values == None:
  47                         return fbnode['pcu']
  48         else:
  49                 if 'pcu' not in fbnode:
  50                         return 'NOPCU'
  51                 else:
  52                         return fbnode['pcu']
  53
  54         if 'reboot' in values:
  55                 rb = values['reboot']
  56                 if rb == 0 or rb == "0":
  57                         return fbnode['pcu'] + "OK  "
  58                         #return fbnode['pcu'] + "OK  "
  59                         #return green(fbnode['pcu'])
  60                 elif "NetDown" == rb  or "Not_Run" == rb:
  61                         return fbnode['pcu'] + "DOWN"
  62                         #return yellow(fbnode['pcu'])
  63                 else:
  64                         return fbnode['pcu'] + "BAD "
  65                         #return red(fbnode['pcu'])
  66         else:
  67                 #return red(fbnode['pcu'])
  68                 return fbnode['pcu'] + "BAD "
  69
  70 def color_boot_state(l):
  71         if    l == "dbg": return yellow("debg")
  72         elif  l == "dbg ": return yellow("debg")
  73         elif  l == "diag": return lightblue(l)
  74         elif  l == "disable": return red("dsbl")
  75         elif  l == "down": return red(l)
  76         elif  l == "boot": return green(l)
  77         elif  l == "rins": return blue(l)
  78         elif  l == "reinstall": return blue(l)
  79         else:
  80                 return l
  81
  82 def diff_time(timestamp, abstime=True):
  83         import math
  84         now = time.time()
  85         if timestamp == None:
  86                 return "unknown"
  87         if abstime:
  88                 diff = now - timestamp
  89         else:
  90                 diff = timestamp
  91         # return the number of seconds as a difference from current time.
  92         t_str = ""
  93         if diff < 60: # sec in min.
  94                 t = diff / 1
  95                 t_str = "%s sec ago" % int(math.ceil(t))
  96         elif diff < 60*60: # sec in hour
  97                 t = diff / (60)
  98                 t_str = "%s min ago" % int(math.ceil(t))
  99         elif diff < 60*60*24: # sec in day
 100                 t = diff / (60*60)
 101                 t_str = "%s hrs ago" % int(math.ceil(t))
 102         elif diff < 60*60*24*14: # sec in week
 103                 t = diff / (60*60*24)
 104                 t_str = "%s days ago" % int(math.ceil(t))
 105         elif diff <= 60*60*24*30: # approx sec in month
 106                 t = diff / (60*60*24*7)
 107                 t_str = "%s wks ago" % int(math.ceil(t))
 108         elif diff > 60*60*24*30: # approx sec in month
 109                 t = diff / (60*60*24*30)
 110                 t_str = "%s mnths ago" % int(t)
 111         return t_str
 112
 113 def getvalue(fb, path):
 114     indexes = path.split("/")
 115     values = fb
 116     for index in indexes:
 117         if index in values:
 118             values = values[index]
 119         else:
 120             return None
 121     return values
 122
 123 def nodegroup_display(node, fb, conf=None):
 124         if node['hostname'] in fb['nodes']:
 125                 node['current'] = get_current_state(fb['nodes'][node['hostname']]['values'])
 126         else:
 127                 node['current'] = 'none'
 128
 129         if fb['nodes'][node['hostname']]['values'] == []:
 130                 return ""
 131
 132         s = fb['nodes'][node['hostname']]['values']['kernel'].split()
 133         if len(s) >=3:
 134                 node['kernel'] = s[2]
 135         else:
 136                 node['kernel'] = fb['nodes'][node['hostname']]['values']['kernel']
 137
 138         if '2.6' not in node['kernel']: node['kernel'] = ""
 139         if conf and not conf.nocolor:
 140             node['boot_state']  = color_boot_state(node['boot_state'])
 141             node['current']     = color_boot_state(node['current'])
 142         #node['boot_state']     = node['boot_state']
 143         #node['current']        = node['current']
 144         node['pcu'] = fb['nodes'][node['hostname']]['values']['pcu']
 145         node['lastupdate'] = diff_time(node['last_contact'])
 146         pf = PersistFlags(node['hostname'], 1, db='node_persistflags')
 147         try:
 148                 node['lc'] = diff_time(pf.last_changed)
 149         except:
 150                 node['lc'] = "err"
 151         ut = fb['nodes'][node['hostname']]['values']['comonstats']['uptime']
 152         if ut != "null":
 153                 ut = diff_time(float(fb['nodes'][node['hostname']]['values']['comonstats']['uptime']), False)
 154         node['uptime'] = ut
 155
 156         return "%(hostname)-42s %(boot_state)8s %(current)5s %(pcu)6s %(key)10.10s... %(kernel)35.35s %(lastupdate)12s, %(lc)s, %(uptime)s" % node
 157
 158 def datetime_fromstr(str):
 159         if '-' in str:
 160                 try:
 161                         tup = time.strptime(str, "%Y-%m-%d")
 162                 except:
 163                         tup = time.strptime(str, "%Y-%m-%d-%H:%M")
 164         elif '/' in str:
 165                 tup = time.strptime(str, "%m/%d/%Y")
 166         else:
 167                 tup = time.strptime(str, "%m/%d/%Y")
 168         ret = datetime.fromtimestamp(time.mktime(tup))
 169         return ret
 170
 171 def get_nodeset(config):
 172         """
 173                 Given the config values passed in, return the set of hostnames that it
 174                 evaluates to.
 175         """
 176         api = plc.getAuthAPI()
 177         l_nodes = database.dbLoad("l_plcnodes")
 178
 179         if config.nodelist:
 180                 f_nodes = util.file.getListFromFile(config.nodelist)
 181                 l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
 182         elif config.node:
 183                 f_nodes = [config.node]
 184                 l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
 185         elif config.nodegroup:
 186                 ng = api.GetNodeGroups({'groupname' : config.nodegroup})
 187                 l_nodes = api.GetNodes(ng[0]['node_ids'], ['hostname'])
 188         elif config.site:
 189                 site = api.GetSites(config.site)
 190                 l_nodes = api.GetNodes(site[0]['node_ids'], ['hostname'])
 191
 192         l_nodes = [node['hostname'] for node in l_nodes]
 193
 194         # perform this query after the above options, so that the filter above
 195         # does not break.
 196         if config.nodeselect:
 197                 fb = database.dbLoad("findbad")
 198                 l_nodes = node_select(config.nodeselect, fb['nodes'].keys(), fb)
 199
 200         return l_nodes
 201
 202 def email_exception(content=None):
 203         import config
 204         from unified_model import Message
 205         import traceback
 206         msg=traceback.format_exc()
 207         if content:
 208                 msg = content + "\n" + msg
 209         m=Message("exception running monitor", msg, False)
 210         m.send([config.cc_email])
 211         return