monitor/common.py

   1
   2 import time
   3 import struct
   4 from monitor import reboot
   5 from monitor import util
   6 from monitor import query
   7 from monitor.wrapper import plc
   8
   9 from datetime import datetime, timedelta
  10 from monitor.model import Message
  11 from monitor.database.info import HistoryNodeRecord
  12
  13 esc = struct.pack('i', 27)
  14 RED     = esc + "[1;31m"
  15 GREEN   = esc + "[1;32m"
  16 YELLOW  = esc + "[1;33m"
  17 BLUE    = esc + "[1;34m"
  18 LIGHTBLUE       = esc + "[1;36m"
  19 NORMAL  = esc + "[0;39m"
  20
  21 def red(str):
  22         return RED + str + NORMAL
  23
  24 def yellow(str):
  25         return YELLOW + str + NORMAL
  26
  27 def green(str):
  28         return GREEN + str + NORMAL
  29
  30 def lightblue(str):
  31         return LIGHTBLUE + str + NORMAL
  32
  33 def blue(str):
  34         return BLUE + str + NORMAL
  35
  36 def get_current_state(fbnode):
  37         if 'observed_status' in fbnode:
  38                 state = fbnode['observed_status']
  39         else:
  40                 state = "none"
  41         l = state.lower()
  42         if l == "debug": l = 'dbg '
  43         return l
  44
  45 def color_pcu_state(fbnode):
  46         if fbnode['plc_pcuid'] is None:
  47                 return 'NOPCU'
  48         else:
  49                 return 'PCU'
  50
  51         if 'plcnode' in fbnode and 'pcu_ids' in fbnode['plcnode'] and len(fbnode['plcnode']['pcu_ids']) > 0 :
  52                 values = reboot.get_pcu_values(fbnode['plcnode']['pcu_ids'][0])
  53                 if values == None:
  54                         return fbnode['pcu']
  55         else:
  56                 print fbnode.keys()
  57                 if 'pcu' not in fbnode:
  58                         return 'NOPCU'
  59                 else:
  60                         return fbnode['pcu']
  61
  62         if 'reboot' in values:
  63                 rb = values['reboot']
  64                 if rb == 0 or rb == "0":
  65                         return fbnode['pcu'] + "OK  "
  66                 elif "NetDown" == rb  or "Not_Run" == rb:
  67                         return fbnode['pcu'] + "DOWN"
  68                 else:
  69                         return fbnode['pcu'] + "BAD "
  70         else:
  71                 return fbnode['pcu'] + "BAD "
  72
  73 def color_boot_state(l):
  74         if    l == "dbg": return yellow("debg")
  75         elif  l == "dbg ": return yellow("debg")
  76         elif  l == "failboot": return yellow("debg")
  77         elif  l == "diag": return lightblue(l)
  78         elif  l == "diagnose": return lightblue(l)
  79         elif  l == "safeboot": return lightblue(l)
  80         elif  l == "disable": return red("dsbl")
  81         elif  l == "disabled": return red("dsbl")
  82         elif  l == "down": return red(l)
  83         elif  l == "boot": return green(l)
  84         elif  l == "rins": return blue(l)
  85         elif  l == "reinstall": return blue(l)
  86         else:
  87                 return l
  88
  89 def diff_time(timestamp, abstime=True):
  90         import math
  91         now = time.time()
  92         if timestamp == None:
  93                 return "unknown"
  94         if type(timestamp) == type(datetime.now()):
  95                 timestamp = time.mktime(timestamp.timetuple())
  96         if abstime:
  97                 diff = now - timestamp
  98         else:
  99                 diff = timestamp
 100         # return the number of seconds as a difference from current time.
 101         t_str = ""
 102         if diff < 60: # sec in min.
 103                 t = diff / 1
 104                 t_str = "%s sec ago" % int(math.ceil(t))
 105         elif diff < 60*60: # sec in hour
 106                 t = diff / (60)
 107                 t_str = "%s min ago" % int(math.ceil(t))
 108         elif diff < 60*60*24: # sec in day
 109                 t = diff / (60*60)
 110                 t_str = "%s hrs ago" % int(math.ceil(t))
 111         elif diff < 60*60*24*14: # sec in week
 112                 t = diff / (60*60*24)
 113                 t_str = "%s days ago" % int(math.ceil(t))
 114         elif diff <= 60*60*24*30: # approx sec in month
 115                 t = diff / (60*60*24*7)
 116                 t_str = "%s wks ago" % int(math.ceil(t))
 117         elif diff > 60*60*24*30: # approx sec in month
 118                 t = diff / (60*60*24*30)
 119                 t_str = "%s mnths ago" % int(t)
 120         return t_str
 121
 122 def getvalue(fb, path):
 123     indexes = path.split("/")
 124     values = fb
 125     for index in indexes:
 126         if index in values:
 127             values = values[index]
 128         else:
 129             return None
 130     return values
 131
 132 def nmap_port_status(status):
 133         ps = {}
 134         l_nmap = status.split()
 135         ports = l_nmap[4:]
 136
 137         continue_probe = False
 138         for port in ports:
 139                 results = port.split('/')
 140                 ps[results[0]] = results[1]
 141                 if results[1] == "open":
 142                         continue_probe = True
 143         return (ps, continue_probe)
 144
 145
 146 def nodegroup_display(node, fbdata, conf=None):
 147         node['current'] = get_current_state(fbdata)
 148
 149         s = fbdata['kernel_version'].split()
 150         if len(s) >=3:
 151                 node['kernel_version'] = s[2]
 152         else:
 153                 node['kernel_version'] = fbdata['kernel_version']
 154
 155         if '2.6' not in node['kernel_version']: node['kernel_version'] = ""
 156         if conf and not conf.nocolor:
 157             node['boot_state']  = color_boot_state(node['boot_state'])
 158             node['current']     = color_boot_state(node['current'])
 159
 160         if type(fbdata['plc_node_stats']['pcu_ids']) == type([]):
 161                 node['pcu'] = "PCU"
 162         node['lastupdate'] = diff_time(node['last_contact'])
 163
 164         pf = HistoryNodeRecord.get_by(hostname=node['hostname'])
 165         try:
 166                 node['lc'] = diff_time(pf.last_changed)
 167         except:
 168                 node['lc'] = "err"
 169
 170         ut = fbdata['comon_stats']['uptime']
 171         if ut != "null":
 172                 ut = diff_time(float(fbdata['comon_stats']['uptime']), False)
 173         node['uptime'] = ut
 174
 175         return "%(hostname)-42s %(boot_state)8s %(current)5s %(pcu)6s %(key)10.10s... %(kernel_version)35.35s %(lastupdate)12s, %(lc)s, %(uptime)s" % node
 176
 177 def datetime_fromstr(str):
 178         if '-' in str:
 179                 try:
 180                         tup = time.strptime(str, "%Y-%m-%d")
 181                 except:
 182                         tup = time.strptime(str, "%Y-%m-%d-%H:%M")
 183         elif '/' in str:
 184                 tup = time.strptime(str, "%m/%d/%Y")
 185         else:
 186                 tup = time.strptime(str, "%m/%d/%Y")
 187         ret = datetime.fromtimestamp(time.mktime(tup))
 188         return ret
 189
 190 def get_nodeset(config):
 191         """
 192                 Given the config values passed in, return the set of hostnames that it
 193                 evaluates to.
 194         """
 195         from monitor.wrapper import plccache
 196         api = plc.getAuthAPI()
 197         l_nodes = plccache.l_nodes
 198
 199         if config.nodelist:
 200                 f_nodes = util.file.getListFromFile(config.nodelist)
 201                 l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
 202         elif config.node:
 203                 f_nodes = [config.node]
 204                 l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
 205         elif config.nodegroup:
 206                 ng = api.GetNodeGroups({'name' : config.nodegroup})
 207                 l_nodes = api.GetNodes(ng[0]['node_ids'], ['hostname'])
 208         elif config.site:
 209                 site = api.GetSites(config.site)
 210                 if len(site) > 0:
 211                         l_nodes = api.GetNodes(site[0]['node_ids'], ['hostname'])
 212                 else:
 213                         print "No site returned for : %s" % config.site
 214                         return []
 215
 216         l_nodes = [node['hostname'] for node in l_nodes]
 217
 218         # perform this query after the above options, so that the filter above
 219         # does not break.
 220         if config.nodeselect:
 221                 fbquery = HistoryNodeRecord.query.all()
 222                 node_list = [ n.hostname for n in fbquery ]
 223                 l_nodes = query.node_select(config.nodeselect, node_list, None)
 224
 225         return l_nodes
 226
 227 def email_exception(content=None, title=None):
 228     import config
 229     from monitor.model import Message
 230     import traceback
 231     msg=traceback.format_exc()
 232     if content:
 233         msg = content + "\n" + msg
 234
 235     full_title = "exception running monitor"
 236     if title:
 237         full_title = "exception running monitor %s" % title
 238
 239     m=Message(full_title, msg, False)
 240     m.send([config.exception_email])
 241     return
 242
 243 def changed_lessthan(last_changed, days):
 244         if datetime.now() - last_changed <= timedelta(days):
 245                 #print "last changed less than %s" % timedelta(days)
 246                 return True
 247         else:
 248                 #print "last changed more than %s" % timedelta(days)
 249                 return False
 250
 251 def changed_greaterthan(last_changed, days):
 252         if last_changed is None:
 253                 return False
 254
 255         if datetime.now() - last_changed > timedelta(days):
 256                 #print "last changed more than %s" % timedelta(days)
 257                 return True
 258         else:
 259                 #print "last changed less than %s" % timedelta(days)
 260                 return False
 261
 262 def found_between(recent_actions, action_type, lower, upper):
 263         return found_before(recent_actions, action_type, upper) and found_within(recent_actions, action_type, lower)
 264
 265 def found_before(recent_actions, action_type, within):
 266         for action in recent_actions:
 267                 if action_type == action.action_type and \
 268                                 action.date_created < (datetime.now() - timedelta(within)):
 269                         return True
 270         return False
 271
 272 def found_within(recent_actions, action_type, within):
 273         for action in recent_actions:
 274                 #print "%s - %s %s > %s - %s (%s) ==> %s" % (action.loginbase, action.action_type, action.date_created, datetime.now(), timedelta(within), datetime.now()-timedelta(within), action.date_created > (datetime.now() - timedelta(within)) )
 275                 if action_type == action.action_type and \
 276                                 action.date_created > (datetime.now() - timedelta(within)):
 277                                 #datetime.now() - action.date_created < timedelta(within):
 278                         # recent action of given type.
 279                         #print "%s found_within %s in recent_actions from %s" % (action_type, timedelta(within), action.date_created)
 280                         return True
 281
 282         print "%s NOT found_within %s in recent_actions" % (action_type, timedelta(within) )
 283         return False
 284
 285
 286 #class Time:
 287 #    @classmethod
 288 #    def dt_to_ts(cls, dt):
 289 #        t = time.mktime(dt.timetuple())
 290 #        return t
 291 #
 292 #    @classmethod
 293 #    def ts_to_dt(cls, ts):
 294 #        d = datetime.fromtimestamp(ts)
 295 #        return d