4 from monitor import reboot
5 from monitor import util
6 from monitor import query
7 from monitor.wrapper import plc
9 from datetime import datetime, timedelta
10 from monitor.model import Message
11 from monitor.database.info import HistoryNodeRecord
13 esc = struct.pack('i', 27)
15 GREEN = esc + "[1;32m"
16 YELLOW = esc + "[1;33m"
18 LIGHTBLUE = esc + "[1;36m"
19 NORMAL = esc + "[0;39m"
22 return RED + str + NORMAL
25 return YELLOW + str + NORMAL
28 return GREEN + str + NORMAL
31 return LIGHTBLUE + str + NORMAL
34 return BLUE + str + NORMAL
36 def get_current_state(fbnode):
37 if 'observed_status' in fbnode:
38 state = fbnode['observed_status']
42 if l == "debug": l = 'dbg '
45 def color_pcu_state(fbnode):
46 if fbnode['plc_pcuid'] is None:
51 if 'plcnode' in fbnode and 'pcu_ids' in fbnode['plcnode'] and len(fbnode['plcnode']['pcu_ids']) > 0 :
52 values = reboot.get_pcu_values(fbnode['plcnode']['pcu_ids'][0])
57 if 'pcu' not in fbnode:
62 if 'reboot' in values:
64 if rb == 0 or rb == "0":
65 return fbnode['pcu'] + "OK "
66 elif "NetDown" == rb or "Not_Run" == rb:
67 return fbnode['pcu'] + "DOWN"
69 return fbnode['pcu'] + "BAD "
71 return fbnode['pcu'] + "BAD "
73 def color_boot_state(l):
74 if l == "dbg": return yellow("debg")
75 elif l == "dbg ": return yellow("debg")
76 elif l == "failboot": return yellow("debg")
77 elif l == "diag": return lightblue(l)
78 elif l == "diagnose": return lightblue(l)
79 elif l == "safeboot": return lightblue(l)
80 elif l == "disable": return red("dsbl")
81 elif l == "disabled": return red("dsbl")
82 elif l == "down": return red(l)
83 elif l == "boot": return green(l)
84 elif l == "rins": return blue(l)
85 elif l == "reinstall": return blue(l)
89 def diff_time(timestamp, abstime=True):
94 if type(timestamp) == type(datetime.now()):
95 timestamp = time.mktime(timestamp.timetuple())
97 diff = now - timestamp
100 # return the number of seconds as a difference from current time.
102 if diff < 60: # sec in min.
104 t_str = "%s sec ago" % int(math.ceil(t))
105 elif diff < 60*60: # sec in hour
107 t_str = "%s min ago" % int(math.ceil(t))
108 elif diff < 60*60*24: # sec in day
110 t_str = "%s hrs ago" % int(math.ceil(t))
111 elif diff < 60*60*24*14: # sec in week
112 t = diff / (60*60*24)
113 t_str = "%s days ago" % int(math.ceil(t))
114 elif diff <= 60*60*24*30: # approx sec in month
115 t = diff / (60*60*24*7)
116 t_str = "%s wks ago" % int(math.ceil(t))
117 elif diff > 60*60*24*30: # approx sec in month
118 t = diff / (60*60*24*30)
119 t_str = "%s mnths ago" % int(t)
122 def getvalue(fb, path):
123 indexes = path.split("/")
125 for index in indexes:
127 values = values[index]
132 def nmap_port_status(status):
134 l_nmap = status.split()
137 continue_probe = False
139 results = port.split('/')
140 ps[results[0]] = results[1]
141 if results[1] == "open":
142 continue_probe = True
143 return (ps, continue_probe)
146 def nodegroup_display(node, fbdata, conf=None):
147 node['current'] = get_current_state(fbdata)
149 s = fbdata['kernel_version'].split()
151 node['kernel_version'] = s[2]
153 node['kernel_version'] = fbdata['kernel_version']
155 if '2.6' not in node['kernel_version']: node['kernel_version'] = ""
156 if conf and not conf.nocolor:
157 node['boot_state'] = color_boot_state(node['boot_state'])
158 node['current'] = color_boot_state(node['current'])
160 if type(fbdata['plc_node_stats']['pcu_ids']) == type([]):
162 node['lastupdate'] = diff_time(node['last_contact'])
164 pf = HistoryNodeRecord.get_by(hostname=node['hostname'])
166 node['lc'] = diff_time(pf.last_changed)
170 ut = fbdata['comon_stats']['uptime']
172 ut = diff_time(float(fbdata['comon_stats']['uptime']), False)
175 return "%(hostname)-42s %(boot_state)8s %(current)5s %(pcu)6s %(key)10.10s... %(kernel_version)35.35s %(lastupdate)12s, %(lc)s, %(uptime)s" % node
177 def datetime_fromstr(str):
180 tup = time.strptime(str, "%Y-%m-%d")
182 tup = time.strptime(str, "%Y-%m-%d-%H:%M")
184 tup = time.strptime(str, "%m/%d/%Y")
186 tup = time.strptime(str, "%m/%d/%Y")
187 ret = datetime.fromtimestamp(time.mktime(tup))
190 def get_nodeset(config):
192 Given the config values passed in, return the set of hostnames that it
195 from monitor.wrapper import plccache
196 api = plc.getAuthAPI()
197 l_nodes = plccache.l_nodes
200 f_nodes = util.file.getListFromFile(config.nodelist)
201 l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
203 f_nodes = [config.node]
204 l_nodes = filter(lambda x: x['hostname'] in f_nodes, l_nodes)
205 elif config.nodegroup:
206 ng = api.GetNodeGroups({'name' : config.nodegroup})
207 l_nodes = api.GetNodes(ng[0]['node_ids'], ['hostname'])
209 site = api.GetSites(config.site)
210 l_nodes = api.GetNodes(site[0]['node_ids'], ['hostname'])
212 l_nodes = [node['hostname'] for node in l_nodes]
214 # perform this query after the above options, so that the filter above
216 if config.nodeselect:
217 fbquery = HistoryNodeRecord.query.all()
218 node_list = [ n.hostname for n in fbquery ]
219 l_nodes = query.node_select(config.nodeselect, node_list, None)
223 def email_exception(content=None, title=None):
225 from monitor.model import Message
227 msg=traceback.format_exc()
229 msg = content + "\n" + msg
231 full_title = "exception running monitor"
233 full_title = "exception running monitor %s" % title
235 m=Message(full_title, msg, False)
236 m.send([config.exception_email])
239 def changed_lessthan(last_changed, days):
240 if datetime.now() - last_changed <= timedelta(days):
241 #print "last changed less than %s" % timedelta(days)
244 #print "last changed more than %s" % timedelta(days)
247 def changed_greaterthan(last_changed, days):
248 if last_changed is None:
251 if datetime.now() - last_changed > timedelta(days):
252 #print "last changed more than %s" % timedelta(days)
255 #print "last changed less than %s" % timedelta(days)
258 def found_between(recent_actions, action_type, lower, upper):
259 return found_before(recent_actions, action_type, upper) and found_within(recent_actions, action_type, lower)
261 def found_before(recent_actions, action_type, within):
262 for action in recent_actions:
263 if action_type == action.action_type and \
264 action.date_created < (datetime.now() - timedelta(within)):
268 def found_within(recent_actions, action_type, within):
269 for action in recent_actions:
270 #print "%s - %s %s > %s - %s (%s) ==> %s" % (action.loginbase, action.action_type, action.date_created, datetime.now(), timedelta(within), datetime.now()-timedelta(within), action.date_created > (datetime.now() - timedelta(within)) )
271 if action_type == action.action_type and \
272 action.date_created > (datetime.now() - timedelta(within)):
273 #datetime.now() - action.date_created < timedelta(within):
274 # recent action of given type.
275 #print "%s found_within %s in recent_actions from %s" % (action_type, timedelta(within), action.date_created)
278 print "%s NOT found_within %s in recent_actions" % (action_type, timedelta(within) )
284 def dt_to_ts(cls, dt):
285 t = time.mktime(dt.timetuple())
289 def ts_to_dt(cls, ts):
290 d = datetime.fromtimestamp(ts)