7 from datetime import datetime,timedelta
9 from monitor.query import verify,query_to_dict,node_select
11 from monitor.common import *
13 from monitor import config
14 from monitor.wrapper import plc,plccache
15 from monitor.const import MINUP
16 from monitor.database.info.model import FindbadNodeRecord, HistoryNodeRecord
17 from monitor.database.dborm import mon_session as session
19 from monitor.model import *
21 api = plc.getAuthAPI()
30 l_plcnodes = plccache.l_nodes
31 l_nodes = get_nodeset(config)
33 checkAndRecordState(l_nodes, l_plcnodes)
37 def check_node_state(rec, node):
39 node_state = rec.observed_status
40 if rec.plc_node_stats:
41 print rec.plc_node_stats
42 boot_state = rec.plc_node_stats['boot_state']
43 run_level = rec.plc_node_stats['run_level']
44 last_contact = rec.plc_node_stats['last_contact']
45 node.plc_nodeid = rec.plc_node_stats['node_id']
46 node.plc_siteid = rec.plc_node_stats['site_id']
47 if len(rec.plc_node_stats['pcu_ids']) > 0:
52 boot_state = "unknown"
56 if boot_state == 'disable': boot_state = 'disabled'
57 if boot_state == 'diag' or boot_state == 'diagnose': boot_state = 'safeboot'
60 node.firewall = rec.firewall
62 # NOTE: 'DOWN' and 'DEBUG' are temporary states, so only need
63 # 'translations' into the node.status state
64 # 'BOOT' is a permanent state, but we want it to have a bit of
65 # hysteresis (less than 0.5 days)
66 #################################################################
67 # "Initialize" the findbad states into nodebad status if they are not already set
69 if node_state == 'DOWN':
70 if boot_state == 'disabled' and changed_lessthan(node.last_changed, 60) and \
71 node.status != 'disabled':
72 # NOTE: if changed less than 2 months, then we can allow this.
73 # otherwise, apply 'down' status after greater than 2 months (below).
75 print "changed status from %s to %s" % (node.status, boot_state)
76 node.status = boot_state
77 node.last_changed = datetime.now()
79 if node.status not in ['offline', 'down', 'disabled']:
80 print "changed status from %s to offline" % node.status
81 node.status = 'offline'
82 node.last_changed = datetime.now()
84 if node_state == 'DEBUG':
85 if boot_state != 'disabled' and boot_state != 'safeboot':
86 print "changed status from %s to failboot" % (node.status)
87 current_status = "failboot"
89 print "changed status from %s to %s" % (node.status, boot_state)
90 current_status = boot_state
92 if current_status != node.status and \
93 current_status in ['failboot', 'disabled', 'safeboot']:
95 node.status = current_status
96 node.last_changed = datetime.now()
98 if node_state == 'BOOT' and node.status != 'online' and node.status != 'good':
99 print "changed status from %s to online" % node.status
100 node.status = 'online'
101 node.last_changed = datetime.now()
103 #################################################################
104 # Switch temporary hystersis states into their 'firm' states.
105 # online -> good after half a day
106 # offline -> down after two days
107 # failboot -> down after 30 days
108 # safeboot -> failboot after 60 days
109 # disabled -> down after 60 days
111 if node.status == 'online' and changed_greaterthan(node.last_changed, 0.5):
112 print "changed status from %s to good" % node.status
114 # NOTE: do not reset last_changed, or you lose how long it's been up.
116 if node.status == 'offline' and changed_greaterthan(node.last_changed, 2):
117 print "changed status from %s to down" % node.status
119 # NOTE: do not reset last_changed, or you lose how long it's been down.
121 if node.status == 'failboot' and changed_greaterthan(node.last_changed, 30):
122 print "changed status from %s to down" % node.status
124 # NOTE: do not reset last_changed, or you lose how long it's been down.
126 if node.status == 'safeboot' and changed_greaterthan(node.last_changed, 60):
127 print "changed status from %s to down" % node.status
128 # NOTE: change an admin mode back into failboot after two months.
129 node.status = 'failboot'
130 node.last_changed = datetime.now()
132 # extreme cases of offline nodes
133 if ( boot_state == 'disabled' or last_contact == None ) and \
134 changed_greaterthan(node.last_changed, 2*30) and \
135 node.status != 'down':
136 print "changed status from %s to down" % node.status
138 node.last_changed = datetime.now()
140 def checkAndRecordState(l_nodes, l_plcnodes):
143 for nodename in l_nodes:
145 nodehist = HistoryNodeRecord.findby_or_create(hostname=nodename,
146 if_new_set={'status' : 'offline',
147 'last_changed' : datetime.now()})
148 nodehist.last_checked = datetime.now()
151 # Find the most recent record
152 noderec = FindbadNodeRecord.get_latest_by(hostname=nodename)
154 print "COULD NOT FIND %s" % nodename
157 print traceback.print_exc()
161 print "none object for %s"% nodename
164 check_node_state(noderec, nodehist)
167 print "%d %35s %s since(%s)" % (count, nodename, nodehist.status, diff_time(time.mktime(nodehist.last_changed.timetuple())))
169 # NOTE: this commits all pending operations to the DB. Do not remove.
174 if __name__ == '__main__':
175 from monitor import parser as parsermodule
176 parser = parsermodule.getParser(['nodesets'])
177 parser.set_defaults(filename=None, node=None, nodeselect=False, nodegroup=None, cachenodes=False)
178 parser = parsermodule.getParser(['defaults'], parser)
179 config = parsermodule.parse_args(parser)
183 except Exception, err:
185 print traceback.print_exc()
186 print "Exception: %s" % err