X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=policy.py;h=aafa7d59b826ca7000936958d01fa7fe6b1b83cb;hb=6481ba179a2fa6ab22cbed7f37a8104b152f77b3;hp=4befbd94b3452fe6bed2c39bc76e29c4ab61abeb;hpb=c9d06f3b274ecbc092a0b3eb1f5ceb6c0f734aad;p=monitor.git diff --git a/policy.py b/policy.py index 4befbd9..aafa7d5 100755 --- a/policy.py +++ b/policy.py @@ -33,7 +33,7 @@ api = plc.getAuthAPI() def logic(): - plc.nodeBootState(host, 'rins') + plc.nodeBootState(host, 'reinstall') node_end_record(host) def main(hostnames, sitenames): @@ -47,6 +47,7 @@ def main(hostnames, sitenames): lb = plccache.plcdb_hn2lb[host] except: print "unknown host in plcdb_hn2lb %s" % host + email_exception(host) continue nodeblack = BlacklistRecord.get_by(hostname=host) @@ -64,35 +65,48 @@ def main(hostnames, sitenames): print "%s %s %s" % (i, nodehist.hostname, nodehist.status) if nodehist.status == 'good' and \ changed_lessthan(nodehist.last_changed, 1.0) and \ + found_within(recent_actions, 'down_notice', 7.0) and \ not found_within(recent_actions, 'online_notice', 0.5): + # NOTE: chronicly flapping nodes will not get 'online' notices + # since, they are never up long enough to be 'good'. + # NOTE: searching for down_notice proves that the node has + # gone through a 'down' state first, rather than just + # flapping through: good, offline, online, ... + # # NOTE: there is a narrow window in which this command must be - # evaluated, otherwise the notice will not go out. this is not ideal. - sitehist.sendMessage('online_notice', hostname=host, viart=False) + # evaluated, otherwise the notice will not go out. + # this is not ideal. + sitehist.sendMessage('online_notice', hostname=host, viart=False, saveact=True) print "send message for host %s online" % host - pass - if ( nodehist.status == 'offline' or nodehist.status == 'down' ) and \ + # if a node is offline and doesn't have a PCU, remind the user that they should have one. + #if not nodehist.haspcu and nodehist.status in ['offline', 'down'] and \ + # changed_greaterthan(nodehist.last_changed,1.0) and \ + # not found_within(recent_actions, 'pcumissing_notice', 7.0): + # + # sitehist.sendMessage('pcumissing_notice', hostname=host) + # print "send message for host %s pcumissing_notice" % host + + # if it is offline and HAS a PCU, then try to use it. + if nodehist.haspcu and nodehist.status in ['offline', 'down'] and \ changed_greaterthan(nodehist.last_changed,1.0) and \ - not found_between(recent_actions, 'first_try_reboot', 3.5, 1): + not found_between(recent_actions, 'try_reboot', 3.5, 1): sitehist.attemptReboot(host) - print "send message for host %s first_try_reboot" % host - pass + print "send message for host %s try_reboot" % host - # NOTE: non-intuitive is that found_between(first_try_reboot, 3.5, 1) + # NOTE: non-intuitive is that found_between(try_reboot, 3.5, 1) # will be false for a day after the above condition is satisfied - if ( nodehist.status == 'offline' or nodehist.status == 'down' ) and \ + if nodehist.haspcu and nodehist.status in ['offline', 'down'] and \ changed_greaterthan(nodehist.last_changed,1.5) and \ - found_between(recent_actions, 'first_try_reboot', 3.5, 1) and \ + found_between(recent_actions, 'try_reboot', 3.5, 1) and \ not found_within(recent_actions, 'pcufailed_notice', 3.5): - # found_within(recent_actions, 'first_try_reboot', 3.5) and \ # send pcu failure message #act = ActionRecord(**kwargs) sitehist.sendMessage('pcufailed_notice', hostname=host) print "send message for host %s PCU Failure" % host - pass if nodehist.status == 'monitordebug' and \ changed_greaterthan(nodehist.last_changed, 1) and \ @@ -111,9 +125,10 @@ def main(hostnames, sitenames): sitehist.sendMessage('down_notice', hostname=host) print "send message for host %s down" % host - pass node_count = node_count + 1 + print "time: ", time.strftime('%Y-%m-%d %H:%M:%S') + sys.stdout.flush() session.flush() for i,site in enumerate(sitenames): @@ -129,42 +144,47 @@ def main(hostnames, sitenames): recent_actions = sitehist.getRecentActions(loginbase=site) print "%s %s %s" % (i, sitehist.db.loginbase, sitehist.db.status) - if sitehist.db.status == 'down': - if not found_within(recent_actions, 'pause_penalty', 30) and \ - not found_within(recent_actions, 'increase_penalty', 7) and \ - changed_greaterthan(sitehist.db.last_changed, 7): - - # TODO: catch errors - sitehist.increasePenalty() - #sitehist.applyPenalty() - sitehist.sendMessage('increase_penalty') - - print "send message for site %s penalty increase" % site - - if sitehist.db.status == 'good': - # clear penalty - # NOTE: because 'all clear' should have an indefinite status, we - # have a boolean value rather than a 'recent action' - if sitehist.db.penalty_applied: - # send message that penalties are cleared. - sitehist.clearPenalty() - #sitehist.applyPenalty() - sitehist.sendMessage('clear_penalty') - sitehist.closeTicket() - - print "send message for site %s penalty cleared" % site - - # find all ticket ids for site ( could be on the site record? ) # determine if there are penalties within the last 30 days? # if so, add a 'pause_penalty' action. - if sitehist.db.message_id != 0 and sitehist.db.message_status == 'open' and sitehist.db.penalty_level > 0: + if sitehist.db.message_id != 0 and sitehist.db.message_status == 'open' and \ + sitehist.db.penalty_level > 0 and not found_within(recent_actions, 'pause_penalty', 30): # pause escalation print "Pausing penalties for %s" % site sitehist.pausePenalty() + else: + + if sitehist.db.status == 'down': + if not found_within(recent_actions, 'pause_penalty', 30) and \ + not found_within(recent_actions, 'increase_penalty', 7) and \ + changed_greaterthan(sitehist.db.last_changed, 7): + + # TODO: catch errors + sitehist.increasePenalty() + sitehist.applyPenalty() + sitehist.sendMessage('increase_penalty') + + print "send message for site %s penalty increase" % site + + if sitehist.db.status == 'good': + # clear penalty + # NOTE: because 'all clear' should have an indefinite status, we + # have a boolean value rather than a 'recent action' + if sitehist.db.penalty_applied: + # send message that penalties are cleared. + + sitehist.clearPenalty() + sitehist.applyPenalty() + sitehist.sendMessage('clear_penalty') + sitehist.closeTicket() + + print "send message for site %s penalty cleared" % site + site_count = site_count + 1 + print "time: ", time.strftime('%Y-%m-%d %H:%M:%S') + sys.stdout.flush() session.flush() session.flush() @@ -227,11 +247,12 @@ if __name__ == "__main__": try: main(hostnames, sitenames) + session.flush() except KeyboardInterrupt: print "Killed by interrupt" session.flush() sys.exit(0) except: - #email_exception() + email_exception() print traceback.print_exc(); print "fail all..."