wait a day before rebooting a node.

[monitor.git] / grouprins.py

diff --git a/grouprins.py b/grouprins.py
index cfefc6a..abb6fa1 100755 (executable)
--- a/grouprins.py
+++ b/grouprins.py
@@ -67,14 +67,23 @@ class Reboot(object):
                        #pflags.resetRecentFlag('pcutried')
                        if not pflags.getRecentFlag('pcutried'):
                                try:
-                                       print "CALLING REBOOT!!!"
-                                       ret = reboot.reboot(host)
+                                       node_pf = PersistFlags(host, 1, db='node_persistflags')
+                                       if  node_pf.checkattr('last_change') and \
+                                               node_pf.last_change < time.time() - 60*60*24 and \
+                                               node_pf.checkattr('status') and \
+                                               node_pf.status != "good":
 
-                                       pflags.setRecentFlag('pcutried')
-                                       pflags.save()
-                                       return ret
+                                               print "CALLING REBOOT!!!"
+                                               ret = reboot.reboot(host)
+
+                                               pflags.setRecentFlag('pcutried')
+                                               pflags.save()
+                                               return ret
+                                       else:
+                                               return True
 
                                except Exception,e:
+                                       email_exception()
                                        print traceback.print_exc(); print e
 
                                        # NOTE: this failure could be an implementation issue on
@@ -87,16 +96,29 @@ class Reboot(object):
 
                        elif not pflags.getRecentFlag('pcu_rins_tried'):
                                try:
-                                       # set node to 'rins' boot state.
-                                       print "CALLING REBOOT +++ RINS"
-                                       plc.nodeBootState(host, 'rins')
-                                       ret = reboot.reboot(host)
-
-                                       pflags.setRecentFlag('pcu_rins_tried')
-                                       pflags.save()
-                                       return ret
+                                       # NOTE: check that the node has been down for at least a
+                                       # day before rebooting it.  this avoids false-reboots/rins
+                                       # from failed node detections. circa 03-12-09
+                                       node_pf = PersistFlags(host, 1, db='node_persistflags')
+                                       if  node_pf.checkattr('last_change') and \
+                                               node_pf.last_change < time.time() - 60*60*24 and \
+                                               node_pf.checkattr('status') and \
+                                               node_pf.status != "good":
+
+                                               # set node to 'rins' boot state.
+                                               print "CALLING REBOOT +++ RINS"
+                                               plc.nodeBootState(host, 'rins')
+                                               ret = reboot.reboot(host)
+
+                                               pflags.setRecentFlag('pcu_rins_tried')
+                                               pflags.save()
+                                               return ret
+
+                                       else:
+                                               return True
 
                                except Exception,e:
+                                       email_exception()
                                        print traceback.print_exc(); print e
 
                                        # NOTE: this failure could be an implementation issue on
@@ -140,6 +162,7 @@ class Reboot(object):
                try:
                        return mailmonitor.reboot(host)
                except Exception, e:
+                       email_exception(host)
                        print traceback.print_exc(); print e
                        return False
 
@@ -262,6 +285,7 @@ for host in hostnames:
                try:
                        node = api.GetNodes(host)[0]
                except:
+                       email_exception()
                        print traceback.print_exc(); 
                        print "FAILED GETNODES for host: %s" % host
                        continue
@@ -286,6 +310,7 @@ for host in hostnames:
                                        # todo: send thank you, etc.
                                        mailmonitor.reboot(host)
                                except Exception, e:
+                                       email_exception()
                                        print traceback.print_exc(); print e
 
                                continue
@@ -356,6 +381,7 @@ for host in hostnames:
                print "Killed by interrupt"
                sys.exit(0)
        except:
+               email_exception()
                print traceback.print_exc();
                print "Continuing..."