wait a day before rebooting a node.

[monitor.git] / grouprins.py

diff --git a/grouprins.py b/grouprins.py
index d859727..abb6fa1 100755 (executable)
--- a/grouprins.py
+++ b/grouprins.py
@@ -67,14 +67,23 @@ class Reboot(object):
                        #pflags.resetRecentFlag('pcutried')
                        if not pflags.getRecentFlag('pcutried'):
                                try:
-                                       print "CALLING REBOOT!!!"
-                                       ret = reboot.reboot(host)
+                                       node_pf = PersistFlags(host, 1, db='node_persistflags')
+                                       if  node_pf.checkattr('last_change') and \
+                                               node_pf.last_change < time.time() - 60*60*24 and \
+                                               node_pf.checkattr('status') and \
+                                               node_pf.status != "good":
 
-                                       pflags.setRecentFlag('pcutried')
-                                       pflags.save()
-                                       return ret
+                                               print "CALLING REBOOT!!!"
+                                               ret = reboot.reboot(host)
+
+                                               pflags.setRecentFlag('pcutried')
+                                               pflags.save()
+                                               return ret
+                                       else:
+                                               return True
 
                                except Exception,e:
+                                       email_exception()
                                        print traceback.print_exc(); print e
 
                                        # NOTE: this failure could be an implementation issue on
@@ -87,16 +96,29 @@ class Reboot(object):
 
                        elif not pflags.getRecentFlag('pcu_rins_tried'):
                                try:
-                                       # set node to 'rins' boot state.
-                                       print "CALLING REBOOT +++ RINS"
-                                       plc.nodeBootState(host, 'rins')
-                                       ret = reboot.reboot(host)
-
-                                       pflags.setRecentFlag('pcu_rins_tried')
-                                       pflags.save()
-                                       return ret
+                                       # NOTE: check that the node has been down for at least a
+                                       # day before rebooting it.  this avoids false-reboots/rins
+                                       # from failed node detections. circa 03-12-09
+                                       node_pf = PersistFlags(host, 1, db='node_persistflags')
+                                       if  node_pf.checkattr('last_change') and \
+                                               node_pf.last_change < time.time() - 60*60*24 and \
+                                               node_pf.checkattr('status') and \
+                                               node_pf.status != "good":
+
+                                               # set node to 'rins' boot state.
+                                               print "CALLING REBOOT +++ RINS"
+                                               plc.nodeBootState(host, 'rins')
+                                               ret = reboot.reboot(host)
+
+                                               pflags.setRecentFlag('pcu_rins_tried')
+                                               pflags.save()
+                                               return ret
+
+                                       else:
+                                               return True
 
                                except Exception,e:
+                                       email_exception()
                                        print traceback.print_exc(); print e
 
                                        # NOTE: this failure could be an implementation issue on
@@ -140,6 +162,7 @@ class Reboot(object):
                try:
                        return mailmonitor.reboot(host)
                except Exception, e:
+                       email_exception(host)
                        print traceback.print_exc(); print e
                        return False
 
@@ -228,6 +251,11 @@ if config.nodegroup:
        nodelist = api.GetNodes(ng[0]['node_ids'])
        hostnames = [ n['hostname'] for n in nodelist ]
 
+if config.site:
+       site = api.GetSites(config.site)
+       l_nodes = api.GetNodes(site[0]['node_ids'], ['hostname'])
+       hostnames = [ n['hostname'] for n in l_nodes ]
+
 if config.node or config.nodelist:
        if config.node: hostnames = [ config.node ] 
        else: hostnames = util.file.getListFromFile(config.nodelist)
@@ -257,6 +285,7 @@ for host in hostnames:
                try:
                        node = api.GetNodes(host)[0]
                except:
+                       email_exception()
                        print traceback.print_exc(); 
                        print "FAILED GETNODES for host: %s" % host
                        continue
@@ -281,6 +310,7 @@ for host in hostnames:
                                        # todo: send thank you, etc.
                                        mailmonitor.reboot(host)
                                except Exception, e:
+                                       email_exception()
                                        print traceback.print_exc(); print e
 
                                continue
@@ -339,10 +369,10 @@ for host in hostnames:
                                print "ALL METHODS OF RESTARTING %s FAILED" % host
                                args = {}
                                args['hostname'] = host
-                               m = PersistMessage(host, "ALL METHODS FAILED for %(hostname)s" % args,
-                                                                                        "CANNOT CONTACT", False, db='suspect_persistmessages')
-                               m.reset()
-                               m.send(['monitor-list@lists.planet-lab.org'])
+                               #m = PersistMessage(host, "ALL METHODS FAILED for %(hostname)s" % args,
+                               #                                                        "CANNOT CONTACT", False, db='suspect_persistmessages')
+                               #m.reset()
+                               #m.send(['monitor-list@lists.planet-lab.org'])
 
                        l = Log(host, record)
                        print l
@@ -351,6 +381,7 @@ for host in hostnames:
                print "Killed by interrupt"
                sys.exit(0)
        except:
+               email_exception()
                print traceback.print_exc();
                print "Continuing..."