changes for 3.0

[monitor.git] / grouprins.py

diff --git a/grouprins.py b/grouprins.py
index d859727..b85bbad 100755 (executable)
--- a/grouprins.py
+++ b/grouprins.py
@@ -67,14 +67,23 @@ class Reboot(object):
                        #pflags.resetRecentFlag('pcutried')
                        if not pflags.getRecentFlag('pcutried'):
                                try:
-                                       print "CALLING REBOOT!!!"
-                                       ret = reboot.reboot(host)
+                                       node_pf = PersistFlags(host, 1, db='node_persistflags')
+                                       if  node_pf.checkattr('last_change') and \
+                                               node_pf.last_change < time.time() - 60*60*24 and \
+                                               node_pf.checkattr('status') and \
+                                               node_pf.status != "good":
 
-                                       pflags.setRecentFlag('pcutried')
-                                       pflags.save()
-                                       return ret
+                                               print "CALLING REBOOT!!!"
+                                               ret = reboot.reboot(host)
+
+                                               pflags.setRecentFlag('pcutried')
+                                               pflags.save()
+                                               return ret
+                                       else:
+                                               return True
 
                                except Exception,e:
+                                       email_exception()
                                        print traceback.print_exc(); print e
 
                                        # NOTE: this failure could be an implementation issue on
@@ -87,16 +96,29 @@ class Reboot(object):
 
                        elif not pflags.getRecentFlag('pcu_rins_tried'):
                                try:
-                                       # set node to 'rins' boot state.
-                                       print "CALLING REBOOT +++ RINS"
-                                       plc.nodeBootState(host, 'rins')
-                                       ret = reboot.reboot(host)
-
-                                       pflags.setRecentFlag('pcu_rins_tried')
-                                       pflags.save()
-                                       return ret
+                                       # NOTE: check that the node has been down for at least a
+                                       # day before rebooting it.  this avoids false-reboots/rins
+                                       # from failed node detections. circa 03-12-09
+                                       node_pf = PersistFlags(host, 1, db='node_persistflags')
+                                       if  node_pf.checkattr('last_change') and \
+                                               node_pf.last_change < time.time() - 60*60*24 and \
+                                               node_pf.checkattr('status') and \
+                                               node_pf.status != "good":
+
+                                               # set node to 'rins' boot state.
+                                               print "CALLING REBOOT +++ RINS"
+                                               plc.nodeBootState(host, 'reinstall')
+                                               ret = reboot.reboot(host)
+
+                                               pflags.setRecentFlag('pcu_rins_tried')
+                                               pflags.save()
+                                               return ret
+
+                                       else:
+                                               return True
 
                                except Exception,e:
+                                       email_exception()
                                        print traceback.print_exc(); print e
 
                                        # NOTE: this failure could be an implementation issue on
@@ -140,6 +162,7 @@ class Reboot(object):
                try:
                        return mailmonitor.reboot(host)
                except Exception, e:
+                       email_exception(host)
                        print traceback.print_exc(); print e
                        return False
 
@@ -166,11 +189,11 @@ def set_node_to_rins(host, fb):
        node = api.GetNodes(host, ['boot_state', 'last_contact', 'last_updated', 'date_created'])
        record = {'observation' : node[0], 
                          'model' : 'USER_REQUEST', 
-                         'action' : 'api.UpdateNode(%s, {"boot_state" : "rins"})' % host, 
+                         'action' : 'api.UpdateNode(%s, {"boot_state" : "reinstall"})' % host, 
                          'time' : time.time()}
        l = Log(host, record)
 
-       ret = api.UpdateNode(host, {'boot_state' : 'rins'})
+       ret = api.UpdateNode(host, {'boot_state' : 'reinstall'})
        if ret:
                # it's nice to see the current status rather than the previous status on the console
                node = api.GetNodes(host)[0]
@@ -224,10 +247,15 @@ config = parsermodule.parse_args(parser)
 
 # COLLECT nodegroups, nodes and node lists
 if config.nodegroup:
-       ng = api.GetNodeGroups({'name' : config.nodegroup})
+       ng = api.GetNodeGroups({'groupname' : config.nodegroup})
        nodelist = api.GetNodes(ng[0]['node_ids'])
        hostnames = [ n['hostname'] for n in nodelist ]
 
+if config.site:
+       site = api.GetSites(config.site)
+       l_nodes = api.GetNodes(site[0]['node_ids'], ['hostname'])
+       hostnames = [ n['hostname'] for n in l_nodes ]
+
 if config.node or config.nodelist:
        if config.node: hostnames = [ config.node ] 
        else: hostnames = util.file.getListFromFile(config.nodelist)
@@ -257,6 +285,7 @@ for host in hostnames:
                try:
                        node = api.GetNodes(host)[0]
                except:
+                       email_exception()
                        print traceback.print_exc(); 
                        print "FAILED GETNODES for host: %s" % host
                        continue
@@ -281,6 +310,7 @@ for host in hostnames:
                                        # todo: send thank you, etc.
                                        mailmonitor.reboot(host)
                                except Exception, e:
+                                       email_exception()
                                        print traceback.print_exc(); print e
 
                                continue
@@ -339,10 +369,10 @@ for host in hostnames:
                                print "ALL METHODS OF RESTARTING %s FAILED" % host
                                args = {}
                                args['hostname'] = host
-                               m = PersistMessage(host, "ALL METHODS FAILED for %(hostname)s" % args,
-                                                                                        "CANNOT CONTACT", False, db='suspect_persistmessages')
-                               m.reset()
-                               m.send(['monitor-list@lists.planet-lab.org'])
+                               #m = PersistMessage(host, "ALL METHODS FAILED for %(hostname)s" % args,
+                               #                                                        "CANNOT CONTACT", False, db='suspect_persistmessages')
+                               #m.reset()
+                               #m.send(['monitor-list@lists.planet-lab.org'])
 
                        l = Log(host, record)
                        print l
@@ -351,6 +381,7 @@ for host in hostnames:
                print "Killed by interrupt"
                sys.exit(0)
        except:
+               email_exception()
                print traceback.print_exc();
                print "Continuing..."