* Added hard limit of 200MB to be considered a hog.

* If you've been reset more than 3 times, kill until you behave.
* Update slice statistics after every slice reset/kill.

diff --git a/swapmon.py b/swapmon.py
index 02a01ca..5f6afd8 100755 (executable)
--- a/swapmon.py
+++ b/swapmon.py
@@ -10,7 +10,7 @@
 # Faiyaz Ahmed <faiyaza@cs.princeton.edu>
 # Copyright (C) 2004-2006 The Trustees of Princeton University
 #
-# $Id: swapmon.py,v 1.5 2006/05/09 03:23:57 mlhuang Exp $
+# $Id: swapmon.py,v 1.9 2006/07/19 19:40:55 faiyaza Exp $
 #
 
 import syslog
@@ -62,8 +62,9 @@ kill_timeout = 120
 # Don't email the same message more than once in the same emailtimeout interval
 email_timeout = 1800
 
-# Minimum physical memory utilization to be considered the largest consumer
-min_thresh = 10
+# Physical size threshold to be considered a consumer.  Rationale is if there are no procs
+# with a size at least as large as this, then there is a slow leaker;  better to just reboot.
+rss_min = 150 * 1024 
 
 # System slices that should not be reset (regexps)
 system_slices = ['root', PLC_SLICE_PREFIX + '_']
@@ -159,7 +160,6 @@ class Reset:
                self.resettimeleft = reset_timeout
                self.resetcount = 0 
                self.resetmail = 0
-               self.kill = False
                self.killtimeleft = kill_timeout
                self.killmail = 0
 
@@ -181,15 +181,12 @@ class Reset:
                else:
                        # Once out of probation period (killtimeleft), remove strikes
                        self.resetcount = 0
-                       self.kill = False
 
 
-       # Check to see if a slice needs to be killed.  If it has rules more than kill_thresh in 
-       # the probation period (kill_timeout) send an email, kill the slice.
+       # Check to see if a slice needs to be killed.  If it has been killed more 
+       # than kill_thresh in the probation period (kill_timeout) send an email, kill the slice.
        def checkkill(self,params):
-               if self.killtimeleft > 0 and self.resetcount >= kill_thresh and \
-               self.kill == False:
-                       self.kill = True
+               if self.killtimeleft > 0 and self.resetcount >= kill_thresh:
                        if debug:
                                 print kill_subject % params
                                 print kill_body % params
@@ -215,7 +212,8 @@ class Reset:
        # Reset slice after checking to see if slice is out of timeout.
        # Increment resetcount, check to see if larger than kill_thresh.
        def reset(self, params):
-               # If its the first reset or if its been reset before
+               # If its the first reset (came back after kill)
+               # or if its been reset before
                # and we are out of the reset timeout.
                if self.resetcount == 0 or self.resettimeleft == 0:
                        # Do we need to kill this slice?  Check history first.
@@ -428,7 +426,7 @@ def summary(names = None, total_rss = memtotal()):
 def main():
     # Defaults
     global debug, verbose, datafile
-    global period, change_thresh, reset_thresh, reboot_thresh, min_thresh, system_slices
+    global period, change_thresh, reset_thresh, reboot_thresh, rss_min, system_slices
     # All slices
     names = []
 
@@ -458,8 +456,6 @@ def main():
             reset_thresh = int(optval)
         elif opt == "--reboot-thresh":
             reboot_thresh = int(optval)
-        elif opt == "--min-thresh":
-            min_thresh = int(optval)
         elif opt == "--system-slice":
             system_slices.append(optval)
         elif opt == "--status":
@@ -490,7 +486,7 @@ def main():
         (version, slices) = pickle.load(f)
         f.close()
         # Check version of data file
-        if version != "$Id: swapmon.py,v 1.5 2006/05/09 03:23:57 mlhuang Exp $":
+        if version != "$Id: swapmon.py,v 1.9 2006/07/19 19:40:55 faiyaza Exp $":
             print "Not using old version '%s' data file %s" % (version, datafile)
             raise Exception
 
@@ -507,7 +503,7 @@ def main():
         # Delete data file
         os.unlink(datafile)
     except Exception:
-        version = "$Id: swapmon.py,v 1.5 2006/05/09 03:23:57 mlhuang Exp $"
+        version = "$Id: swapmon.py,v 1.9 2006/07/19 19:40:55 faiyaza Exp $"
         slices = {}
 
     # Query process table every 30 seconds, or when a large change in
@@ -530,6 +526,7 @@ def main():
        
         if last_used is None:
             last_used = used
+
        if verbose:
             print "%d%% swap consumed" % used
 
@@ -547,13 +544,15 @@ def main():
                 bwlimit.run("/bin/sync; /sbin/reboot -f")
 
         elif used >= reset_thresh:
+           if debug:
+               print "Memory used = %s" %(used)
             # Try and find a hog
             slicelist = slices.values()
             slicelist.sort(lambda a, b: b['rss'] - a['rss'])
             for slice in slicelist:
                 percent = 100. * slice['rss'] / total_rss
 
-                if percent < min_thresh:
+                if slice['rss'] < rss_min:
                     continue
                
                 print "%d%% swap consumed, slice %s is using %s (%d%%) of memory" % \
@@ -597,8 +596,9 @@ def main():
                        if not resetlist.has_key(slice['name']):
                                resetlist[slice['name']] = Reset(slice['name'])
                         resetlist[slice['name']].reset(params)
+                       slices = slicestat(names)
 
-        elif timer <= 0 or used >= (last_used + change_thresh):
+        if timer <= 0 or used >= (last_used + change_thresh):
             if used >= (last_used + change_thresh):
                 print "%d%% swap consumed, %d%% in last %d seconds" % \
                       (used, used - last_used, period - timer)