# Faiyaz Ahmed <faiyaza@cs.princeton.edu>
# Copyright (C) 2004-2006 The Trustees of Princeton University
#
-# $Id: swapmon.py,v 1.9 2006/07/19 19:40:55 faiyaza Exp $
+# $Id: swapmon.py,v 1.10 2006/08/16 16:18:45 faiyaza Exp $
#
import syslog
# Don't email the same message more than once in the same emailtimeout interval
email_timeout = 1800
-# Minimum physical memory utilization to be considered the largest consumer
-min_thresh = 10
+# Physical size threshold to be considered a consumer. Rationale is if there are no procs
+# with a size at least as large as this, then there is a slow leaker; better to just reboot.
+rss_min = 150 * 1024
# System slices that should not be reset (regexps)
system_slices = ['root', PLC_SLICE_PREFIX + '_']
self.resettimeleft = reset_timeout
self.resetcount = 0
self.resetmail = 0
- self.kill = False
self.killtimeleft = kill_timeout
self.killmail = 0
else:
# Once out of probation period (killtimeleft), remove strikes
self.resetcount = 0
- self.kill = False
- # Check to see if a slice needs to be killed. If it has rules more than kill_thresh in
- # the probation period (kill_timeout) send an email, kill the slice.
+ # Check to see if a slice needs to be killed. If it has been killed more
+ # than kill_thresh in the probation period (kill_timeout) send an email, kill the slice.
def checkkill(self,params):
- if self.killtimeleft > 0 and self.resetcount >= kill_thresh and \
- self.kill == False:
- self.kill = True
+ if self.killtimeleft > 0 and self.resetcount >= kill_thresh:
if debug:
print kill_subject % params
print kill_body % params
# Reset slice after checking to see if slice is out of timeout.
# Increment resetcount, check to see if larger than kill_thresh.
def reset(self, params):
- # If its the first reset or if its been reset before
+ # If its the first reset (came back after kill)
+ # or if its been reset before
# and we are out of the reset timeout.
if self.resetcount == 0 or self.resettimeleft == 0:
# Do we need to kill this slice? Check history first.
def main():
# Defaults
global debug, verbose, datafile
- global period, change_thresh, reset_thresh, reboot_thresh, min_thresh, system_slices
+ global period, change_thresh, reset_thresh, reboot_thresh, rss_min, system_slices
# All slices
names = []
reset_thresh = int(optval)
elif opt == "--reboot-thresh":
reboot_thresh = int(optval)
- elif opt == "--min-thresh":
- min_thresh = int(optval)
elif opt == "--system-slice":
system_slices.append(optval)
elif opt == "--status":
(version, slices) = pickle.load(f)
f.close()
# Check version of data file
- if version != "$Id: swapmon.py,v 1.9 2006/07/19 19:40:55 faiyaza Exp $":
+ if version != "$Id: swapmon.py,v 1.10 2006/08/16 16:18:45 faiyaza Exp $":
print "Not using old version '%s' data file %s" % (version, datafile)
raise Exception
# Delete data file
os.unlink(datafile)
except Exception:
- version = "$Id: swapmon.py,v 1.9 2006/07/19 19:40:55 faiyaza Exp $"
+ version = "$Id: swapmon.py,v 1.10 2006/08/16 16:18:45 faiyaza Exp $"
slices = {}
# Query process table every 30 seconds, or when a large change in
if last_used is None:
last_used = used
+
if verbose:
print "%d%% swap consumed" % used
bwlimit.run("/bin/sync; /sbin/reboot -f")
elif used >= reset_thresh:
+ if debug:
+ print "Memory used = %s" %(used)
# Try and find a hog
slicelist = slices.values()
slicelist.sort(lambda a, b: b['rss'] - a['rss'])
for slice in slicelist:
percent = 100. * slice['rss'] / total_rss
- if percent < min_thresh:
+ if slice['rss'] < rss_min:
continue
print "%d%% swap consumed, slice %s is using %s (%d%%) of memory" % \
if not resetlist.has_key(slice['name']):
resetlist[slice['name']] = Reset(slice['name'])
resetlist[slice['name']].reset(params)
+ slices = slicestat(names)
- elif timer <= 0 or used >= (last_used + change_thresh):
+ if timer <= 0 or used >= (last_used + change_thresh):
if used >= (last_used + change_thresh):
print "%d%% swap consumed, %d%% in last %d seconds" % \
(used, used - last_used, period - timer)