Only restart services which are enabled.

[mom.git] / swapmon.py

diff --git a/swapmon.py b/swapmon.py
index 8425b4d..e2215b2 100755 (executable)
--- a/swapmon.py
+++ b/swapmon.py
@@ -10,7 +10,7 @@
 # Faiyaz Ahmed <faiyaza@cs.princeton.edu>
 # Copyright (C) 2004-2006 The Trustees of Princeton University
 #
 # Faiyaz Ahmed <faiyaza@cs.princeton.edu>
 # Copyright (C) 2004-2006 The Trustees of Princeton University
 #

-# $Id: swapmon.py,v 1.5 2006/05/09 03:23:57 mlhuang Exp $
+# $Id$

 #
 
 import syslog
 #
 
 import syslog


@@ -22,10 +22,6 @@ import pickle
 import socket
 import time
 
 import socket
 import time
 

-# util-vserver/python/vserver.py allows us to control slices directly
-# from Python
-from vserver import VServer
-

 # bwlimit exports a few useful functions like run(), get_xid(), and get_slice()
 import bwlimit
 
 # bwlimit exports a few useful functions like run(), get_xid(), and get_slice()
 import bwlimit
 


@@ -35,8 +31,8 @@ from pl_mom import *
 # Defaults
 debug = False
 verbose = 0
 # Defaults
 debug = False
 verbose = 0

-datafile = "/var/lib/misc/swapmon.dat"
-
+DATAFILE = "/var/lib/misc/swapmon.dat"
+VERSION = "$Id$"

 # Seconds between process analysis
 period = 30
 
 # Seconds between process analysis
 period = 30
 


@@ -50,20 +46,12 @@ reset_thresh = 80
 # Swap utilization at which the machine is rebooted
 reboot_thresh = 95
 
 # Swap utilization at which the machine is rebooted
 reboot_thresh = 95
 

-# Time to wait before checking slice again after reset
-reset_timeout = 15
-
-# Number of strikes before killing (strike, strike, kill)
-kill_thresh = 2
-
-# Time to wait before removing slice from kill queue (probation) 
-kill_timeout = 120 
-

 # Don't email the same message more than once in the same emailtimeout interval
 email_timeout = 1800
 
 # Don't email the same message more than once in the same emailtimeout interval
 email_timeout = 1800
 

-# Minimum physical memory utilization to be considered the largest consumer
-min_thresh = 10
+# Physical size threshold to be considered a consumer.  Rationale is if there are no procs
+# with a size at least as large as this, then there is a slow leaker;  better to just reboot.
+rss_min = 150 * 1024 

 
 # System slices that should not be reset (regexps)
 system_slices = ['root', PLC_SLICE_PREFIX + '_']
 
 # System slices that should not be reset (regexps)
 system_slices = ['root', PLC_SLICE_PREFIX + '_']


@@ -83,26 +71,6 @@ statistics are not entirely accurate due to threading.
 %(date)s %(hostname)s reboot
 """.lstrip()
 
 %(date)s %(hostname)s reboot
 """.lstrip()
 

-# Message sent after a hog is reset
-reset_subject = "pl_mom reset slice %(slice)s on %(hostname)s"
-reset_body = \
-"""
-Sometime before %(date)s, swap space was
-nearly exhausted on %(hostname)s.
-
-Slice %(slice)s was reset since it was the largest consumer of
-physical memory at %(rss)s (%(percent)4.1f%%).
-
-Please reply to this message explaining the nature of your experiment,
-and what you are doing to address the problem.
-
-%(slice)s processes prior to reset:
-
-%(table)s
-
-%(date)s %(hostname)s reset %(slice)s
-""".lstrip()
-

 # Message sent to system slices that should not be reset
 alarm_subject = "pl_mom alarm slice %(slice)s on %(hostname)s"
 alarm_body = \
 # Message sent to system slices that should not be reset
 alarm_subject = "pl_mom alarm slice %(slice)s on %(hostname)s"
 alarm_body = \


@@ -111,8 +79,8 @@ Sometime before %(date)s, swap space was
 nearly exhausted on %(hostname)s.
 
 System slice %(slice)s was the largest consumer of physical memory at
 nearly exhausted on %(hostname)s.
 
 System slice %(slice)s was the largest consumer of physical memory at

-%(rss)s (%(percent)4.1f%%). It was not reset, but please verify its
-behavior.
+%(rss)s (%(percent)4.1f%%) (%(sz)s writable). It was not reset,
+but please verify its behavior.

 
 %(slice)s processes prior to alarm:
 
 
 %(slice)s processes prior to alarm:
 


@@ -129,7 +97,8 @@ Sometime before %(date)s, swap space was
 nearly exhausted on %(hostname)s.
 
 Slice %(slice)s was killed since it was the largest consumer of
 nearly exhausted on %(hostname)s.
 
 Slice %(slice)s was killed since it was the largest consumer of

-physical memory at %(rss)s (%(percent)4.1f%%) after repeated restarts.
+physical memory at %(rss)s (%(percent)4.1f%%) (%(sz)s writable)
+after repeated restarts.

 
 Please reply to this message explaining the nature of your experiment,
 and what you are doing to address the problem.
 
 Please reply to this message explaining the nature of your experiment,
 and what you are doing to address the problem.


@@ -141,112 +110,8 @@ and what you are doing to address the problem.
 %(date)s %(hostname)s reset %(slice)s
 """.lstrip()
 
 %(date)s %(hostname)s reset %(slice)s
 """.lstrip()
 

-
-
-class Reset:
-       """
-       Keeps track of state information for resets and kills
-
-       resettimeleft - timeout before checking for next reset
-       resetcount - number of strikes 
-       killtimeleft - time out before removing from kill queue
-       {kill,reset}mail - Time of last email
-       kill - State of kill.  If slice is already being killed, wait before retry.
-       """
-
-       def __init__(self,name):
-               self.name = name
-               self.resettimeleft = reset_timeout
-               self.resetcount = 0 
-               self.resetmail = 0
-               self.kill = False
-               self.killtimeleft = kill_timeout
-               self.killmail = 0
-
-       def __repr__(self):
-               return self.name
-       
-       def update(self):
-               # Count down for next check of reset slice.
-                       if self.resettimeleft > 0:
-                        self.resettimeleft -= 1
-                        if debug and verbose:
-                                       print "%s has %s seconds in probation" \
-                                       %(self.name, self.killtimeleft)
-               if self.killtimeleft > 0:
-                       # Count down kill probation timer (killtimeleft)
-                       self.killtimeleft -= 1
-                       if self.killtimeleft == 1:
-                               print "%s is out of probation" % self.name
-               else:
-                       # Once out of probation period (killtimeleft), remove strikes
-                       self.resetcount = 0
-                       self.kill = False
-
-
-       # Check to see if a slice needs to be killed.  If it has rules more than kill_thresh in 
-       # the probation period (kill_timeout) send an email, kill the slice.
-       def checkkill(self,params):
-               if self.killtimeleft > 0 and self.resetcount >= kill_thresh and \
-               self.kill == False:
-                       self.kill = True
-                       if debug:
-                                print kill_subject % params
-                                print kill_body % params
-                       try:
-                               pid = os.fork()
-                               if pid == 0:
-                                       print "Slice %s is being killed." % self.name   
-                                               vserver = VServer(self.name)
-                                               vserver.stop()
-                                       os._exit(0)
-                               else:
-                                       os.waitpid(pid,0)
-                       except Exception, err:
-                                       print "Warning: Exception received while killing slice %s: %s" \
-                                       % self.name, err
-                       if (time.time() - self.killmail) > email_timeout:
-                               slicemail(self.name, kill_subject % params, kill_body % params)
-                               print "Sending KILL email for slice %s" % self.name
-                               self.killmail = time.time() 
-                       return True
-               return False 
-
-       # Reset slice after checking to see if slice is out of timeout.
-       # Increment resetcount, check to see if larger than kill_thresh.
-       def reset(self, params):
-               # If its the first reset or if its been reset before
-               # and we are out of the reset timeout.
-               if self.resetcount == 0 or self.resettimeleft == 0:
-                       # Do we need to kill this slice?  Check history first.
-                       if self.checkkill(params):
-                               return
-                       # Update counters
-                       self.resetcount += 1
-                       self.killtimeleft = kill_timeout
-                       self.resettimeleft = reset_timeout
-                       print "%s has %s seconds to die and has been reset %s times" \
-                               %(self.name, self.resettimeleft, self.resetcount)
-                       if debug:
-                               print reset_subject % params
-                               print reset_body % params
-                       try:
-                               pid = os.fork()
-                               if pid == 0:
-                                               print "Resetting slice " + self.name 
-                                               vserver = VServer(self.name)
-                                               vserver.stop()
-                                               vserver.start(wait = False)
-                                               os._exit(0)
-                               else:
-                                       os.waitpid(pid,0)
-                       except Exception, err:
-                                       print "Warning: Exception received while resetting slice %s:" \
-                                       % self.name, err
-                       if (time.time() - self.resetmail) > email_timeout:
-                                       slicemail(self.name, reset_subject % params, reset_body % params)
-                               print "Sending Reset email for slice %s" % self.name
-                               self.resetmail = time.time() 
+def killsliverprocs(xid):
+    bwlimit.run("/usr/sbin/vkill -s 9 -c %s 0" % xid)    

 
 
 def usage():
 
 
 def usage():


@@ -265,26 +130,27 @@ Options:
         --system-slice=SLICE    System slice that should not be reset
         --status                Print memory usage statistics and exit
         -h, --help              This message
         --system-slice=SLICE    System slice that should not be reset
         --status                Print memory usage statistics and exit
         -h, --help              This message

-""".lstrip() % (sys.argv[0], debug, verbose, datafile, format_period(period))
+""".lstrip() % (sys.argv[0], debug, verbose, DATAFILE, format_period(period))

 
 def slicestat(names = None):
     """
     Get status of specified slices (if names is None or empty, all
 
 def slicestat(names = None):
     """
     Get status of specified slices (if names is None or empty, all

-    slices). vsize and rss are in KiB. Returns
-
+    slices). vsize, sz, and rss are in KiB. Returns
+    PID CONTEXT             VSZ    SZ   RSS %MEM CMD

     {xid: {'xid': slice_id,
     {xid: {'xid': slice_id,

-           'name': slice_name,
-           'procs': [{'pid': pid, 'xid': slice_id, 'user', username, 'cmd': command,
-                      'vsize': virtual_kib, 'rss': physical_kib,
+            'name': slice_name,
+            'procs': [{'pid': pid, 'xid': slice_id, 'cmd': command,
+                      'vsize': virtual_kib, 'sz': potential_kib, 'rss': physical_kib,

                       'pcpu': cpu_percent, 'pmem': mem_percent}]
                       'pcpu': cpu_percent, 'pmem': mem_percent}]

-           'vsize': total_virtual_kib,
-           'rss': total_physical_kib}}
+            'vsize': total_virtual_kib,
+            'sz': total_potential_kib,
+            'rss': total_physical_kib}}

     """
     
     # Mandatory fields. xid is a virtual field inserted by vps. Make
     # sure cmd is last so that it does not get truncated
     # automatically.
     """
     
     # Mandatory fields. xid is a virtual field inserted by vps. Make
     # sure cmd is last so that it does not get truncated
     # automatically.

-    fields = ['pid', 'xid', 'user', 'vsize', 'rss', 'pcpu', 'pmem', 'cmd']
+    fields = ['pid', 'xid', 'vsize', 'sz', 'rss', 'pmem', 'cmd']

 
     # vps inserts xid after pid in the output, but ps doesn't know
     # what the field means.
 
     # vps inserts xid after pid in the output, but ps doesn't know
     # what the field means.


@@ -296,7 +162,7 @@ def slicestat(names = None):
     # Eat the header line. vps depends on the header to figure out
     # which column is the PID column, so we can't just tell ps not to
     # print it.
     # Eat the header line. vps depends on the header to figure out
     # which column is the PID column, so we can't just tell ps not to
     # print it.

-    for line in bwlimit.run("/usr/sbin/vps -e -o " + ",".join(ps_fields))[1:]:
+    for line in bwlimit.run("/usr/sbin/vps -e -o " + ":16,".join(ps_fields))[1:]:

         # Chomp newline
         line = line.strip()
 
         # Chomp newline
         line = line.strip()
 


@@ -321,14 +187,16 @@ def slicestat(names = None):
                 except ValueError:
                     pass
 
                 except ValueError:
                     pass
 

-        # vps sometimes prints ERR instead of a context ID if it
+        # vps sometimes prints ERR or the name of the slice 
+            # instead of a context ID if it

         # cannot identify the context of an orphaned (usually dying)
         # process. Skip these processes.
         # cannot identify the context of an orphaned (usually dying)
         # process. Skip these processes.

-        if type(proc['xid']) != int:
+        if (type(proc['xid']) != int) or (type(proc['vsize']) !=int):

             continue
 
         # Assign (pl_)sshd processes to slice instead of root
         m = re.search(r"sshd: ([a-zA-Z_]+)", proc['cmd'])
             continue
 
         # Assign (pl_)sshd processes to slice instead of root
         m = re.search(r"sshd: ([a-zA-Z_]+)", proc['cmd'])

+

         if m is not None:
             xid = bwlimit.get_xid(m.group(1))
             if xid is not None:
         if m is not None:
             xid = bwlimit.get_xid(m.group(1))
             if xid is not None:


@@ -352,45 +220,50 @@ def slicestat(names = None):
         proc['rss'] += 12
 
         # Include additional page table overhead
         proc['rss'] += 12
 
         # Include additional page table overhead

-        if proc['vsize'] > 4096:
-            proc['rss'] += 4 * ((proc['vsize'] - 1) / 4096)
+        try:
+            if proc['vsize'] > 4096:
+                proc['rss'] += 4 * ((proc['vsize'] - 1) / 4096)
+        except: pass

 
         if slices.has_key(proc['xid']):
             slice = slices[proc['xid']]
         else:
 
         if slices.has_key(proc['xid']):
             slice = slices[proc['xid']]
         else:

-            slice = {'xid': proc['xid'], 'name': name, 'procs': [], 'vsize': 0, 'rss': 0}
+            slice = {'xid': proc['xid'], 'name': name, 'procs': [], 'vsize': 0, 'sz': 0, 'rss': 0}

 
         slice['procs'].append(proc)
         slice['vsize'] += proc['vsize']
 
         slice['procs'].append(proc)
         slice['vsize'] += proc['vsize']

+        slice['sz'] += proc['sz']

         slice['rss'] += proc['rss']
 
         slices[proc['xid']] = slice
         slice['rss'] += proc['rss']
 
         slices[proc['xid']] = slice

-
+       

     return slices
 
 def memtotal():
     """
     return slices
 
 def memtotal():
     """

-    Returns total physical memory on the system in KiB.
+    Returns total physical and swap memory on the system in KiB.

     """
     """

-
+    mem = 0
+    swap = 0

     meminfo = open("/proc/meminfo", "r")
     meminfo = open("/proc/meminfo", "r")

-    line = meminfo.readline()
+    for line in meminfo.readlines():
+        try:
+            (name, value, kb) = line.split()
+        except:
+            continue
+        if name == "MemTotal:": 
+            mem = int(value)
+        elif name == "SwapTotal:":
+            swap = int(value)

     meminfo.close()
     meminfo.close()

-    if line[0:8] == "MemTotal":
-        # MemTotal: 255396 kB
-        (name, value, kb) = line.split()
-        return int(value)
-
-    return 0
+    return (mem, swap)

 
 def swap_used():
     """
     Returns swap utilization on the system as a whole percentage (0-100).
     """
 
 def swap_used():
     """
     Returns swap utilization on the system as a whole percentage (0-100).
     """

-

     total_swap = 0
     total_used = 0
     total_swap = 0
     total_used = 0

-

     try:
         swaps = open("/proc/swaps", "r")
         # Eat header line
     try:
         swaps = open("/proc/swaps", "r")
         # Eat header line


@@ -404,33 +277,112 @@ def swap_used():
                 total_used += int(used)
             except ValueEror, err:
                 pass
                 total_used += int(used)
             except ValueEror, err:
                 pass

-    except (IOError, KeyError), err:
-        pass
+    except (IOError, KeyError), err:  pass

 
 

-    return 100 * total_used / total_swap
+    swapused = 100 * total_used / total_swap
+    if debug: print "%s percent swap used" % swapused
+    return swapused

 
 

-def summary(names = None, total_rss = memtotal()):
+def summary(slices = None, total_mem = None, total_swap = None):

     """
     Return a summary of memory usage by slice.
     """
     """
     Return a summary of memory usage by slice.
     """

-    slicelist = slicestat(names).values()
-    slicelist.sort(lambda a, b: b['rss'] - a['rss'])
+    if not slices:  slices = slicestat()
+    slicelist = slices.values()
+    slicelist.sort(lambda a, b: b['sz'] - a['sz'])
+    if total_mem is None or total_swap is None:
+        (total_mem, total_swap) = memtotal()

 
 

-    table = "%-20s%10s%24s\n\n" % ("Slice", "Processes", "Memory Usage")
+    table = "%-20s%10s%24s%24s\n\n" % ("Slice", "Processes", "Memory Usage", "Potential Usage")

     for slice in slicelist:
     for slice in slicelist:

-        table += "%-20s%10d%16s (%4.1f%%)\n" % \
+        table += "%-20s%10d%16s (%4.1f%%)%16s (%4.1f%%)\n" % \

                  (slice['name'], len(slice['procs']),
                   format_bytes(slice['rss'] * 1024, si = False),
                  (slice['name'], len(slice['procs']),
                   format_bytes(slice['rss'] * 1024, si = False),

-                  100. * slice['rss'] / total_rss)
-
+                  100. * slice['rss'] / total_mem,
+                  format_bytes(slice['sz'] * 1024, si = False),
+                  100. * slice['sz'] / (total_mem + total_swap))

     return table
 
     return table
 

+def formtable(slice, percent):
+    '''
+    Makes pretty message to email with human readable ps values.
+    '''
+    table = "%5s %10s %10s %10s %4s %4s %s\n\n" % \
+        ("PID", "VIRT", "SZ", "RES", '%CPU', '%MEM', 'COMMAND')
+    for proc in slice['procs']:
+        table += "%5s %10s %10s %10s %4.1f %s\n" % \
+            (proc['pid'],
+            format_bytes(proc['vsize'] * 1024, si = False),
+            format_bytes(proc['sz'] * 1024, si = False),
+            format_bytes(proc['rss'] * 1024, si = False),
+            proc['pmem'],
+            proc['cmd'])
+    
+    prettytable = {'hostname': socket.gethostname(),
+             'date': time.asctime(time.gmtime()) + " GMT",
+             'table': table,
+             'slice': slice['name'],
+             'rss': format_bytes(slice['rss'] * 1024, si = False),
+             'sz': format_bytes(slice['sz'] * 1024, si = False),
+             'percent': percent}
+    return prettytable
+
+def readdat():
+    '''
+    Return dictionary of vps (slicestat) from datfile left behind by OOM
+    before rebooting.  If none file, just grab the latest dict (slicestat)
+    and return that.  If dat file found, means we rebooted, send an email to 
+    pl_mom@pl.
+    '''
+    try:
+        f = open(DATAFILE, "r+")
+        if verbose:
+            print "Loading %s" % DATAFILE
+        (v, slices) = pickle.load(f)
+        f.close()
+        # Check version of data file
+        if v != VERSION:
+            print "Not using old version '%s' data file %s" % (v, DATAFILE)
+            raise Exception
+
+        params = {'hostname': socket.gethostname(),
+                  'date': time.asctime(time.gmtime()) + " GMT",
+                  'table': summary(slices, total_mem, total_swap)}
+        if debug:
+            print rebooted_subject % params
+            print rebooted_body % params
+        else:
+            slicemail(None, rebooted_subject % params, rebooted_body % params)
+
+        # Delete data file
+        os.unlink(DATAFILE)
+    except Exception:
+        slices = slicestat()
+
+    return slices
+
+
+def writedat(slices):
+    """
+    Write (slices) to pickled datfile.
+    """
+    if verbose:  print "Saving %s" % DATAFILE
+    f = open(DATAFILE, "w")
+    pickle.dump((VERSION, slices), f)
+    f.close()
+
+

 def main():
     # Defaults
 def main():
     # Defaults

-    global debug, verbose, datafile
-    global period, change_thresh, reset_thresh, reboot_thresh, min_thresh, system_slices
+    global debug, verbose, DATAFILE, VERSION 
+    global period, change_thresh, reset_thresh, reboot_thresh, rss_min, system_slices

     # All slices
     names = []
     # All slices
     names = []

+    timer = period
+    last_used = None
+    used = None
+    warned = []
+    emailed = {}

 
     try:
         longopts = ["debug", "verbose", "file=", "slice=", "status", "help"]
 
     try:
         longopts = ["debug", "verbose", "file=", "slice=", "status", "help"]


@@ -447,7 +399,7 @@ def main():
         elif opt == "-v" or opt == "--verbose":
             verbose += 1
         elif opt == "-f" or opt == "--file":
         elif opt == "-v" or opt == "--verbose":
             verbose += 1
         elif opt == "-f" or opt == "--file":

-            datafile = optval
+            DATAFILE = optval

         elif opt == "-s" or opt == "--slice":
             names.append(optval)
         elif opt == "-p" or opt == "--period":
         elif opt == "-s" or opt == "--slice":
             names.append(optval)
         elif opt == "-p" or opt == "--period":


@@ -459,11 +411,11 @@ def main():
         elif opt == "--reboot-thresh":
             reboot_thresh = int(optval)
         elif opt == "--min-thresh":
         elif opt == "--reboot-thresh":
             reboot_thresh = int(optval)
         elif opt == "--min-thresh":

-            min_thresh = int(optval)
+            rss_min = int(optval)

         elif opt == "--system-slice":
             system_slices.append(optval)
         elif opt == "--status":
         elif opt == "--system-slice":
             system_slices.append(optval)
         elif opt == "--status":

-            print summary(names)
+            print summary(slicestat(names))

             sys.exit(0)
         else:
             usage()
             sys.exit(0)
         else:
             usage()


@@ -480,135 +432,77 @@ def main():
         syslog.openlog("swapmon")
         sys.stdout = sys.stderr = Logger()
 
         syslog.openlog("swapmon")
         sys.stdout = sys.stderr = Logger()
 

-    # Get total physical memory
-    total_rss = memtotal()
-
-    try:
-        f = open(datafile, "r+")
-        if verbose:
-            print "Loading %s" % datafile
-        (version, slices) = pickle.load(f)
-        f.close()
-        # Check version of data file
-        if version != "$Id: swapmon.py,v 1.5 2006/05/09 03:23:57 mlhuang Exp $":
-            print "Not using old version '%s' data file %s" % (version, datafile)
-            raise Exception
-
-        params = {'hostname': socket.gethostname(),
-                  'date': time.asctime(time.gmtime()) + " GMT",
-                  'table': summary(total_rss)}
-
-        if debug:
-            print rebooted_subject % params
-            print rebooted_body % params
-        else:
-            slicemail(None, rebooted_subject % params, rebooted_body % params)
-
-        # Delete data file
-        os.unlink(datafile)
-    except Exception:
-        version = "$Id: swapmon.py,v 1.5 2006/05/09 03:23:57 mlhuang Exp $"
-        slices = {}
+    # Get total memory
+    (total_mem, total_swap) = memtotal()
+    slices = readdat()

 
     # Query process table every 30 seconds, or when a large change in
     # swap utilization is detected.
 
     # Query process table every 30 seconds, or when a large change in
     # swap utilization is detected.

-    timer = period
-    last_used = None
-    used = None
-
-    # System slices that we have warned but could not reset
-    warned = []
-
-    # Slices that were reset
-    resetlist = {}

 
     while True:
         used = swap_used()
 
     while True:
         used = swap_used()

-
-       for resetslice in resetlist.keys():
-               resetlist[resetslice].update()
-       
-        if last_used is None:
-            last_used = used
-       if verbose:
-            print "%d%% swap consumed" % used
-
+        if last_used is None:  last_used = used
+   

         if used >= reboot_thresh:
             # Dump slice state before rebooting
         if used >= reboot_thresh:
             # Dump slice state before rebooting

-            if verbose:
-                print "Saving %s" % datafile
-            f = open(datafile, "w")
-            pickle.dump((version, slices), f)
-            f.close()
-
+            writedat(slices)    

             # Goodbye, cruel world
             print "%d%% swap consumed, rebooting" % used
             # Goodbye, cruel world
             print "%d%% swap consumed, rebooting" % used

-            if not debug:
-                bwlimit.run("/bin/sync; /sbin/reboot -f")
-
+            if not debug:  bwlimit.run("/bin/sync; /sbin/reboot -f")

         elif used >= reset_thresh:
             # Try and find a hog
             slicelist = slices.values()
         elif used >= reset_thresh:
             # Try and find a hog
             slicelist = slices.values()

+            # Puts largest on top.

             slicelist.sort(lambda a, b: b['rss'] - a['rss'])
             for slice in slicelist:
             slicelist.sort(lambda a, b: b['rss'] - a['rss'])
             for slice in slicelist:

-                percent = 100. * slice['rss'] / total_rss
-
-                if percent < min_thresh:
-                    continue
-               
+                percent = 100. * slice['rss'] / total_mem
+                if slice['rss'] < rss_min: continue

                 print "%d%% swap consumed, slice %s is using %s (%d%%) of memory" % \
                 print "%d%% swap consumed, slice %s is using %s (%d%%) of memory" % \

-                      (used,
-                       slice['name'],
-                       format_bytes(slice['rss'] * 1024, si = False),
-                       percent)
-
+                    (used,
+                    slice['name'],
+                    format_bytes(slice['rss'] * 1024, si = False),
+                    percent)

                 slice['procs'].sort(lambda a, b: b['rss'] - a['rss'])
                 slice['procs'].sort(lambda a, b: b['rss'] - a['rss'])

-
-                table = "%5s %10s %10s %4s %4s %s\n\n" % ("PID", "VIRT", "RES", '%CPU', '%MEM', 'COMMAND')
-                for proc in slice['procs']:
-                    table += "%5s %10s %10s %4.1f %4.1f %s\n" % \
-                             (proc['pid'],
-                              format_bytes(proc['vsize'] * 1024, si = False),
-                              format_bytes(proc['rss'] * 1024, si = False),
-                              proc['pcpu'], proc['pmem'], proc['cmd'])
-
-                params = {'hostname': socket.gethostname(),
-                          'date': time.asctime(time.gmtime()) + " GMT",
-                          'table': table,
-                          'slice': slice['name'],
-                          'rss': format_bytes(slice['rss'] * 1024, si = False),
-                          'percent': percent}
-
+                # Make a pretty table.
+                params = formtable(slice, percent)

                 # Match slice name against system slice patterns
                 # Match slice name against system slice patterns

-                is_system_slice = filter(None, [re.match(pattern, slice['name']) for pattern in system_slices])
-
-                if is_system_slice and params['rss'] > 100:
-                    # Do not reset system slices, just warn once
+                is_system_slice = filter(None, 
+                    [re.match(pattern, slice['name']) for pattern in system_slices])
+    
+                # Do not reset system slices, just warn once
+                if is_system_slice: 

                     if slice['name'] not in warned:
                         warned.append(slice['name'])
                     if slice['name'] not in warned:
                         warned.append(slice['name'])

+                        print "Warning slice " + slice['name']

                         if debug:
                             print alarm_subject % params
                             print alarm_body % params
                         else:
                         if debug:
                             print alarm_subject % params
                             print alarm_body % params
                         else:

-                            print "Warning slice " + slice['name']
-                            slicemail(slice['name'], alarm_subject % params, alarm_body % params)
+                            slicemail(slice['name'], alarm_subject % params, 
+                              alarm_body % params)

                 else:
                 else:

-                       # Reset slice
-                       if not resetlist.has_key(slice['name']):
-                               resetlist[slice['name']] = Reset(slice['name'])
-                        resetlist[slice['name']].reset(params)
-
-        elif timer <= 0 or used >= (last_used + change_thresh):
+                    # Reset slice
+                    if not debug:
+                        if emailed.get(slice['name'], (time.time() + email_timeout + 1)) > (time.time() + email_timeout): 
+                            slicemail(slice['name'], kill_subject % params, kill_body % params)
+                            emailed[slice['name']] = time.time()
+                    else:
+                        print kill_subject % params
+                        print kill_body % params
+                    print "Killing procs in %s" % slice['name']
+                    killsliverprocs(slice['xid'])
+
+        # wait period before recalculating swap.  If in danger, recalc.
+        if timer <= 0 or used >= (last_used + change_thresh):

             if used >= (last_used + change_thresh):
                 print "%d%% swap consumed, %d%% in last %d seconds" % \
             if used >= (last_used + change_thresh):
                 print "%d%% swap consumed, %d%% in last %d seconds" % \

-                      (used, used - last_used, period - timer)
+                    (used, used - last_used, period - timer)

             # Get slice state
             slices = slicestat(names)
             # Reset timer
             timer = period
             # Keep track of large changes in swap utilization
             last_used = used
             # Get slice state
             slices = slicestat(names)
             # Reset timer
             timer = period
             # Keep track of large changes in swap utilization
             last_used = used

-

         timer -= 1
         time.sleep(1)
 
         timer -= 1
         time.sleep(1)