Just kill the processes and stop screwing with vservers.

[mom.git] / swapmon.py

diff --git a/swapmon.py b/swapmon.py
index 655ff47..ead402e 100755 (executable)
--- a/swapmon.py
+++ b/swapmon.py
@@ -7,9 +7,10 @@
 #
 # Mark Huang <mlhuang@cs.princeton.edu>
 # Andy Bavier <acb@cs.princeton.edu>
 #
 # Mark Huang <mlhuang@cs.princeton.edu>
 # Andy Bavier <acb@cs.princeton.edu>

+# Faiyaz Ahmed <faiyaza@cs.princeton.edu>

 # Copyright (C) 2004-2006 The Trustees of Princeton University
 #
 # Copyright (C) 2004-2006 The Trustees of Princeton University
 #

-# $Id: swapmon.py,v 1.3 2006/05/01 18:28:01 mlhuang Exp $
+# $Id$

 #
 
 import syslog
 #
 
 import syslog


@@ -34,8 +35,8 @@ from pl_mom import *
 # Defaults
 debug = False
 verbose = 0
 # Defaults
 debug = False
 verbose = 0

-datafile = "/var/lib/misc/swapmon.dat"
-
+DATAFILE = "/var/lib/misc/swapmon.dat"
+VERSION = "$Id$"

 # Seconds between process analysis
 period = 30
 
 # Seconds between process analysis
 period = 30
 


@@ -44,13 +45,20 @@ period = 30
 change_thresh = 5
 
 # Swap utilization at which the largest consumer of physical memory is reset
 change_thresh = 5
 
 # Swap utilization at which the largest consumer of physical memory is reset

-reset_thresh = 85
+reset_thresh = 80

 
 # Swap utilization at which the machine is rebooted
 reboot_thresh = 95
 
 
 # Swap utilization at which the machine is rebooted
 reboot_thresh = 95
 

-# Minimum physical memory utilization to be considered the largest consumer
-min_thresh = 10
+# Time to wait before checking slice again after reset
+reset_timeout = 25
+
+# Don't email the same message more than once in the same emailtimeout interval
+email_timeout = 1800
+
+# Physical size threshold to be considered a consumer.  Rationale is if there are no procs
+# with a size at least as large as this, then there is a slow leaker;  better to just reboot.
+rss_min = 150 * 1024 

 
 # System slices that should not be reset (regexps)
 system_slices = ['root', PLC_SLICE_PREFIX + '_']
 
 # System slices that should not be reset (regexps)
 system_slices = ['root', PLC_SLICE_PREFIX + '_']


@@ -78,11 +86,13 @@ Sometime before %(date)s, swap space was
 nearly exhausted on %(hostname)s.
 
 Slice %(slice)s was reset since it was the largest consumer of
 nearly exhausted on %(hostname)s.
 
 Slice %(slice)s was reset since it was the largest consumer of

-physical memory at %(rss)s (%(percent)4.1f%%).
+physical memory at %(rss)s (%(percent)4.1f%%) (%(sz)s writable).

 
 Please reply to this message explaining the nature of your experiment,
 and what you are doing to address the problem.
 
 
 Please reply to this message explaining the nature of your experiment,
 and what you are doing to address the problem.
 

+http://summer.cs.princeton.edu/status/tabulator.cgi?table=slices/table_%(slice)s
+

 %(slice)s processes prior to reset:
 
 %(table)s
 %(slice)s processes prior to reset:
 
 %(table)s


@@ -98,8 +108,8 @@ Sometime before %(date)s, swap space was
 nearly exhausted on %(hostname)s.
 
 System slice %(slice)s was the largest consumer of physical memory at
 nearly exhausted on %(hostname)s.
 
 System slice %(slice)s was the largest consumer of physical memory at

-%(rss)s (%(percent)4.1f%%). It was not reset, but please verify its
-behavior.
+%(rss)s (%(percent)4.1f%%) (%(sz)s writable). It was not reset,
+but please verify its behavior.

 
 %(slice)s processes prior to alarm:
 
 
 %(slice)s processes prior to alarm:
 


@@ -108,6 +118,74 @@ behavior.
 %(date)s %(hostname)s alarm %(slice)s
 """.lstrip()
 
 %(date)s %(hostname)s alarm %(slice)s
 """.lstrip()
 

+# Message sent after a slice has been killed
+kill_subject = "pl_mom killed slice %(slice)s on %(hostname)s"
+kill_body = \
+"""
+Sometime before %(date)s, swap space was
+nearly exhausted on %(hostname)s.
+
+Slice %(slice)s was killed since it was the largest consumer of
+physical memory at %(rss)s (%(percent)4.1f%%) (%(sz)s writable)
+after repeated restarts.
+
+Please reply to this message explaining the nature of your experiment,
+and what you are doing to address the problem.
+
+%(slice)s processes prior to reset:
+
+%(table)s
+
+%(date)s %(hostname)s reset %(slice)s
+""".lstrip()
+
+
+
+class Reset:
+    """
+    Keeps track of state information for resets and kills
+
+    resettimeleft - timeout before checking for next reset
+    resetcount - number of strikes 
+    killtimeleft - time out before removing from kill queue
+    {kill,reset}mail - Time of last email
+    kill - State of kill.  If slice is already being killed, wait before retry.
+    """
+
+    def __init__(self,name):
+        self.name = name
+        self.resetmail = 0
+        self.killmail = 0
+
+    def __repr__(self):
+        return self.name
+    
+    # Reset slice 
+    def reset(self, params):
+        if self.resetcount == 0 or self.resettimeleft == 0:
+            print "%s has %s seconds to die and has been reset %s times" \
+                %(self.name, self.resettimeleft, self.resetcount)
+            if debug:
+                print reset_subject % params
+                print reset_body % params
+            try:
+                pid = os.fork()
+                if pid == 0:  
+                    print "Resetting slice " + self.name 
+                    vserver = VServer(self.name)
+                    vserver.stop()
+                    vserver.start()
+                    os._exit(0)
+                else:
+                    os.waitpid(pid,0)
+            except Exception, err:
+                print "Warning: Exception received while resetting slice %s:" \
+                        % self.name, err
+            if (time.time() - self.resetmail) > email_timeout:
+                slicemail(self.name, reset_subject % params, reset_body % params)
+                print "Sending Reset email for slice %s" % self.name
+                self.resetmail = time.time() 
+

 def usage():
     print """
 Usage: %s [OPTIONS]...
 def usage():
     print """
 Usage: %s [OPTIONS]...


@@ -124,26 +202,27 @@ Options:
         --system-slice=SLICE    System slice that should not be reset
         --status                Print memory usage statistics and exit
         -h, --help              This message
         --system-slice=SLICE    System slice that should not be reset
         --status                Print memory usage statistics and exit
         -h, --help              This message

-""".lstrip() % (sys.argv[0], debug, verbose, datafile, format_period(period))
+""".lstrip() % (sys.argv[0], debug, verbose, DATAFILE, format_period(period))

 
 def slicestat(names = None):
     """
     Get status of specified slices (if names is None or empty, all
 
 def slicestat(names = None):
     """
     Get status of specified slices (if names is None or empty, all

-    slices). vsize and rss are in KiB. Returns
-
+    slices). vsize, sz, and rss are in KiB. Returns
+    PID CONTEXT             VSZ    SZ   RSS %MEM CMD

     {xid: {'xid': slice_id,
     {xid: {'xid': slice_id,

-           'name': slice_name,
-           'procs': [{'pid': pid, 'xid': slice_id, 'user', username, 'cmd': command,
-                      'vsize': virtual_kib, 'rss': physical_kib,
+            'name': slice_name,
+            'procs': [{'pid': pid, 'xid': slice_id, 'cmd': command,
+                      'vsize': virtual_kib, 'sz': potential_kib, 'rss': physical_kib,

                       'pcpu': cpu_percent, 'pmem': mem_percent}]
                       'pcpu': cpu_percent, 'pmem': mem_percent}]

-           'vsize': total_virtual_kib,
-           'rss': total_physical_kib}}
+            'vsize': total_virtual_kib,
+            'sz': total_potential_kib,
+            'rss': total_physical_kib}}

     """
     
     # Mandatory fields. xid is a virtual field inserted by vps. Make
     # sure cmd is last so that it does not get truncated
     # automatically.
     """
     
     # Mandatory fields. xid is a virtual field inserted by vps. Make
     # sure cmd is last so that it does not get truncated
     # automatically.

-    fields = ['pid', 'xid', 'user', 'vsize', 'rss', 'pcpu', 'pmem', 'cmd']
+    fields = ['pid', 'xid', 'vsize', 'sz', 'rss', 'pmem', 'cmd']

 
     # vps inserts xid after pid in the output, but ps doesn't know
     # what the field means.
 
     # vps inserts xid after pid in the output, but ps doesn't know
     # what the field means.


@@ -155,7 +234,7 @@ def slicestat(names = None):
     # Eat the header line. vps depends on the header to figure out
     # which column is the PID column, so we can't just tell ps not to
     # print it.
     # Eat the header line. vps depends on the header to figure out
     # which column is the PID column, so we can't just tell ps not to
     # print it.

-    for line in bwlimit.run("/usr/sbin/vps -e -o " + ",".join(ps_fields))[1:]:
+    for line in bwlimit.run("/usr/sbin/vps -e -o " + ":16,".join(ps_fields))[1:]:

         # Chomp newline
         line = line.strip()
 
         # Chomp newline
         line = line.strip()
 


@@ -180,21 +259,25 @@ def slicestat(names = None):
                 except ValueError:
                     pass
 
                 except ValueError:
                     pass
 

+        # vps sometimes prints ERR or the name of the slice 
+            # instead of a context ID if it
+        # cannot identify the context of an orphaned (usually dying)
+        # process. Skip these processes.
+        if (type(proc['xid']) != int) or (type(proc['vsize']) !=int):
+            continue
+

         # Assign (pl_)sshd processes to slice instead of root
         m = re.search(r"sshd: ([a-zA-Z_]+)", proc['cmd'])
         # Assign (pl_)sshd processes to slice instead of root
         m = re.search(r"sshd: ([a-zA-Z_]+)", proc['cmd'])

+

         if m is not None:
             xid = bwlimit.get_xid(m.group(1))
             if xid is not None:
                 proc['xid'] = xid
 
         if m is not None:
             xid = bwlimit.get_xid(m.group(1))
             if xid is not None:
                 proc['xid'] = xid
 

-        try:
-            name = bwlimit.get_slice(proc['xid'])
-            if name is None:
-                # Orphaned (not associated with a slice) class
-                name = "%d?" % proc['xid']
-        except Exception, err:
-            print "Warning: Exception received while parsing vps output", err
-            print proc
+        name = bwlimit.get_slice(proc['xid'])
+        if name is None:
+            # Orphaned (not associated with a slice) class
+            name = "%d?" % proc['xid']

 
         # Monitor only the specified slices
         if names and name not in names:
 
         # Monitor only the specified slices
         if names and name not in names:


@@ -209,45 +292,50 @@ def slicestat(names = None):
         proc['rss'] += 12
 
         # Include additional page table overhead
         proc['rss'] += 12
 
         # Include additional page table overhead

-        if proc['vsize'] > 4096:
-            proc['rss'] += 4 * ((proc['vsize'] - 1) / 4096)
+        try:
+            if proc['vsize'] > 4096:
+                proc['rss'] += 4 * ((proc['vsize'] - 1) / 4096)
+        except: pass

 
         if slices.has_key(proc['xid']):
             slice = slices[proc['xid']]
         else:
 
         if slices.has_key(proc['xid']):
             slice = slices[proc['xid']]
         else:

-            slice = {'xid': proc['xid'], 'name': name, 'procs': [], 'vsize': 0, 'rss': 0}
+            slice = {'xid': proc['xid'], 'name': name, 'procs': [], 'vsize': 0, 'sz': 0, 'rss': 0}

 
         slice['procs'].append(proc)
         slice['vsize'] += proc['vsize']
 
         slice['procs'].append(proc)
         slice['vsize'] += proc['vsize']

+        slice['sz'] += proc['sz']

         slice['rss'] += proc['rss']
 
         slices[proc['xid']] = slice
         slice['rss'] += proc['rss']
 
         slices[proc['xid']] = slice

-
+       

     return slices
 
 def memtotal():
     """
     return slices
 
 def memtotal():
     """

-    Returns total physical memory on the system in KiB.
+    Returns total physical and swap memory on the system in KiB.

     """
     """

-
+    mem = 0
+    swap = 0

     meminfo = open("/proc/meminfo", "r")
     meminfo = open("/proc/meminfo", "r")

-    line = meminfo.readline()
+    for line in meminfo.readlines():
+        try:
+            (name, value, kb) = line.split()
+        except:
+            continue
+        if name == "MemTotal:": 
+            mem = int(value)
+        elif name == "SwapTotal:":
+            swap = int(value)

     meminfo.close()
     meminfo.close()

-    if line[0:8] == "MemTotal":
-        # MemTotal: 255396 kB
-        (name, value, kb) = line.split()
-        return int(value)
-
-    return 0
+    return (mem, swap)

 
 def swap_used():
     """
     Returns swap utilization on the system as a whole percentage (0-100).
     """
 
 def swap_used():
     """
     Returns swap utilization on the system as a whole percentage (0-100).
     """

-

     total_swap = 0
     total_used = 0
     total_swap = 0
     total_used = 0

-

     try:
         swaps = open("/proc/swaps", "r")
         # Eat header line
     try:
         swaps = open("/proc/swaps", "r")
         # Eat header line


@@ -261,31 +349,105 @@ def swap_used():
                 total_used += int(used)
             except ValueEror, err:
                 pass
                 total_used += int(used)
             except ValueEror, err:
                 pass

-    except (IOError, KeyError), err:
-        pass
+    except (IOError, KeyError), err:  pass

 
 

-    return 100 * total_used / total_swap
+    swapused = 100 * total_used / total_swap
+    if debug: print "%s percent swap used" % swapused
+    return swapused

 
 

-def summary(names = None, total_rss = memtotal()):
+def summary(slices = None, total_mem = None, total_swap = None):

     """
     Return a summary of memory usage by slice.
     """
     """
     Return a summary of memory usage by slice.
     """

-    slicelist = slicestat(names).values()
-    slicelist.sort(lambda a, b: b['rss'] - a['rss'])
+    if not slices:  slices = slicestat()
+    slicelist = slices.values()
+    slicelist.sort(lambda a, b: b['sz'] - a['sz'])
+    if total_mem is None or total_swap is None:
+        (total_mem, total_swap) = memtotal()

 
 

-    table = "%-20s%10s%24s\n\n" % ("Slice", "Processes", "Memory Usage")
+    table = "%-20s%10s%24s%24s\n\n" % ("Slice", "Processes", "Memory Usage", "Potential Usage")

     for slice in slicelist:
     for slice in slicelist:

-        table += "%-20s%10d%16s (%4.1f%%)\n" % \
+        table += "%-20s%10d%16s (%4.1f%%)%16s (%4.1f%%)\n" % \

                  (slice['name'], len(slice['procs']),
                   format_bytes(slice['rss'] * 1024, si = False),
                  (slice['name'], len(slice['procs']),
                   format_bytes(slice['rss'] * 1024, si = False),

-                  100. * slice['rss'] / total_rss)
-
+                  100. * slice['rss'] / total_mem,
+                  format_bytes(slice['sz'] * 1024, si = False),
+                  100. * slice['sz'] / (total_mem + total_swap))

     return table
 
     return table
 

+def formtable(slice, percent):
+    '''
+    Makes pretty message to email with human readable ps values.
+    '''
+    table = "%5s %10s %10s %10s %4s %4s %s\n\n" % \
+        ("PID", "VIRT", "SZ", "RES", '%CPU', '%MEM', 'COMMAND')
+    for proc in slice['procs']:
+        table += "%5s %10s %10s %10s %4.1f %s\n" % \
+            (proc['pid'],
+            format_bytes(proc['vsize'] * 1024, si = False),
+            format_bytes(proc['sz'] * 1024, si = False),
+            format_bytes(proc['rss'] * 1024, si = False),
+            proc['pmem'],
+            proc['cmd'])
+    
+    prettytable = {'hostname': socket.gethostname(),
+             'date': time.asctime(time.gmtime()) + " GMT",
+             'table': table,
+             'slice': slice['name'],
+             'rss': format_bytes(slice['rss'] * 1024, si = False),
+             'sz': format_bytes(slice['sz'] * 1024, si = False),
+             'percent': percent}
+    return prettytable
+
+def readdat():
+    '''
+    Return dictionary of vps (slicestat) from datfile left behind by OOM
+    before rebooting.  If none file, just grab the latest dict (slicestat)
+    and return that.  If dat file found, means we rebooted, send an email to 
+    pl_mom@pl.
+    '''
+    try:
+        f = open(DATAFILE, "r+")
+        if verbose:
+            print "Loading %s" % DATAFILE
+        (v, slices) = pickle.load(f)
+        f.close()
+        # Check version of data file
+        if v != VERSION:
+            print "Not using old version '%s' data file %s" % (v, DATAFILE)
+            raise Exception
+
+        params = {'hostname': socket.gethostname(),
+                  'date': time.asctime(time.gmtime()) + " GMT",
+                  'table': summary(slices, total_mem, total_swap)}
+        if debug:
+            print rebooted_subject % params
+            print rebooted_body % params
+        else:
+            slicemail(None, rebooted_subject % params, rebooted_body % params)
+
+        # Delete data file
+        os.unlink(DATAFILE)
+    except Exception:
+        slices = slicestat()
+
+    return slices
+
+
+def writedat(slices):
+    """
+    Write (slices) to pickled datfile.
+    """
+    if verbose:  print "Saving %s" % DATAFILE
+    f = open(DATAFILE, "w")
+    pickle.dump((VERSION, slices), f)
+    f.close()
+
+

 def main():
     # Defaults
 def main():
     # Defaults

-    global debug, verbose, datafile
-    global period, change_thresh, reset_thresh, reboot_thresh, min_thresh, system_slices
+    global debug, verbose, DATAFILE, VERSION 
+    global period, change_thresh, reset_thresh, reboot_thresh, rss_min, system_slices

     # All slices
     names = []
 
     # All slices
     names = []
 


@@ -304,7 +466,7 @@ def main():
         elif opt == "-v" or opt == "--verbose":
             verbose += 1
         elif opt == "-f" or opt == "--file":
         elif opt == "-v" or opt == "--verbose":
             verbose += 1
         elif opt == "-f" or opt == "--file":

-            datafile = optval
+            DATAFILE = optval

         elif opt == "-s" or opt == "--slice":
             names.append(optval)
         elif opt == "-p" or opt == "--period":
         elif opt == "-s" or opt == "--slice":
             names.append(optval)
         elif opt == "-p" or opt == "--period":


@@ -316,11 +478,11 @@ def main():
         elif opt == "--reboot-thresh":
             reboot_thresh = int(optval)
         elif opt == "--min-thresh":
         elif opt == "--reboot-thresh":
             reboot_thresh = int(optval)
         elif opt == "--min-thresh":

-            min_thresh = int(optval)
+            rss_min = int(optval)

         elif opt == "--system-slice":
             system_slices.append(optval)
         elif opt == "--status":
         elif opt == "--system-slice":
             system_slices.append(optval)
         elif opt == "--status":

-            print summary(names)
+            print summary(slicestat(names))

             sys.exit(0)
         else:
             usage()
             sys.exit(0)
         else:
             usage()


@@ -337,35 +499,9 @@ def main():
         syslog.openlog("swapmon")
         sys.stdout = sys.stderr = Logger()
 
         syslog.openlog("swapmon")
         sys.stdout = sys.stderr = Logger()
 

-    # Get total physical memory
-    total_rss = memtotal()
-
-    try:
-        f = open(datafile, "r+")
-        if verbose:
-            print "Loading %s" % datafile
-        (version, slices) = pickle.load(f)
-        f.close()
-        # Check version of data file
-        if version != "$Id: swapmon.py,v 1.3 2006/05/01 18:28:01 mlhuang Exp $":
-            print "Not using old version '%s' data file %s" % (version, datafile)
-            raise Exception
-
-        params = {'hostname': socket.gethostname(),
-                  'date': time.asctime(time.gmtime()) + " GMT",
-                  'table': summary(total_rss)}
-
-        if debug:
-            print rebooted_subject % params
-            print rebooted_body % params
-        else:
-            slicemail(None, rebooted_subject % params, rebooted_body % params)
-
-        # Delete data file
-        os.unlink(datafile)
-    except Exception:
-        version = "$Id: swapmon.py,v 1.3 2006/05/01 18:28:01 mlhuang Exp $"
-        slices = {}
+    # Get total memory
+    (total_mem, total_swap) = memtotal()
+    slices = readdat()

 
     # Query process table every 30 seconds, or when a large change in
     # swap utilization is detected.
 
     # Query process table every 30 seconds, or when a large change in
     # swap utilization is detected.


@@ -373,106 +509,63 @@ def main():
     last_used = None
     used = None
 
     last_used = None
     used = None
 

-    # System slices that we have warned but could not reset
-    warned = []
-

     while True:
         used = swap_used()
     while True:
         used = swap_used()

-        if last_used is None:
-            last_used = used
-        if verbose:
-            print "%d%% swap consumed" % used
-
+        if last_used is None:  last_used = used
+        
+   

         if used >= reboot_thresh:
             # Dump slice state before rebooting
         if used >= reboot_thresh:
             # Dump slice state before rebooting

-            if verbose:
-                print "Saving %s" % datafile
-            f = open(datafile, "w")
-            pickle.dump((version, slices), f)
-            f.close()
-
+            writedat(slices)    

             # Goodbye, cruel world
             print "%d%% swap consumed, rebooting" % used
             # Goodbye, cruel world
             print "%d%% swap consumed, rebooting" % used

-            if not debug:
-                bwlimit.run("/bin/sync; /sbin/reboot -f")
-
+            if not debug:  bwlimit.run("/bin/sync; /sbin/reboot -f")

         elif used >= reset_thresh:
             # Try and find a hog
             slicelist = slices.values()
         elif used >= reset_thresh:
             # Try and find a hog
             slicelist = slices.values()

+            # Puts largest on top.

             slicelist.sort(lambda a, b: b['rss'] - a['rss'])
             for slice in slicelist:
             slicelist.sort(lambda a, b: b['rss'] - a['rss'])
             for slice in slicelist:

-                percent = 100. * slice['rss'] / total_rss
-                if percent < min_thresh:
-                    continue
-
+                percent = 100. * slice['rss'] / total_mem
+                if slice['rss'] < rss_min: continue

                 print "%d%% swap consumed, slice %s is using %s (%d%%) of memory" % \
                 print "%d%% swap consumed, slice %s is using %s (%d%%) of memory" % \

-                      (used,
-                       slice['name'],
-                       format_bytes(slice['rss'] * 1024, si = False),
-                       percent)
-
+                    (used,
+                    slice['name'],
+                    format_bytes(slice['rss'] * 1024, si = False),
+                    percent)

                 slice['procs'].sort(lambda a, b: b['rss'] - a['rss'])
                 slice['procs'].sort(lambda a, b: b['rss'] - a['rss'])

-
-                table = "%5s %10s %10s %4s %4s %s\n\n" % ("PID", "VIRT", "RES", '%CPU', '%MEM', 'COMMAND')
-                for proc in slice['procs']:
-                    table += "%5s %10s %10s %4.1f %4.1f %s\n" % \
-                             (proc['pid'],
-                              format_bytes(proc['vsize'] * 1024, si = False),
-                              format_bytes(proc['rss'] * 1024, si = False),
-                              proc['pcpu'], proc['pmem'], proc['cmd'])
-
-                params = {'hostname': socket.gethostname(),
-                          'date': time.asctime(time.gmtime()) + " GMT",
-                          'table': table,
-                          'slice': slice['name'],
-                          'rss': format_bytes(slice['rss'] * 1024, si = False),
-                          'percent': percent}
-
+                # Make a pretty table.
+                params = formtable(slice, percent)

                 # Match slice name against system slice patterns
                 # Match slice name against system slice patterns

-                is_system_slice = filter(None, [re.match(pattern, slice['name']) for pattern in system_slices])
-
-                if is_system_slice:
-                    # Do not reset system slices, just warn once
+                is_system_slice = filter(None, 
+                    [re.match(pattern, slice['name']) for pattern in system_slices])
+    
+                # Do not reset system slices, just warn once
+                if is_system_slice: 

                     if slice['name'] not in warned:
                         warned.append(slice['name'])
                     if slice['name'] not in warned:
                         warned.append(slice['name'])

+                        print "Warning slice " + slice['name']

                         if debug:
                             print alarm_subject % params
                             print alarm_body % params
                         else:
                         if debug:
                             print alarm_subject % params
                             print alarm_body % params
                         else:

-                            print "Warning slice " + slice['name']
-                            slicemail(slice['name'], alarm_subject % params, alarm_body % params)
+                            slicemail(slice['name'], alarm_subject % params, 
+                              alarm_body % params)

                 else:
                 else:

-                    # Otherwise, reset
-                    if debug:
-                        print reset_subject % params
-                        print reset_body % params
-                    else:
-                        try:
-                            pid = os.fork()
-                            if pid == 0:
-                                print "Resetting slice " + slice['name']
-                                vserver = VServer(slice['name'])
-                                vserver.stop()
-                                vserver.start(wait = False)
-                                os._exit(0)
-                            else:
-                                os.waitpid(pid, 0)
-                        except Exception, err:
-                            print "Warning: Exception received while resetting slice %s:" % slice['name'], err
-                        slicemail(slice['name'], reset_subject % params, reset_body % params)
-                    break
-
-        elif timer <= 0 or used >= (last_used + change_thresh):
+                    # Reset slice
+                       if not debug: slicemail(self.name, reset_subject % params, reset_body % params)
+
+        # wait period before recalculating swap.  If in danger, recalc.
+        if timer <= 0 or used >= (last_used + change_thresh):

             if used >= (last_used + change_thresh):
                 print "%d%% swap consumed, %d%% in last %d seconds" % \
             if used >= (last_used + change_thresh):
                 print "%d%% swap consumed, %d%% in last %d seconds" % \

-                      (used, used - last_used, period - timer)
+                    (used, used - last_used, period - timer)

             # Get slice state
             slices = slicestat(names)
             # Reset timer
             timer = period
             # Keep track of large changes in swap utilization
             last_used = used
             # Get slice state
             slices = slicestat(names)
             # Reset timer
             timer = period
             # Keep track of large changes in swap utilization
             last_used = used

-

         timer -= 1
         time.sleep(1)
 
         timer -= 1
         time.sleep(1)