Somewhere in the CVS to SVN conversion, or maybe it was just me using ViM incorrectly...

..This was never written correctly and is badly in need of a rewrite.

diff --git a/swapmon.py b/swapmon.py
index f15879f..da8f3a0 100755 (executable)
--- a/swapmon.py
+++ b/swapmon.py
@@ -10,7 +10,7 @@
 # Faiyaz Ahmed <faiyaza@cs.princeton.edu>
 # Copyright (C) 2004-2006 The Trustees of Princeton University
 #
 # Faiyaz Ahmed <faiyaza@cs.princeton.edu>
 # Copyright (C) 2004-2006 The Trustees of Princeton University
 #

-# $Id: swapmon.py,v 1.11 2006/12/02 19:11:47 mlhuang Exp $
+# $Id$

 #
 
 import syslog
 #
 
 import syslog


@@ -51,7 +51,7 @@ reset_thresh = 80
 reboot_thresh = 95
 
 # Time to wait before checking slice again after reset
 reboot_thresh = 95
 
 # Time to wait before checking slice again after reset

-reset_timeout = 15
+reset_timeout = 25

 
 # Number of strikes before killing (strike, strike, kill)
 kill_thresh = 2
 
 # Number of strikes before killing (strike, strike, kill)
 kill_thresh = 2


@@ -97,6 +97,8 @@ physical memory at %(rss)s (%(percent)4.1f%%) (%(sz)s writable).
 Please reply to this message explaining the nature of your experiment,
 and what you are doing to address the problem.
 
 Please reply to this message explaining the nature of your experiment,
 and what you are doing to address the problem.
 

+http://summer.cs.princeton.edu/status/tabulator.cgi?table=slices/table_%(slice)s
+

 %(slice)s processes prior to reset:
 
 %(table)s
 %(slice)s processes prior to reset:
 
 %(table)s


@@ -188,8 +190,8 @@ class Reset:
     def checkkill(self,params):
         if self.killtimeleft > 0 and self.resetcount >= kill_thresh:
             if debug:
     def checkkill(self,params):
         if self.killtimeleft > 0 and self.resetcount >= kill_thresh:
             if debug:

-                                print kill_subject % params
-                                print kill_body % params
+                print kill_subject % params
+                print kill_body % params

             try:
                 pid = os.fork()
                 if pid == 0:
             try:
                 pid = os.fork()
                 if pid == 0:


@@ -269,13 +271,13 @@ def slicestat(names = None):
     slices). vsize, sz, and rss are in KiB. Returns
 
     {xid: {'xid': slice_id,
     slices). vsize, sz, and rss are in KiB. Returns
 
     {xid: {'xid': slice_id,

-           'name': slice_name,
-           'procs': [{'pid': pid, 'xid': slice_id, 'user', username, 'cmd': command,
+            'name': slice_name,
+            'procs': [{'pid': pid, 'xid': slice_id, 'user', username, 'cmd': command,

                       'vsize': virtual_kib, 'sz': potential_kib, 'rss': physical_kib,
                       'pcpu': cpu_percent, 'pmem': mem_percent}]
                       'vsize': virtual_kib, 'sz': potential_kib, 'rss': physical_kib,
                       'pcpu': cpu_percent, 'pmem': mem_percent}]

-           'vsize': total_virtual_kib,
-       'sz': total_potential_kib,
-           'rss': total_physical_kib}}
+            'vsize': total_virtual_kib,
+            'sz': total_potential_kib,
+            'rss': total_physical_kib}}

     """
     
     # Mandatory fields. xid is a virtual field inserted by vps. Make
     """
     
     # Mandatory fields. xid is a virtual field inserted by vps. Make


@@ -370,10 +372,8 @@ def memtotal():
     """
     Returns total physical and swap memory on the system in KiB.
     """
     """
     Returns total physical and swap memory on the system in KiB.
     """

-

     mem = 0
     swap = 0
     mem = 0
     swap = 0

-

     meminfo = open("/proc/meminfo", "r")
     for line in meminfo.readlines():
         try:
     meminfo = open("/proc/meminfo", "r")
     for line in meminfo.readlines():
         try:


@@ -385,17 +385,14 @@ def memtotal():
         elif name == "SwapTotal:":
             swap = int(value)
     meminfo.close()
         elif name == "SwapTotal:":
             swap = int(value)
     meminfo.close()

-

     return (mem, swap)
 
 def swap_used():
     """
     Returns swap utilization on the system as a whole percentage (0-100).
     """
     return (mem, swap)
 
 def swap_used():
     """
     Returns swap utilization on the system as a whole percentage (0-100).
     """

-

     total_swap = 0
     total_used = 0
     total_swap = 0
     total_used = 0

-

     try:
         swaps = open("/proc/swaps", "r")
         # Eat header line
     try:
         swaps = open("/proc/swaps", "r")
         # Eat header line


@@ -409,17 +406,17 @@ def swap_used():
                 total_used += int(used)
             except ValueEror, err:
                 pass
                 total_used += int(used)
             except ValueEror, err:
                 pass

-    except (IOError, KeyError), err:
-        pass
+    except (IOError, KeyError), err:  pass

 
 

-    return 100 * total_used / total_swap
+    swapused = 100 * total_used / total_swap
+    if debug: print "%s percent swap used" % swapused
+    return swapused

 
 def summary(slices = None, total_mem = None, total_swap = None):
     """
     Return a summary of memory usage by slice.
     """
 
 def summary(slices = None, total_mem = None, total_swap = None):
     """
     Return a summary of memory usage by slice.
     """

-    if not slices:
-        slices = slicestat()
+    if not slices:  slices = slicestat()

     slicelist = slices.values()
     slicelist.sort(lambda a, b: b['sz'] - a['sz'])
     if total_mem is None or total_swap is None:
     slicelist = slices.values()
     slicelist.sort(lambda a, b: b['sz'] - a['sz'])
     if total_mem is None or total_swap is None:


@@ -433,10 +430,79 @@ def summary(slices = None, total_mem = None, total_swap = None):
                   100. * slice['rss'] / total_mem,
                   format_bytes(slice['sz'] * 1024, si = False),
                   100. * slice['sz'] / (total_mem + total_swap))
                   100. * slice['rss'] / total_mem,
                   format_bytes(slice['sz'] * 1024, si = False),
                   100. * slice['sz'] / (total_mem + total_swap))

-          
-

     return table
 
     return table
 

+def formtable(slice, percent):
+    '''
+    Makes pretty message to email with human readable ps values.
+    '''
+    table = "%5s %10s %10s %10s %4s %4s %s\n\n" % \
+        ("PID", "VIRT", "SZ", "RES", '%CPU', '%MEM', 'COMMAND')
+    for proc in slice['procs']:
+        table += "%5s %10s %10s %10s %4.1f %4.1f %s\n" % \
+            (proc['pid'],
+            format_bytes(proc['vsize'] * 1024, si = False),
+            format_bytes(proc['sz'] * 1024, si = False),
+            format_bytes(proc['rss'] * 1024, si = False),
+            proc['pcpu'],
+            proc['pmem'],
+            proc['cmd'])
+    
+    prettytable = {'hostname': socket.gethostname(),
+             'date': time.asctime(time.gmtime()) + " GMT",
+             'table': table,
+             'slice': slice['name'],
+             'rss': format_bytes(slice['rss'] * 1024, si = False),
+             'sz': format_bytes(slice['sz'] * 1024, si = False),
+             'percent': percent}
+    return prettytable
+
+def readdat():
+    '''
+    Return dictionary of vps (slicestat) from datfile left behind by OOM
+    before rebooting.  If none file, just grab the latest dict (slicestat)
+    and return that.  If dat file found, means we rebooted, send an email to 
+    pl_mom@pl.
+    '''
+    try:
+        f = open(datafile, "r+")
+        if verbose:
+            print "Loading %s" % datafile
+        (version, slices) = pickle.load(f)
+        f.close()
+        # Check version of data file
+        if version != "$Id$":
+            print "Not using old version '%s' data file %s" % (version, datafile)
+            raise Exception
+
+        params = {'hostname': socket.gethostname(),
+                  'date': time.asctime(time.gmtime()) + " GMT",
+                  'table': summary(slices, total_mem, total_swap)}
+        if debug:
+            print rebooted_subject % params
+            print rebooted_body % params
+        else:
+            slicemail(None, rebooted_subject % params, rebooted_body % params)
+
+        # Delete data file
+        os.unlink(datafile)
+    except Exception:
+        version = "$Id$"
+        slices = slicestat()
+
+    return slices
+
+
+def writedat(slices):
+    """
+    Write (slices) to pickled datfile.
+    """
+    if verbose:  print "Saving %s" % datafile
+    f = open(datafile, "w")
+    pickle.dump((version, slices), f)
+    f.close()
+
+

 def main():
     # Defaults
     global debug, verbose, datafile
 def main():
     # Defaults
     global debug, verbose, datafile


@@ -494,33 +560,7 @@ def main():
 
     # Get total memory
     (total_mem, total_swap) = memtotal()
 
     # Get total memory
     (total_mem, total_swap) = memtotal()

-
-    try:
-        f = open(datafile, "r+")
-        if verbose:
-            print "Loading %s" % datafile
-        (version, slices) = pickle.load(f)
-        f.close()
-        # Check version of data file
-        if version != "$Id: swapmon.py,v 1.11 2006/12/02 19:11:47 mlhuang Exp $":
-            print "Not using old version '%s' data file %s" % (version, datafile)
-            raise Exception
-
-        params = {'hostname': socket.gethostname(),
-                  'date': time.asctime(time.gmtime()) + " GMT",
-                  'table': summary(slices, total_mem, total_swap)}
-
-        if debug:
-            print rebooted_subject % params
-            print rebooted_body % params
-        else:
-            slicemail(None, rebooted_subject % params, rebooted_body % params)
-
-        # Delete data file
-        os.unlink(datafile)
-    except Exception:
-        version = "$Id: swapmon.py,v 1.11 2006/12/02 19:11:47 mlhuang Exp $"
-        slices = {}
+    slices = readdat()

 
     # Query process table every 30 seconds, or when a large change in
     # swap utilization is detected.
 
     # Query process table every 30 seconds, or when a large change in
     # swap utilization is detected.


@@ -536,95 +576,72 @@ def main():
 
     while True:
         used = swap_used()
 
     while True:
         used = swap_used()

-
-    for resetslice in resetlist.keys():
-        resetlist[resetslice].update()
+        if last_used is None:  last_used = used
+        
+        # If we've reset you recently, update timers.
+        for resetslice in resetlist.keys(): 
+            resetlist[resetslice].update()
+            # If you've been good, remove you from our list.
+            if resetlist[resetslice].killtimeleft == 0 and \
+            resetlist[resetslice].resettimeleft == 0:
+                del resetlist[resetslice]
+
+        if verbose:  print "%d%% swap consumed" % used
+    
+        if used >= reboot_thresh:
+            # Dump slice state before rebooting
+            writedat(slices)    
+            # Goodbye, cruel world
+            print "%d%% swap consumed, rebooting" % used
+            if not debug:  bwlimit.run("/bin/sync; /sbin/reboot -f")
+        elif used >= reset_thresh:
+            # Try and find a hog
+            slicelist = slices.values()
+            # Puts largest on top.
+            slicelist.sort(lambda a, b: b['rss'] - a['rss'])
+            for slice in slicelist:
+                percent = 100. * slice['rss'] / total_mem
+                if slice['rss'] < rss_min: continue
+                print "%d%% swap consumed, slice %s is using %s (%d%%) of memory" % \
+                    (used,
+                    slice['name'],
+                    format_bytes(slice['rss'] * 1024, si = False),
+                    percent)
+                slice['procs'].sort(lambda a, b: b['rss'] - a['rss'])
+                # Make a pretty table.
+                params = formtable(slice, percent)
+                # Match slice name against system slice patterns
+                is_system_slice = filter(None, 
+                    [re.match(pattern, slice['name']) for pattern in system_slices])

     
     

-        if last_used is None:
-            last_used = used
-
-    if verbose:
-            print "%d%% swap consumed" % used
-
-    if used >= reboot_thresh:
-        # Dump slice state before rebooting
-        if verbose:  print "Saving %s" % datafile
-        f = open(datafile, "w")
-        pickle.dump((version, slices), f)
-        f.close()
-
-        # Goodbye, cruel world
-        print "%d%% swap consumed, rebooting" % used
-        if not debug:  bwlimit.run("/bin/sync; /sbin/reboot -f")
-
-    elif used >= reset_thresh:
-        if debug:  print "Memory used = %s" %(used)
-        # Try and find a hog
-        slicelist = slices.values()
-        slicelist.sort(lambda a, b: b['rss'] - a['rss'])
-        for slice in slicelist:
-            percent = 100. * slice['rss'] / total_mem
-            if slice['rss'] < rss_min: continue
-            print "%d%% swap consumed, slice %s is using %s (%d%%) of memory" % \
-                (used,
-                slice['name'],
-                format_bytes(slice['rss'] * 1024, si = False),
-                percent)
-
-        slice['procs'].sort(lambda a, b: b['rss'] - a['rss'])
-        table = "%5s %10s %10s %10s %4s %4s %s\n\n" % \
-            ("PID", "VIRT", "SZ", "RES", '%CPU', '%MEM', 'COMMAND')
-        for proc in slice['procs']:
-            table += "%5s %10s %10s %10s %4.1f %4.1f %s\n" % \
-                (proc['pid'],
-                format_bytes(proc['vsize'] * 1024, si = False),
-                format_bytes(proc['sz'] * 1024, si = False),
-                format_bytes(proc['rss'] * 1024, si = False),
-                proc['pcpu'],
-                proc['pmem'],
-                proc['cmd'])
-
-            params = {'hostname': socket.gethostname(),
-                'date': time.asctime(time.gmtime()) + " GMT",
-                'table': table,
-                'slice': slice['name'],
-                'rss': format_bytes(slice['rss'] * 1024, si = False),
-                'sz': format_bytes(slice['sz'] * 1024, si = False),
-                'percent': percent}
-
-            # Match slice name against system slice patterns
-            is_system_slice = filter(None, 
-                                [re.match(pattern, slice['name']) for pattern in system_slices])
-
-            if is_system_slice: 

                 # Do not reset system slices, just warn once
                 # Do not reset system slices, just warn once

-                if slice['name'] not in warned:
-                     warned.append(slice['name'])
-                if debug:
-                    print alarm_subject % params
-                    print alarm_body % params
+                if is_system_slice: 
+                    if slice['name'] not in warned:
+                        warned.append(slice['name'])
+                        print "Warning slice " + slice['name']
+                        if debug:
+                            print alarm_subject % params
+                            print alarm_body % params
+                        else:
+                            slicemail(slice['name'], alarm_subject % params, 
+                              alarm_body % params)

                 else:
                 else:

-                    print "Warning slice " + slice['name']
-                    slicemail(slice['name'], alarm_subject % params, 
-                        alarm_body % params)
-            else:
-                # Reset slice
-                if not resetlist.has_key(slice['name']):
-                    resetlist[slice['name']] = Reset(slice['name'])
-                resetlist[slice['name']].reset(params)
-                slices = slicestat(names)
+                    # Reset slice
+                    if not resetlist.has_key(slice['name']):
+                        resetlist[slice['name']] = Reset(slice['name'])
+                    resetlist[slice['name']].reset(params)

 
 

+        # wait period vefore recalculating swap.  If in danger, recalc.

         if timer <= 0 or used >= (last_used + change_thresh):
             if used >= (last_used + change_thresh):
                 print "%d%% swap consumed, %d%% in last %d seconds" % \
         if timer <= 0 or used >= (last_used + change_thresh):
             if used >= (last_used + change_thresh):
                 print "%d%% swap consumed, %d%% in last %d seconds" % \

-                      (used, used - last_used, period - timer)
+                    (used, used - last_used, period - timer)

             # Get slice state
             slices = slicestat(names)
             # Reset timer
             timer = period
             # Keep track of large changes in swap utilization
             last_used = used
             # Get slice state
             slices = slicestat(names)
             # Reset timer
             timer = period
             # Keep track of large changes in swap utilization
             last_used = used

-

         timer -= 1
         time.sleep(1)
 
         timer -= 1
         time.sleep(1)