* Queries NM for: "nm_net_max_byte",

[mom.git] / swapmon.py

diff --git a/swapmon.py b/swapmon.py
index 02a01ca..37743b1 100755 (executable)
--- a/swapmon.py
+++ b/swapmon.py
@@ -10,7 +10,7 @@
 # Faiyaz Ahmed <faiyaza@cs.princeton.edu>
 # Copyright (C) 2004-2006 The Trustees of Princeton University
 #
 # Faiyaz Ahmed <faiyaza@cs.princeton.edu>
 # Copyright (C) 2004-2006 The Trustees of Princeton University
 #

-# $Id: swapmon.py,v 1.5 2006/05/09 03:23:57 mlhuang Exp $
+# $Id: swapmon.py,v 1.10 2006/08/16 16:18:45 faiyaza Exp $

 #
 
 import syslog
 #
 
 import syslog


@@ -62,8 +62,9 @@ kill_timeout = 120
 # Don't email the same message more than once in the same emailtimeout interval
 email_timeout = 1800
 
 # Don't email the same message more than once in the same emailtimeout interval
 email_timeout = 1800
 

-# Minimum physical memory utilization to be considered the largest consumer
-min_thresh = 10
+# Physical size threshold to be considered a consumer.  Rationale is if there are no procs
+# with a size at least as large as this, then there is a slow leaker;  better to just reboot.
+rss_min = 150 * 1024 

 
 # System slices that should not be reset (regexps)
 system_slices = ['root', PLC_SLICE_PREFIX + '_']
 
 # System slices that should not be reset (regexps)
 system_slices = ['root', PLC_SLICE_PREFIX + '_']


@@ -91,7 +92,7 @@ Sometime before %(date)s, swap space was
 nearly exhausted on %(hostname)s.
 
 Slice %(slice)s was reset since it was the largest consumer of
 nearly exhausted on %(hostname)s.
 
 Slice %(slice)s was reset since it was the largest consumer of

-physical memory at %(rss)s (%(percent)4.1f%%).
+physical memory at %(rss)s (%(percent)4.1f%%) (%(sz)s writable).

 
 Please reply to this message explaining the nature of your experiment,
 and what you are doing to address the problem.
 
 Please reply to this message explaining the nature of your experiment,
 and what you are doing to address the problem.


@@ -111,8 +112,8 @@ Sometime before %(date)s, swap space was
 nearly exhausted on %(hostname)s.
 
 System slice %(slice)s was the largest consumer of physical memory at
 nearly exhausted on %(hostname)s.
 
 System slice %(slice)s was the largest consumer of physical memory at

-%(rss)s (%(percent)4.1f%%). It was not reset, but please verify its
-behavior.
+%(rss)s (%(percent)4.1f%%) (%(sz)s writable). It was not reset,
+but please verify its behavior.

 
 %(slice)s processes prior to alarm:
 
 
 %(slice)s processes prior to alarm:
 


@@ -129,7 +130,8 @@ Sometime before %(date)s, swap space was
 nearly exhausted on %(hostname)s.
 
 Slice %(slice)s was killed since it was the largest consumer of
 nearly exhausted on %(hostname)s.
 
 Slice %(slice)s was killed since it was the largest consumer of

-physical memory at %(rss)s (%(percent)4.1f%%) after repeated restarts.
+physical memory at %(rss)s (%(percent)4.1f%%) (%(sz)s writable)
+after repeated restarts.

 
 Please reply to this message explaining the nature of your experiment,
 and what you are doing to address the problem.
 
 Please reply to this message explaining the nature of your experiment,
 and what you are doing to address the problem.


@@ -159,7 +161,6 @@ class Reset:
                self.resettimeleft = reset_timeout
                self.resetcount = 0 
                self.resetmail = 0
                self.resettimeleft = reset_timeout
                self.resetcount = 0 
                self.resetmail = 0

-               self.kill = False

                self.killtimeleft = kill_timeout
                self.killmail = 0
 
                self.killtimeleft = kill_timeout
                self.killmail = 0
 


@@ -181,15 +182,12 @@ class Reset:
                else:
                        # Once out of probation period (killtimeleft), remove strikes
                        self.resetcount = 0
                else:
                        # Once out of probation period (killtimeleft), remove strikes
                        self.resetcount = 0

-                       self.kill = False

 
 
 
 

-       # Check to see if a slice needs to be killed.  If it has rules more than kill_thresh in 
-       # the probation period (kill_timeout) send an email, kill the slice.
+       # Check to see if a slice needs to be killed.  If it has been killed more 
+       # than kill_thresh in the probation period (kill_timeout) send an email, kill the slice.

        def checkkill(self,params):
        def checkkill(self,params):

-               if self.killtimeleft > 0 and self.resetcount >= kill_thresh and \
-               self.kill == False:
-                       self.kill = True
+               if self.killtimeleft > 0 and self.resetcount >= kill_thresh:

                        if debug:
                                 print kill_subject % params
                                 print kill_body % params
                        if debug:
                                 print kill_subject % params
                                 print kill_body % params


@@ -215,7 +213,8 @@ class Reset:
        # Reset slice after checking to see if slice is out of timeout.
        # Increment resetcount, check to see if larger than kill_thresh.
        def reset(self, params):
        # Reset slice after checking to see if slice is out of timeout.
        # Increment resetcount, check to see if larger than kill_thresh.
        def reset(self, params):

-               # If its the first reset or if its been reset before
+               # If its the first reset (came back after kill)
+               # or if its been reset before

                # and we are out of the reset timeout.
                if self.resetcount == 0 or self.resettimeleft == 0:
                        # Do we need to kill this slice?  Check history first.
                # and we are out of the reset timeout.
                if self.resetcount == 0 or self.resettimeleft == 0:
                        # Do we need to kill this slice?  Check history first.


@@ -270,21 +269,22 @@ Options:
 def slicestat(names = None):
     """
     Get status of specified slices (if names is None or empty, all
 def slicestat(names = None):
     """
     Get status of specified slices (if names is None or empty, all

-    slices). vsize and rss are in KiB. Returns
+    slices). vsize, sz, and rss are in KiB. Returns

 
     {xid: {'xid': slice_id,
            'name': slice_name,
            'procs': [{'pid': pid, 'xid': slice_id, 'user', username, 'cmd': command,
 
     {xid: {'xid': slice_id,
            'name': slice_name,
            'procs': [{'pid': pid, 'xid': slice_id, 'user', username, 'cmd': command,

-                      'vsize': virtual_kib, 'rss': physical_kib,
+                      'vsize': virtual_kib, 'sz': potential_kib, 'rss': physical_kib,

                       'pcpu': cpu_percent, 'pmem': mem_percent}]
            'vsize': total_virtual_kib,
                       'pcpu': cpu_percent, 'pmem': mem_percent}]
            'vsize': total_virtual_kib,

+          'sz': total_potential_kib,

            'rss': total_physical_kib}}
     """
     
     # Mandatory fields. xid is a virtual field inserted by vps. Make
     # sure cmd is last so that it does not get truncated
     # automatically.
            'rss': total_physical_kib}}
     """
     
     # Mandatory fields. xid is a virtual field inserted by vps. Make
     # sure cmd is last so that it does not get truncated
     # automatically.

-    fields = ['pid', 'xid', 'user', 'vsize', 'rss', 'pcpu', 'pmem', 'cmd']
+    fields = ['pid', 'xid', 'user', 'vsize', 'sz', 'rss', 'pcpu', 'pmem', 'cmd']

 
     # vps inserts xid after pid in the output, but ps doesn't know
     # what the field means.
 
     # vps inserts xid after pid in the output, but ps doesn't know
     # what the field means.


@@ -358,10 +358,11 @@ def slicestat(names = None):
         if slices.has_key(proc['xid']):
             slice = slices[proc['xid']]
         else:
         if slices.has_key(proc['xid']):
             slice = slices[proc['xid']]
         else:

-            slice = {'xid': proc['xid'], 'name': name, 'procs': [], 'vsize': 0, 'rss': 0}
+            slice = {'xid': proc['xid'], 'name': name, 'procs': [], 'vsize': 0, 'sz': 0, 'rss': 0}

 
         slice['procs'].append(proc)
         slice['vsize'] += proc['vsize']
 
         slice['procs'].append(proc)
         slice['vsize'] += proc['vsize']

+       slice['sz'] += proc['sz']

         slice['rss'] += proc['rss']
 
         slices[proc['xid']] = slice
         slice['rss'] += proc['rss']
 
         slices[proc['xid']] = slice


@@ -370,18 +371,25 @@ def slicestat(names = None):
 
 def memtotal():
     """
 
 def memtotal():
     """

-    Returns total physical memory on the system in KiB.
+    Returns total physical and swap memory on the system in KiB.

     """
 
     """
 

+    mem = 0
+    swap = 0
+

     meminfo = open("/proc/meminfo", "r")
     meminfo = open("/proc/meminfo", "r")

-    line = meminfo.readline()
+    for line in meminfo.readlines():
+       try:
+           (name, value, kb) = line.split()
+       except:
+           continue
+       if name == "MemTotal:": 
+           mem = int(value)
+       elif name == "SwapTotal:":
+           swap = int(value)

     meminfo.close()
     meminfo.close()

-    if line[0:8] == "MemTotal":
-        # MemTotal: 255396 kB
-        (name, value, kb) = line.split()
-        return int(value)

 
 

-    return 0
+    return (mem, swap)

 
 def swap_used():
     """
 
 def swap_used():
     """


@@ -409,26 +417,33 @@ def swap_used():
 
     return 100 * total_used / total_swap
 
 
     return 100 * total_used / total_swap
 

-def summary(names = None, total_rss = memtotal()):
+def summary(slices = None, total_mem = None, total_swap = None):

     """
     Return a summary of memory usage by slice.
     """
     """
     Return a summary of memory usage by slice.
     """

-    slicelist = slicestat(names).values()
-    slicelist.sort(lambda a, b: b['rss'] - a['rss'])
-
-    table = "%-20s%10s%24s\n\n" % ("Slice", "Processes", "Memory Usage")
+    if not slices:
+       slices = slicestat()
+    slicelist = slices.values()
+    slicelist.sort(lambda a, b: b['sz'] - a['sz'])
+    if total_mem is None or total_swap is None:
+       (total_mem, total_swap) = memtotal()
+
+    table = "%-20s%10s%24s%24s\n\n" % ("Slice", "Processes", "Memory Usage", "Potential Usage")

     for slice in slicelist:
     for slice in slicelist:

-        table += "%-20s%10d%16s (%4.1f%%)\n" % \
+        table += "%-20s%10d%16s (%4.1f%%)%16s (%4.1f%%)\n" % \

                  (slice['name'], len(slice['procs']),
                   format_bytes(slice['rss'] * 1024, si = False),
                  (slice['name'], len(slice['procs']),
                   format_bytes(slice['rss'] * 1024, si = False),

-                  100. * slice['rss'] / total_rss)
+                  100. * slice['rss'] / total_mem,
+                  format_bytes(slice['sz'] * 1024, si = False),
+                  100. * slice['sz'] / (total_mem + total_swap))
+                 

 
     return table
 
 def main():
     # Defaults
     global debug, verbose, datafile
 
     return table
 
 def main():
     # Defaults
     global debug, verbose, datafile

-    global period, change_thresh, reset_thresh, reboot_thresh, min_thresh, system_slices
+    global period, change_thresh, reset_thresh, reboot_thresh, rss_min, system_slices

     # All slices
     names = []
 
     # All slices
     names = []
 


@@ -459,11 +474,11 @@ def main():
         elif opt == "--reboot-thresh":
             reboot_thresh = int(optval)
         elif opt == "--min-thresh":
         elif opt == "--reboot-thresh":
             reboot_thresh = int(optval)
         elif opt == "--min-thresh":

-            min_thresh = int(optval)
+            rss_min = int(optval)

         elif opt == "--system-slice":
             system_slices.append(optval)
         elif opt == "--status":
         elif opt == "--system-slice":
             system_slices.append(optval)
         elif opt == "--status":

-            print summary(names)
+            print summary(slicestat(names))

             sys.exit(0)
         else:
             usage()
             sys.exit(0)
         else:
             usage()


@@ -480,8 +495,8 @@ def main():
         syslog.openlog("swapmon")
         sys.stdout = sys.stderr = Logger()
 
         syslog.openlog("swapmon")
         sys.stdout = sys.stderr = Logger()
 

-    # Get total physical memory
-    total_rss = memtotal()
+    # Get total memory
+    (total_mem, total_swap) = memtotal()

 
     try:
         f = open(datafile, "r+")
 
     try:
         f = open(datafile, "r+")


@@ -490,13 +505,13 @@ def main():
         (version, slices) = pickle.load(f)
         f.close()
         # Check version of data file
         (version, slices) = pickle.load(f)
         f.close()
         # Check version of data file

-        if version != "$Id: swapmon.py,v 1.5 2006/05/09 03:23:57 mlhuang Exp $":
+        if version != "$Id: swapmon.py,v 1.10 2006/08/16 16:18:45 faiyaza Exp $":

             print "Not using old version '%s' data file %s" % (version, datafile)
             raise Exception
 
         params = {'hostname': socket.gethostname(),
                   'date': time.asctime(time.gmtime()) + " GMT",
             print "Not using old version '%s' data file %s" % (version, datafile)
             raise Exception
 
         params = {'hostname': socket.gethostname(),
                   'date': time.asctime(time.gmtime()) + " GMT",

-                  'table': summary(total_rss)}
+                  'table': summary(slices, total_mem, total_swap)}

 
         if debug:
             print rebooted_subject % params
 
         if debug:
             print rebooted_subject % params


@@ -507,7 +522,7 @@ def main():
         # Delete data file
         os.unlink(datafile)
     except Exception:
         # Delete data file
         os.unlink(datafile)
     except Exception:

-        version = "$Id: swapmon.py,v 1.5 2006/05/09 03:23:57 mlhuang Exp $"
+        version = "$Id: swapmon.py,v 1.10 2006/08/16 16:18:45 faiyaza Exp $"

         slices = {}
 
     # Query process table every 30 seconds, or when a large change in
         slices = {}
 
     # Query process table every 30 seconds, or when a large change in


@@ -526,10 +541,11 @@ def main():
         used = swap_used()
 
        for resetslice in resetlist.keys():
         used = swap_used()
 
        for resetslice in resetlist.keys():

-               resetlist[resetslice].update()
+           resetlist[resetslice].update()

        
         if last_used is None:
             last_used = used
        
         if last_used is None:
             last_used = used

+

        if verbose:
             print "%d%% swap consumed" % used
 
        if verbose:
             print "%d%% swap consumed" % used
 


@@ -547,13 +563,15 @@ def main():
                 bwlimit.run("/bin/sync; /sbin/reboot -f")
 
         elif used >= reset_thresh:
                 bwlimit.run("/bin/sync; /sbin/reboot -f")
 
         elif used >= reset_thresh:

+           if debug:
+               print "Memory used = %s" %(used)

             # Try and find a hog
             slicelist = slices.values()
             slicelist.sort(lambda a, b: b['rss'] - a['rss'])
             for slice in slicelist:
             # Try and find a hog
             slicelist = slices.values()
             slicelist.sort(lambda a, b: b['rss'] - a['rss'])
             for slice in slicelist:

-                percent = 100. * slice['rss'] / total_rss
+                percent = 100. * slice['rss'] / total_mem

 
 

-                if percent < min_thresh:
+                if slice['rss'] < rss_min:

                     continue
                
                 print "%d%% swap consumed, slice %s is using %s (%d%%) of memory" % \
                     continue
                
                 print "%d%% swap consumed, slice %s is using %s (%d%%) of memory" % \


@@ -564,11 +582,12 @@ def main():
 
                 slice['procs'].sort(lambda a, b: b['rss'] - a['rss'])
 
 
                 slice['procs'].sort(lambda a, b: b['rss'] - a['rss'])
 

-                table = "%5s %10s %10s %4s %4s %s\n\n" % ("PID", "VIRT", "RES", '%CPU', '%MEM', 'COMMAND')
+                table = "%5s %10s %10s %10s %4s %4s %s\n\n" % ("PID", "VIRT", "SZ", "RES", '%CPU', '%MEM', 'COMMAND')

                 for proc in slice['procs']:
                 for proc in slice['procs']:

-                    table += "%5s %10s %10s %4.1f %4.1f %s\n" % \
+                    table += "%5s %10s %10s %10s %4.1f %4.1f %s\n" % \

                              (proc['pid'],
                               format_bytes(proc['vsize'] * 1024, si = False),
                              (proc['pid'],
                               format_bytes(proc['vsize'] * 1024, si = False),

+                              format_bytes(proc['sz'] * 1024, si = False),

                               format_bytes(proc['rss'] * 1024, si = False),
                               proc['pcpu'], proc['pmem'], proc['cmd'])
 
                               format_bytes(proc['rss'] * 1024, si = False),
                               proc['pcpu'], proc['pmem'], proc['cmd'])
 


@@ -577,28 +596,31 @@ def main():
                           'table': table,
                           'slice': slice['name'],
                           'rss': format_bytes(slice['rss'] * 1024, si = False),
                           'table': table,
                           'slice': slice['name'],
                           'rss': format_bytes(slice['rss'] * 1024, si = False),

+                         'sz': format_bytes(slice['sz'] * 1024, si = False),

                           'percent': percent}
 
                 # Match slice name against system slice patterns
                 is_system_slice = filter(None, [re.match(pattern, slice['name']) for pattern in system_slices])
 
                 if is_system_slice: 
                           'percent': percent}
 
                 # Match slice name against system slice patterns
                 is_system_slice = filter(None, [re.match(pattern, slice['name']) for pattern in system_slices])
 
                 if is_system_slice: 

-                       if slice['name'] not in warned:
-                                       warned.append(slice['name'])
-                                       if debug:
-                                               print alarm_subject % params
-                                               print alarm_body % params
-                               else:
-                                       print "Warning slice " + slice['name']
-                                       slicemail(slice['name'], alarm_subject % params, 
-                                       alarm_body % params)
+                   # Do not reset system slices, just warn once
+                   if slice['name'] not in warned:
+                       warned.append(slice['name'])
+                       if debug:
+                           print alarm_subject % params
+                           print alarm_body % params
+                       else:
+                           print "Warning slice " + slice['name']
+                           slicemail(slice['name'], alarm_subject % params, 
+                                     alarm_body % params)

                 else:
                 else:

-                       # Reset slice
-                       if not resetlist.has_key(slice['name']):
-                               resetlist[slice['name']] = Reset(slice['name'])
-                        resetlist[slice['name']].reset(params)
+                   # Reset slice
+                   if not resetlist.has_key(slice['name']):
+                       resetlist[slice['name']] = Reset(slice['name'])
+                   resetlist[slice['name']].reset(params)
+                   slices = slicestat(names)

 
 

-        elif timer <= 0 or used >= (last_used + change_thresh):
+        if timer <= 0 or used >= (last_used + change_thresh):

             if used >= (last_used + change_thresh):
                 print "%d%% swap consumed, %d%% in last %d seconds" % \
                       (used, used - last_used, period - timer)
             if used >= (last_used + change_thresh):
                 print "%d%% swap consumed, %d%% in last %d seconds" % \
                       (used, used - last_used, period - timer)