On vserver create, the slice goes to init 1 which causes an extra column to get print...

[mom.git] / swapmon.py

diff --git a/swapmon.py b/swapmon.py
index f15879f..35a98e8 100755 (executable)
--- a/swapmon.py
+++ b/swapmon.py
@@ -10,7 +10,7 @@
 # Faiyaz Ahmed <faiyaza@cs.princeton.edu>
 # Copyright (C) 2004-2006 The Trustees of Princeton University
 #
 # Faiyaz Ahmed <faiyaza@cs.princeton.edu>
 # Copyright (C) 2004-2006 The Trustees of Princeton University
 #

-# $Id: swapmon.py,v 1.11 2006/12/02 19:11:47 mlhuang Exp $
+# $Id$

 #
 
 import syslog
 #
 
 import syslog


@@ -35,8 +35,8 @@ from pl_mom import *
 # Defaults
 debug = False
 verbose = 0
 # Defaults
 debug = False
 verbose = 0

-datafile = "/var/lib/misc/swapmon.dat"
-
+DATAFILE = "/var/lib/misc/swapmon.dat"
+VERSION = "$Id$"

 # Seconds between process analysis
 period = 30
 
 # Seconds between process analysis
 period = 30
 


@@ -51,7 +51,7 @@ reset_thresh = 80
 reboot_thresh = 95
 
 # Time to wait before checking slice again after reset
 reboot_thresh = 95
 
 # Time to wait before checking slice again after reset

-reset_timeout = 15
+reset_timeout = 25

 
 # Number of strikes before killing (strike, strike, kill)
 kill_thresh = 2
 
 # Number of strikes before killing (strike, strike, kill)
 kill_thresh = 2


@@ -97,6 +97,8 @@ physical memory at %(rss)s (%(percent)4.1f%%) (%(sz)s writable).
 Please reply to this message explaining the nature of your experiment,
 and what you are doing to address the problem.
 
 Please reply to this message explaining the nature of your experiment,
 and what you are doing to address the problem.
 

+http://summer.cs.princeton.edu/status/tabulator.cgi?table=slices/table_%(slice)s
+

 %(slice)s processes prior to reset:
 
 %(table)s
 %(slice)s processes prior to reset:
 
 %(table)s


@@ -188,14 +190,17 @@ class Reset:
     def checkkill(self,params):
         if self.killtimeleft > 0 and self.resetcount >= kill_thresh:
             if debug:
     def checkkill(self,params):
         if self.killtimeleft > 0 and self.resetcount >= kill_thresh:
             if debug:

-                                print kill_subject % params
-                                print kill_body % params
+                print kill_subject % params
+                print kill_body % params

             try:
                 pid = os.fork()
                 if pid == 0:
                    print "Slice %s is being killed." % self.name   
                    vserver = VServer(self.name)
                    vserver.stop()
             try:
                 pid = os.fork()
                 if pid == 0:
                    print "Slice %s is being killed." % self.name   
                    vserver = VServer(self.name)
                    vserver.stop()

+                   # ignore initscripts.  Don't run anything at start.
+                   vserver.INITSCRIPTS = []
+                   vserver.start()

                    os._exit(0)
                 else:
                     os.waitpid(pid,0)
                    os._exit(0)
                 else:
                     os.waitpid(pid,0)


@@ -227,18 +232,18 @@ class Reset:
             if debug:
                 print reset_subject % params
                 print reset_body % params
             if debug:
                 print reset_subject % params
                 print reset_body % params

-                try:
-                    pid = os.fork()
-                    if pid == 0:  
-                        print "Resetting slice " + self.name 
-                        vserver = VServer(self.name)
-                        vserver.stop()
-                        vserver.start(wait = False)
-                        os._exit(0)
-                    else:
-                        os.waitpid(pid,0)
-                except Exception, err:
-                    print "Warning: Exception received while resetting slice %s:" \
+            try:
+                pid = os.fork()
+                if pid == 0:  
+                    print "Resetting slice " + self.name 
+                    vserver = VServer(self.name)
+                    vserver.stop()
+                    vserver.start()
+                    os._exit(0)
+                else:
+                    os.waitpid(pid,0)
+            except Exception, err:
+                print "Warning: Exception received while resetting slice %s:" \

                         % self.name, err
             if (time.time() - self.resetmail) > email_timeout:
                 slicemail(self.name, reset_subject % params, reset_body % params)
                         % self.name, err
             if (time.time() - self.resetmail) > email_timeout:
                 slicemail(self.name, reset_subject % params, reset_body % params)


@@ -261,27 +266,27 @@ Options:
         --system-slice=SLICE    System slice that should not be reset
         --status                Print memory usage statistics and exit
         -h, --help              This message
         --system-slice=SLICE    System slice that should not be reset
         --status                Print memory usage statistics and exit
         -h, --help              This message

-""".lstrip() % (sys.argv[0], debug, verbose, datafile, format_period(period))
+""".lstrip() % (sys.argv[0], debug, verbose, DATAFILE, format_period(period))

 
 def slicestat(names = None):
     """
     Get status of specified slices (if names is None or empty, all
     slices). vsize, sz, and rss are in KiB. Returns
 
 def slicestat(names = None):
     """
     Get status of specified slices (if names is None or empty, all
     slices). vsize, sz, and rss are in KiB. Returns

-
+    PID CONTEXT             VSZ    SZ   RSS %MEM CMD

     {xid: {'xid': slice_id,
     {xid: {'xid': slice_id,

-           'name': slice_name,
-           'procs': [{'pid': pid, 'xid': slice_id, 'user', username, 'cmd': command,
+            'name': slice_name,
+            'procs': [{'pid': pid, 'xid': slice_id, 'cmd': command,

                       'vsize': virtual_kib, 'sz': potential_kib, 'rss': physical_kib,
                       'pcpu': cpu_percent, 'pmem': mem_percent}]
                       'vsize': virtual_kib, 'sz': potential_kib, 'rss': physical_kib,
                       'pcpu': cpu_percent, 'pmem': mem_percent}]

-           'vsize': total_virtual_kib,
-       'sz': total_potential_kib,
-           'rss': total_physical_kib}}
+            'vsize': total_virtual_kib,
+            'sz': total_potential_kib,
+            'rss': total_physical_kib}}

     """
     
     # Mandatory fields. xid is a virtual field inserted by vps. Make
     # sure cmd is last so that it does not get truncated
     # automatically.
     """
     
     # Mandatory fields. xid is a virtual field inserted by vps. Make
     # sure cmd is last so that it does not get truncated
     # automatically.

-    fields = ['pid', 'xid', 'user', 'vsize', 'sz', 'rss', 'pcpu', 'pmem', 'cmd']
+    fields = ['pid', 'xid', 'vsize', 'sz', 'rss', 'pmem', 'cmd']

 
     # vps inserts xid after pid in the output, but ps doesn't know
     # what the field means.
 
     # vps inserts xid after pid in the output, but ps doesn't know
     # what the field means.


@@ -293,7 +298,7 @@ def slicestat(names = None):
     # Eat the header line. vps depends on the header to figure out
     # which column is the PID column, so we can't just tell ps not to
     # print it.
     # Eat the header line. vps depends on the header to figure out
     # which column is the PID column, so we can't just tell ps not to
     # print it.

-    for line in bwlimit.run("/usr/sbin/vps -e -o " + ",".join(ps_fields))[1:]:
+    for line in bwlimit.run("/usr/sbin/vps -e -o " + ":16,".join(ps_fields))[1:]:

         # Chomp newline
         line = line.strip()
 
         # Chomp newline
         line = line.strip()
 


@@ -318,14 +323,16 @@ def slicestat(names = None):
                 except ValueError:
                     pass
 
                 except ValueError:
                     pass
 

-        # vps sometimes prints ERR instead of a context ID if it
+        # vps sometimes prints ERR or the name of the slice 
+            # instead of a context ID if it

         # cannot identify the context of an orphaned (usually dying)
         # process. Skip these processes.
         # cannot identify the context of an orphaned (usually dying)
         # process. Skip these processes.

-        if type(proc['xid']) != int:
+        if (type(proc['xid']) != int) or (type(proc['vsize']) !=int):

             continue
 
         # Assign (pl_)sshd processes to slice instead of root
         m = re.search(r"sshd: ([a-zA-Z_]+)", proc['cmd'])
             continue
 
         # Assign (pl_)sshd processes to slice instead of root
         m = re.search(r"sshd: ([a-zA-Z_]+)", proc['cmd'])

+

         if m is not None:
             xid = bwlimit.get_xid(m.group(1))
             if xid is not None:
         if m is not None:
             xid = bwlimit.get_xid(m.group(1))
             if xid is not None:


@@ -349,8 +356,10 @@ def slicestat(names = None):
         proc['rss'] += 12
 
         # Include additional page table overhead
         proc['rss'] += 12
 
         # Include additional page table overhead

-        if proc['vsize'] > 4096:
-            proc['rss'] += 4 * ((proc['vsize'] - 1) / 4096)
+        try:
+            if proc['vsize'] > 4096:
+                proc['rss'] += 4 * ((proc['vsize'] - 1) / 4096)
+        except: pass

 
         if slices.has_key(proc['xid']):
             slice = slices[proc['xid']]
 
         if slices.has_key(proc['xid']):
             slice = slices[proc['xid']]


@@ -363,17 +372,15 @@ def slicestat(names = None):
         slice['rss'] += proc['rss']
 
         slices[proc['xid']] = slice
         slice['rss'] += proc['rss']
 
         slices[proc['xid']] = slice

-
+       

     return slices
 
 def memtotal():
     """
     Returns total physical and swap memory on the system in KiB.
     """
     return slices
 
 def memtotal():
     """
     Returns total physical and swap memory on the system in KiB.
     """

-

     mem = 0
     swap = 0
     mem = 0
     swap = 0

-

     meminfo = open("/proc/meminfo", "r")
     for line in meminfo.readlines():
         try:
     meminfo = open("/proc/meminfo", "r")
     for line in meminfo.readlines():
         try:


@@ -385,17 +392,14 @@ def memtotal():
         elif name == "SwapTotal:":
             swap = int(value)
     meminfo.close()
         elif name == "SwapTotal:":
             swap = int(value)
     meminfo.close()

-

     return (mem, swap)
 
 def swap_used():
     """
     Returns swap utilization on the system as a whole percentage (0-100).
     """
     return (mem, swap)
 
 def swap_used():
     """
     Returns swap utilization on the system as a whole percentage (0-100).
     """

-

     total_swap = 0
     total_used = 0
     total_swap = 0
     total_used = 0

-

     try:
         swaps = open("/proc/swaps", "r")
         # Eat header line
     try:
         swaps = open("/proc/swaps", "r")
         # Eat header line


@@ -409,17 +413,17 @@ def swap_used():
                 total_used += int(used)
             except ValueEror, err:
                 pass
                 total_used += int(used)
             except ValueEror, err:
                 pass

-    except (IOError, KeyError), err:
-        pass
+    except (IOError, KeyError), err:  pass

 
 

-    return 100 * total_used / total_swap
+    swapused = 100 * total_used / total_swap
+    if debug: print "%s percent swap used" % swapused
+    return swapused

 
 def summary(slices = None, total_mem = None, total_swap = None):
     """
     Return a summary of memory usage by slice.
     """
 
 def summary(slices = None, total_mem = None, total_swap = None):
     """
     Return a summary of memory usage by slice.
     """

-    if not slices:
-        slices = slicestat()
+    if not slices:  slices = slicestat()

     slicelist = slices.values()
     slicelist.sort(lambda a, b: b['sz'] - a['sz'])
     if total_mem is None or total_swap is None:
     slicelist = slices.values()
     slicelist.sort(lambda a, b: b['sz'] - a['sz'])
     if total_mem is None or total_swap is None:


@@ -433,13 +437,80 @@ def summary(slices = None, total_mem = None, total_swap = None):
                   100. * slice['rss'] / total_mem,
                   format_bytes(slice['sz'] * 1024, si = False),
                   100. * slice['sz'] / (total_mem + total_swap))
                   100. * slice['rss'] / total_mem,
                   format_bytes(slice['sz'] * 1024, si = False),
                   100. * slice['sz'] / (total_mem + total_swap))

-          
-

     return table
 
     return table
 

+def formtable(slice, percent):
+    '''
+    Makes pretty message to email with human readable ps values.
+    '''
+    table = "%5s %10s %10s %10s %4s %4s %s\n\n" % \
+        ("PID", "VIRT", "SZ", "RES", '%CPU', '%MEM', 'COMMAND')
+    for proc in slice['procs']:
+        table += "%5s %10s %10s %10s %4.1f %s\n" % \
+            (proc['pid'],
+            format_bytes(proc['vsize'] * 1024, si = False),
+            format_bytes(proc['sz'] * 1024, si = False),
+            format_bytes(proc['rss'] * 1024, si = False),
+            proc['pmem'],
+            proc['cmd'])
+    
+    prettytable = {'hostname': socket.gethostname(),
+             'date': time.asctime(time.gmtime()) + " GMT",
+             'table': table,
+             'slice': slice['name'],
+             'rss': format_bytes(slice['rss'] * 1024, si = False),
+             'sz': format_bytes(slice['sz'] * 1024, si = False),
+             'percent': percent}
+    return prettytable
+
+def readdat():
+    '''
+    Return dictionary of vps (slicestat) from datfile left behind by OOM
+    before rebooting.  If none file, just grab the latest dict (slicestat)
+    and return that.  If dat file found, means we rebooted, send an email to 
+    pl_mom@pl.
+    '''
+    try:
+        f = open(DATAFILE, "r+")
+        if verbose:
+            print "Loading %s" % DATAFILE
+        (v, slices) = pickle.load(f)
+        f.close()
+        # Check version of data file
+        if v != VERSION:
+            print "Not using old version '%s' data file %s" % (v, DATAFILE)
+            raise Exception
+
+        params = {'hostname': socket.gethostname(),
+                  'date': time.asctime(time.gmtime()) + " GMT",
+                  'table': summary(slices, total_mem, total_swap)}
+        if debug:
+            print rebooted_subject % params
+            print rebooted_body % params
+        else:
+            slicemail(None, rebooted_subject % params, rebooted_body % params)
+
+        # Delete data file
+        os.unlink(DATAFILE)
+    except Exception:
+        slices = slicestat()
+
+    return slices
+
+
+def writedat(slices):
+    """
+    Write (slices) to pickled datfile.
+    """
+    if verbose:  print "Saving %s" % DATAFILE
+    f = open(DATAFILE, "w")
+    pickle.dump((VERSION, slices), f)
+    f.close()
+
+

 def main():
     # Defaults
 def main():
     # Defaults

-    global debug, verbose, datafile
+    global debug, verbose, DATAFILE, VERSION 

     global period, change_thresh, reset_thresh, reboot_thresh, rss_min, system_slices
     # All slices
     names = []
     global period, change_thresh, reset_thresh, reboot_thresh, rss_min, system_slices
     # All slices
     names = []


@@ -459,7 +530,7 @@ def main():
         elif opt == "-v" or opt == "--verbose":
             verbose += 1
         elif opt == "-f" or opt == "--file":
         elif opt == "-v" or opt == "--verbose":
             verbose += 1
         elif opt == "-f" or opt == "--file":

-            datafile = optval
+            DATAFILE = optval

         elif opt == "-s" or opt == "--slice":
             names.append(optval)
         elif opt == "-p" or opt == "--period":
         elif opt == "-s" or opt == "--slice":
             names.append(optval)
         elif opt == "-p" or opt == "--period":


@@ -494,33 +565,7 @@ def main():
 
     # Get total memory
     (total_mem, total_swap) = memtotal()
 
     # Get total memory
     (total_mem, total_swap) = memtotal()

-
-    try:
-        f = open(datafile, "r+")
-        if verbose:
-            print "Loading %s" % datafile
-        (version, slices) = pickle.load(f)
-        f.close()
-        # Check version of data file
-        if version != "$Id: swapmon.py,v 1.11 2006/12/02 19:11:47 mlhuang Exp $":
-            print "Not using old version '%s' data file %s" % (version, datafile)
-            raise Exception
-
-        params = {'hostname': socket.gethostname(),
-                  'date': time.asctime(time.gmtime()) + " GMT",
-                  'table': summary(slices, total_mem, total_swap)}
-
-        if debug:
-            print rebooted_subject % params
-            print rebooted_body % params
-        else:
-            slicemail(None, rebooted_subject % params, rebooted_body % params)
-
-        # Delete data file
-        os.unlink(datafile)
-    except Exception:
-        version = "$Id: swapmon.py,v 1.11 2006/12/02 19:11:47 mlhuang Exp $"
-        slices = {}
+    slices = readdat()

 
     # Query process table every 30 seconds, or when a large change in
     # swap utilization is detected.
 
     # Query process table every 30 seconds, or when a large change in
     # swap utilization is detected.


@@ -536,95 +581,70 @@ def main():
 
     while True:
         used = swap_used()
 
     while True:
         used = swap_used()

-
-    for resetslice in resetlist.keys():
-        resetlist[resetslice].update()
+        if last_used is None:  last_used = used
+        
+        # If we've reset you recently, update timers.
+        for resetslice in resetlist.keys(): 
+            resetlist[resetslice].update()
+            # If you've been good, remove you from our list.
+            if resetlist[resetslice].killtimeleft == 0 and \
+            resetlist[resetslice].resettimeleft == 0:
+                del resetlist[resetslice]
+    
+        if used >= reboot_thresh:
+            # Dump slice state before rebooting
+            writedat(slices)    
+            # Goodbye, cruel world
+            print "%d%% swap consumed, rebooting" % used
+            if not debug:  bwlimit.run("/bin/sync; /sbin/reboot -f")
+        elif used >= reset_thresh:
+            # Try and find a hog
+            slicelist = slices.values()
+            # Puts largest on top.
+            slicelist.sort(lambda a, b: b['rss'] - a['rss'])
+            for slice in slicelist:
+                percent = 100. * slice['rss'] / total_mem
+                if slice['rss'] < rss_min: continue
+                print "%d%% swap consumed, slice %s is using %s (%d%%) of memory" % \
+                    (used,
+                    slice['name'],
+                    format_bytes(slice['rss'] * 1024, si = False),
+                    percent)
+                slice['procs'].sort(lambda a, b: b['rss'] - a['rss'])
+                # Make a pretty table.
+                params = formtable(slice, percent)
+                # Match slice name against system slice patterns
+                is_system_slice = filter(None, 
+                    [re.match(pattern, slice['name']) for pattern in system_slices])

     
     

-        if last_used is None:
-            last_used = used
-
-    if verbose:
-            print "%d%% swap consumed" % used
-
-    if used >= reboot_thresh:
-        # Dump slice state before rebooting
-        if verbose:  print "Saving %s" % datafile
-        f = open(datafile, "w")
-        pickle.dump((version, slices), f)
-        f.close()
-
-        # Goodbye, cruel world
-        print "%d%% swap consumed, rebooting" % used
-        if not debug:  bwlimit.run("/bin/sync; /sbin/reboot -f")
-
-    elif used >= reset_thresh:
-        if debug:  print "Memory used = %s" %(used)
-        # Try and find a hog
-        slicelist = slices.values()
-        slicelist.sort(lambda a, b: b['rss'] - a['rss'])
-        for slice in slicelist:
-            percent = 100. * slice['rss'] / total_mem
-            if slice['rss'] < rss_min: continue
-            print "%d%% swap consumed, slice %s is using %s (%d%%) of memory" % \
-                (used,
-                slice['name'],
-                format_bytes(slice['rss'] * 1024, si = False),
-                percent)
-
-        slice['procs'].sort(lambda a, b: b['rss'] - a['rss'])
-        table = "%5s %10s %10s %10s %4s %4s %s\n\n" % \
-            ("PID", "VIRT", "SZ", "RES", '%CPU', '%MEM', 'COMMAND')
-        for proc in slice['procs']:
-            table += "%5s %10s %10s %10s %4.1f %4.1f %s\n" % \
-                (proc['pid'],
-                format_bytes(proc['vsize'] * 1024, si = False),
-                format_bytes(proc['sz'] * 1024, si = False),
-                format_bytes(proc['rss'] * 1024, si = False),
-                proc['pcpu'],
-                proc['pmem'],
-                proc['cmd'])
-
-            params = {'hostname': socket.gethostname(),
-                'date': time.asctime(time.gmtime()) + " GMT",
-                'table': table,
-                'slice': slice['name'],
-                'rss': format_bytes(slice['rss'] * 1024, si = False),
-                'sz': format_bytes(slice['sz'] * 1024, si = False),
-                'percent': percent}
-
-            # Match slice name against system slice patterns
-            is_system_slice = filter(None, 
-                                [re.match(pattern, slice['name']) for pattern in system_slices])
-
-            if is_system_slice: 

                 # Do not reset system slices, just warn once
                 # Do not reset system slices, just warn once

-                if slice['name'] not in warned:
-                     warned.append(slice['name'])
-                if debug:
-                    print alarm_subject % params
-                    print alarm_body % params
+                if is_system_slice: 
+                    if slice['name'] not in warned:
+                        warned.append(slice['name'])
+                        print "Warning slice " + slice['name']
+                        if debug:
+                            print alarm_subject % params
+                            print alarm_body % params
+                        else:
+                            slicemail(slice['name'], alarm_subject % params, 
+                              alarm_body % params)

                 else:
                 else:

-                    print "Warning slice " + slice['name']
-                    slicemail(slice['name'], alarm_subject % params, 
-                        alarm_body % params)
-            else:
-                # Reset slice
-                if not resetlist.has_key(slice['name']):
-                    resetlist[slice['name']] = Reset(slice['name'])
-                resetlist[slice['name']].reset(params)
-                slices = slicestat(names)
+                    # Reset slice
+                    if not resetlist.has_key(slice['name']):
+                        resetlist[slice['name']] = Reset(slice['name'])
+                    resetlist[slice['name']].reset(params)

 
 

+        # wait period vefore recalculating swap.  If in danger, recalc.

         if timer <= 0 or used >= (last_used + change_thresh):
             if used >= (last_used + change_thresh):
                 print "%d%% swap consumed, %d%% in last %d seconds" % \
         if timer <= 0 or used >= (last_used + change_thresh):
             if used >= (last_used + change_thresh):
                 print "%d%% swap consumed, %d%% in last %d seconds" % \

-                      (used, used - last_used, period - timer)
+                    (used, used - last_used, period - timer)

             # Get slice state
             slices = slicestat(names)
             # Reset timer
             timer = period
             # Keep track of large changes in swap utilization
             last_used = used
             # Get slice state
             slices = slicestat(names)
             # Reset timer
             timer = period
             # Keep track of large changes in swap utilization
             last_used = used

-

         timer -= 1
         time.sleep(1)
 
         timer -= 1
         time.sleep(1)