Only restart services which are enabled.

[mom.git] / swapmon.py

diff --git a/swapmon.py b/swapmon.py
index b0c3bb3..e2215b2 100755 (executable)
--- a/swapmon.py
+++ b/swapmon.py
@@ -22,10 +22,6 @@ import pickle
 import socket
 import time
 
 import socket
 import time
 

-# util-vserver/python/vserver.py allows us to control slices directly
-# from Python
-from vserver import VServer
-

 # bwlimit exports a few useful functions like run(), get_xid(), and get_slice()
 import bwlimit
 
 # bwlimit exports a few useful functions like run(), get_xid(), and get_slice()
 import bwlimit
 


@@ -50,15 +46,6 @@ reset_thresh = 80
 # Swap utilization at which the machine is rebooted
 reboot_thresh = 95
 
 # Swap utilization at which the machine is rebooted
 reboot_thresh = 95
 

-# Time to wait before checking slice again after reset
-reset_timeout = 25
-
-# Number of strikes before killing (strike, strike, kill)
-kill_thresh = 2
-
-# Time to wait before removing slice from kill queue (probation) 
-kill_timeout = 120 
-

 # Don't email the same message more than once in the same emailtimeout interval
 email_timeout = 1800
 
 # Don't email the same message more than once in the same emailtimeout interval
 email_timeout = 1800
 


@@ -84,28 +71,6 @@ statistics are not entirely accurate due to threading.
 %(date)s %(hostname)s reboot
 """.lstrip()
 
 %(date)s %(hostname)s reboot
 """.lstrip()
 

-# Message sent after a hog is reset
-reset_subject = "pl_mom reset slice %(slice)s on %(hostname)s"
-reset_body = \
-"""
-Sometime before %(date)s, swap space was
-nearly exhausted on %(hostname)s.
-
-Slice %(slice)s was reset since it was the largest consumer of
-physical memory at %(rss)s (%(percent)4.1f%%) (%(sz)s writable).
-
-Please reply to this message explaining the nature of your experiment,
-and what you are doing to address the problem.
-
-http://summer.cs.princeton.edu/status/tabulator.cgi?table=slices/table_%(slice)s
-
-%(slice)s processes prior to reset:
-
-%(table)s
-
-%(date)s %(hostname)s reset %(slice)s
-""".lstrip()
-

 # Message sent to system slices that should not be reset
 alarm_subject = "pl_mom alarm slice %(slice)s on %(hostname)s"
 alarm_body = \
 # Message sent to system slices that should not be reset
 alarm_subject = "pl_mom alarm slice %(slice)s on %(hostname)s"
 alarm_body = \


@@ -145,108 +110,10 @@ and what you are doing to address the problem.
 %(date)s %(hostname)s reset %(slice)s
 """.lstrip()
 
 %(date)s %(hostname)s reset %(slice)s
 """.lstrip()
 

+def killsliverprocs(xid):
+    bwlimit.run("/usr/sbin/vkill -s 9 -c %s 0" % xid)    

 
 
 
 

-class Reset:
-    """
-    Keeps track of state information for resets and kills
-
-    resettimeleft - timeout before checking for next reset
-    resetcount - number of strikes 
-    killtimeleft - time out before removing from kill queue
-    {kill,reset}mail - Time of last email
-    kill - State of kill.  If slice is already being killed, wait before retry.
-    """
-
-    def __init__(self,name):
-        self.name = name
-        self.resettimeleft = reset_timeout
-        self.resetcount = 0 
-        self.resetmail = 0
-        self.killtimeleft = kill_timeout
-        self.killmail = 0
-
-    def __repr__(self):
-        return self.name
-    
-    def update(self):
-        # Count down for next check of reset slice.
-        if self.resettimeleft > 0:
-           self.resettimeleft -= 1
-           if debug and verbose:  print "%s has %s seconds in probation" \
-                                    %(self.name, self.killtimeleft)
-        if self.killtimeleft > 0:
-            # Count down kill probation timer (killtimeleft)
-            self.killtimeleft -= 1
-            if self.killtimeleft == 1:
-                print "%s is out of probation" % self.name
-        else:
-            # Once out of probation period (killtimeleft), remove strikes
-            self.resetcount = 0
-
-
-    # Check to see if a slice needs to be killed.  If it has been killed more 
-    # than kill_thresh in the probation period (kill_timeout) send an email, kill the slice.
-    def checkkill(self,params):
-        if self.killtimeleft > 0 and self.resetcount >= kill_thresh:
-            if debug:
-                print kill_subject % params
-                print kill_body % params
-            try:
-                pid = os.fork()
-                if pid == 0:
-                   print "Slice %s is being killed." % self.name   
-                   vserver = VServer(self.name)
-                   vserver.stop()
-                   os._exit(0)
-                else:
-                    os.waitpid(pid,0)
-            except Exception, err:
-                print "Warning: Exception received while killing slice %s: %s" \
-                   % (self.name, err)
-            if (time.time() - self.killmail) > email_timeout:
-                slicemail(self.name, kill_subject % params, kill_body % params)
-                print "Sending KILL email for slice %s" % self.name
-                self.killmail = time.time() 
-            return True
-        return False 
-
-    # Reset slice after checking to see if slice is out of timeout.
-    # Increment resetcount, check to see if larger than kill_thresh.
-    def reset(self, params):
-        # If its the first reset (came back after kill)
-        # or if its been reset before
-        # and we are out of the reset timeout.
-        if self.resetcount == 0 or self.resettimeleft == 0:
-            # Do we need to kill this slice?  Check history first.
-            if self.checkkill(params):  return
-            # Update counters
-            self.resetcount += 1
-            self.killtimeleft = kill_timeout
-            self.resettimeleft = reset_timeout
-            print "%s has %s seconds to die and has been reset %s times" \
-                %(self.name, self.resettimeleft, self.resetcount)
-            if debug:
-                print reset_subject % params
-                print reset_body % params
-                try:
-                    pid = os.fork()
-                    if pid == 0:  
-                        print "Resetting slice " + self.name 
-                        vserver = VServer(self.name)
-                        vserver.stop()
-                        vserver.start(wait = False)
-                        os._exit(0)
-                    else:
-                        os.waitpid(pid,0)
-                except Exception, err:
-                    print "Warning: Exception received while resetting slice %s:" \
-                        % self.name, err
-            if (time.time() - self.resetmail) > email_timeout:
-                slicemail(self.name, reset_subject % params, reset_body % params)
-                print "Sending Reset email for slice %s" % self.name
-                self.resetmail = time.time() 
-

 def usage():
     print """
 Usage: %s [OPTIONS]...
 def usage():
     print """
 Usage: %s [OPTIONS]...


@@ -269,10 +136,10 @@ def slicestat(names = None):
     """
     Get status of specified slices (if names is None or empty, all
     slices). vsize, sz, and rss are in KiB. Returns
     """
     Get status of specified slices (if names is None or empty, all
     slices). vsize, sz, and rss are in KiB. Returns

-
+    PID CONTEXT             VSZ    SZ   RSS %MEM CMD

     {xid: {'xid': slice_id,
             'name': slice_name,
     {xid: {'xid': slice_id,
             'name': slice_name,

-            'procs': [{'pid': pid, 'xid': slice_id, 'user', username, 'cmd': command,
+            'procs': [{'pid': pid, 'xid': slice_id, 'cmd': command,

                       'vsize': virtual_kib, 'sz': potential_kib, 'rss': physical_kib,
                       'pcpu': cpu_percent, 'pmem': mem_percent}]
             'vsize': total_virtual_kib,
                       'vsize': virtual_kib, 'sz': potential_kib, 'rss': physical_kib,
                       'pcpu': cpu_percent, 'pmem': mem_percent}]
             'vsize': total_virtual_kib,


@@ -283,7 +150,7 @@ def slicestat(names = None):
     # Mandatory fields. xid is a virtual field inserted by vps. Make
     # sure cmd is last so that it does not get truncated
     # automatically.
     # Mandatory fields. xid is a virtual field inserted by vps. Make
     # sure cmd is last so that it does not get truncated
     # automatically.

-    fields = ['pid', 'xid', 'user', 'vsize', 'sz', 'rss', 'pcpu', 'pmem', 'cmd']
+    fields = ['pid', 'xid', 'vsize', 'sz', 'rss', 'pmem', 'cmd']

 
     # vps inserts xid after pid in the output, but ps doesn't know
     # what the field means.
 
     # vps inserts xid after pid in the output, but ps doesn't know
     # what the field means.


@@ -295,7 +162,7 @@ def slicestat(names = None):
     # Eat the header line. vps depends on the header to figure out
     # which column is the PID column, so we can't just tell ps not to
     # print it.
     # Eat the header line. vps depends on the header to figure out
     # which column is the PID column, so we can't just tell ps not to
     # print it.

-    for line in bwlimit.run("/usr/sbin/vps -e -o " + ",".join(ps_fields))[1:]:
+    for line in bwlimit.run("/usr/sbin/vps -e -o " + ":16,".join(ps_fields))[1:]:

         # Chomp newline
         line = line.strip()
 
         # Chomp newline
         line = line.strip()
 


@@ -320,14 +187,16 @@ def slicestat(names = None):
                 except ValueError:
                     pass
 
                 except ValueError:
                     pass
 

-        # vps sometimes prints ERR instead of a context ID if it
+        # vps sometimes prints ERR or the name of the slice 
+            # instead of a context ID if it

         # cannot identify the context of an orphaned (usually dying)
         # process. Skip these processes.
         # cannot identify the context of an orphaned (usually dying)
         # process. Skip these processes.

-        if type(proc['xid']) != int:
+        if (type(proc['xid']) != int) or (type(proc['vsize']) !=int):

             continue
 
         # Assign (pl_)sshd processes to slice instead of root
         m = re.search(r"sshd: ([a-zA-Z_]+)", proc['cmd'])
             continue
 
         # Assign (pl_)sshd processes to slice instead of root
         m = re.search(r"sshd: ([a-zA-Z_]+)", proc['cmd'])

+

         if m is not None:
             xid = bwlimit.get_xid(m.group(1))
             if xid is not None:
         if m is not None:
             xid = bwlimit.get_xid(m.group(1))
             if xid is not None:


@@ -351,8 +220,10 @@ def slicestat(names = None):
         proc['rss'] += 12
 
         # Include additional page table overhead
         proc['rss'] += 12
 
         # Include additional page table overhead

-        if proc['vsize'] > 4096:
-            proc['rss'] += 4 * ((proc['vsize'] - 1) / 4096)
+        try:
+            if proc['vsize'] > 4096:
+                proc['rss'] += 4 * ((proc['vsize'] - 1) / 4096)
+        except: pass

 
         if slices.has_key(proc['xid']):
             slice = slices[proc['xid']]
 
         if slices.has_key(proc['xid']):
             slice = slices[proc['xid']]


@@ -365,7 +236,7 @@ def slicestat(names = None):
         slice['rss'] += proc['rss']
 
         slices[proc['xid']] = slice
         slice['rss'] += proc['rss']
 
         slices[proc['xid']] = slice

-
+       

     return slices
 
 def memtotal():
     return slices
 
 def memtotal():


@@ -439,12 +310,11 @@ def formtable(slice, percent):
     table = "%5s %10s %10s %10s %4s %4s %s\n\n" % \
         ("PID", "VIRT", "SZ", "RES", '%CPU', '%MEM', 'COMMAND')
     for proc in slice['procs']:
     table = "%5s %10s %10s %10s %4s %4s %s\n\n" % \
         ("PID", "VIRT", "SZ", "RES", '%CPU', '%MEM', 'COMMAND')
     for proc in slice['procs']:

-        table += "%5s %10s %10s %10s %4.1f %4.1f %s\n" % \
+        table += "%5s %10s %10s %10s %4.1f %s\n" % \

             (proc['pid'],
             format_bytes(proc['vsize'] * 1024, si = False),
             format_bytes(proc['sz'] * 1024, si = False),
             format_bytes(proc['rss'] * 1024, si = False),
             (proc['pid'],
             format_bytes(proc['vsize'] * 1024, si = False),
             format_bytes(proc['sz'] * 1024, si = False),
             format_bytes(proc['rss'] * 1024, si = False),

-            proc['pcpu'],

             proc['pmem'],
             proc['cmd'])
     
             proc['pmem'],
             proc['cmd'])
     


@@ -508,6 +378,11 @@ def main():
     global period, change_thresh, reset_thresh, reboot_thresh, rss_min, system_slices
     # All slices
     names = []
     global period, change_thresh, reset_thresh, reboot_thresh, rss_min, system_slices
     # All slices
     names = []

+    timer = period
+    last_used = None
+    used = None
+    warned = []
+    emailed = {}

 
     try:
         longopts = ["debug", "verbose", "file=", "slice=", "status", "help"]
 
     try:
         longopts = ["debug", "verbose", "file=", "slice=", "status", "help"]


@@ -563,30 +438,11 @@ def main():
 
     # Query process table every 30 seconds, or when a large change in
     # swap utilization is detected.
 
     # Query process table every 30 seconds, or when a large change in
     # swap utilization is detected.

-    timer = period
-    last_used = None
-    used = None
-
-    # System slices that we have warned but could not reset
-    warned = []
-
-    # Slices that were reset
-    resetlist = {}

 
     while True:
         used = swap_used()
         if last_used is None:  last_used = used
 
     while True:
         used = swap_used()
         if last_used is None:  last_used = used

-        
-        # If we've reset you recently, update timers.
-        for resetslice in resetlist.keys(): 
-            resetlist[resetslice].update()
-            # If you've been good, remove you from our list.
-            if resetlist[resetslice].killtimeleft == 0 and \
-            resetlist[resetslice].resettimeleft == 0:
-                del resetlist[resetslice]
-
-        if verbose:  print "%d%% swap consumed" % used
-    
+   

         if used >= reboot_thresh:
             # Dump slice state before rebooting
             writedat(slices)    
         if used >= reboot_thresh:
             # Dump slice state before rebooting
             writedat(slices)    


@@ -626,11 +482,17 @@ def main():
                               alarm_body % params)
                 else:
                     # Reset slice
                               alarm_body % params)
                 else:
                     # Reset slice

-                    if not resetlist.has_key(slice['name']):
-                        resetlist[slice['name']] = Reset(slice['name'])
-                    resetlist[slice['name']].reset(params)
+                    if not debug:
+                        if emailed.get(slice['name'], (time.time() + email_timeout + 1)) > (time.time() + email_timeout): 
+                            slicemail(slice['name'], kill_subject % params, kill_body % params)
+                            emailed[slice['name']] = time.time()
+                    else:
+                        print kill_subject % params
+                        print kill_body % params
+                    print "Killing procs in %s" % slice['name']
+                    killsliverprocs(slice['xid'])

 
 

-        # wait period vefore recalculating swap.  If in danger, recalc.
+        # wait period before recalculating swap.  If in danger, recalc.

         if timer <= 0 or used >= (last_used + change_thresh):
             if used >= (last_used + change_thresh):
                 print "%d%% swap consumed, %d%% in last %d seconds" % \
         if timer <= 0 or used >= (last_used + change_thresh):
             if used >= (last_used + change_thresh):
                 print "%d%% swap consumed, %d%% in last %d seconds" % \