Added state info so same reset email does not keep getting sent. If slice is already

being killed, will wait for a timeout before trying to kill again.

diff --git a/swapmon.py b/swapmon.py
index a29a59e..83b4ec1 100755 (executable)
--- a/swapmon.py
+++ b/swapmon.py
@@ -45,19 +45,22 @@ period = 30
 change_thresh = 5
 
 # Swap utilization at which the largest consumer of physical memory is reset
-reset_thresh = 75
+reset_thresh = 80
 
 # Swap utilization at which the machine is rebooted
 reboot_thresh = 95
 
 # Time to wait before checking slice again after reset
-reset_timeout = 10
+reset_timeout = 15
 
 # Number of strikes before killing (strike, strike, kill)
 kill_thresh = 2
 
-# Time to wait before removing slice from kill queue 
-kill_timeout = 30 
+# Time to wait before removing slice from kill queue (probation) 
+kill_timeout = 120 
+
+# Don't email the same message more than once in the same emailtimeout interval
+email_timeout = 1800
 
 # Minimum physical memory utilization to be considered the largest consumer
 min_thresh = 10
@@ -146,14 +149,19 @@ class Reset:
 
        resettimeleft - timeout before checking for next reset
        resetcount - number of strikes 
-       killtimeleft - time out before removing from kill queue 
+       killtimeleft - time out before removing from kill queue
+       {kill,reset}mail - Time of last email
+       kill - State of kill.  If slice is already being killed, wait before retry.
        """
 
        def __init__(self,name):
                self.name = name
                self.resettimeleft = reset_timeout
                self.resetcount = 0 
+               self.resetmail = 0
+               self.kill = False
                self.killtimeleft = kill_timeout
+               self.killmail = 0
 
        def __repr__(self):
                return self.name
@@ -163,36 +171,41 @@ class Reset:
                        if self.resettimeleft > 0:
                         self.resettimeleft -= 1
                         if debug and verbose:
-                                       print "%s has %s seconds to die and has been reset %s times" \
-                                       %(self.name, self.resettimeleft, self.resetcount)
                                        print "%s has %s seconds in probation" \
                                        %(self.name, self.killtimeleft)
                if self.killtimeleft > 0:
                        # Count down kill probation timer (killtimeleft)
                        self.killtimeleft -= 1
+                       if self.killtimeleft == 1:
+                               print "%s is out of probation" % self.name
                else:
-                       # Once out of probation period (killtimeleft), remove strike
+                       # Once out of probation period (killtimeleft), remove strikes
                        self.resetcount = 0
+                       self.kill = False
 
 
        # Check to see if a slice needs to be killed.  If it has rules more than kill_thresh in 
        # the probation period (kill_timeout) send an email, kill the slice.
        def checkkill(self,params):
-               if self.killtimeleft > 0 and self.resetcount >= kill_thresh:
+               if self.killtimeleft > 0 and self.resetcount >= kill_thresh and \
+               self.kill == False:
+                       self.kill = True
                        if debug:
                                 print kill_subject % params
                                 print kill_body % params
                        try:
-                               pid = os.fork()
-                               if pid == 0:
+                               self.pid = os.fork()
+                               if self.pid == 0:
                                        print "Slice %s is being killed." % self.name   
-                                       vserver = VServer(self.name)
-                                       vserver.stop()
-                               else:
-                                       os.waitpid(pid,0)
+                                               vserver = VServer(self.name)
+                                               vserver.stop()
+                                       os._exit(0)
                        except Exception, err:
-                                print "Warning: Exception received while killing slice %s: %s" % self.name, err
-                       slicemail(self.name, kill_subject % params, kill_body % params)
+                                       print "Warning: Exception received while killing slice %s: %s" % self.name, err
+                       if (time.time() - self.killmail) > email_timeout:
+                               slicemail(self.name, kill_subject % params, kill_body % params)
+                               print "Sending KILL email for slice %s" % self.name
+                               self.killmail = time.time() 
                        return True
                return False 
 
@@ -209,24 +222,26 @@ class Reset:
                        self.resetcount += 1
                        self.killtimeleft = kill_timeout
                        self.resettimeleft = reset_timeout
+                       print "%s has %s seconds to die and has been reset %s times" \
+                               %(self.name, self.resettimeleft, self.resetcount)
                        if debug:
                                print reset_subject % params
                                print reset_body % params
                        try:
                                pid = os.fork()
                                if pid == 0:
-                                       print "Resetting slice " + self.name 
-                                       vserver = VServer(self.name)
-                                       vserver.stop()
-                                       vserver.start(wait = False)
-                                       os._exit(0)
-                               else:
-                                       os.waitpid(pid, 0)
+                                               print "Resetting slice " + self.name 
+                                               vserver = VServer(self.name)
+                                               vserver.stop()
+                                               vserver.start(wait = False)
+                                               os._exit(0)
                        except Exception, err:
-                               print "Warning: Exception received while resetting slice %s:" \
+                                       print "Warning: Exception received while resetting slice %s:" \
                                        % self.name, err
-                       slicemail(self.name, reset_subject % params, reset_body % params)
-
+                       if (time.time() - self.resetmail) > email_timeout:
+                                       slicemail(self.name, reset_subject % params, reset_body % params)
+                               print "Sending Reset email for slice %s" % self.name
+                               self.resetmail = time.time() 
 
 
 def usage():
@@ -562,7 +577,7 @@ def main():
                 # Match slice name against system slice patterns
                 is_system_slice = filter(None, [re.match(pattern, slice['name']) for pattern in system_slices])
 
-                if is_system_slice:
+                if is_system_slice and params['rss'] > 100:
                     # Do not reset system slices, just warn once
                     if slice['name'] not in warned:
                         warned.append(slice['name'])