daemon: Throttle max respawning rate.

If a monitored daemon dies quickly at startup, the system can waste a lot
of CPU time continually restarting it.  This commit prevents a given
daemon from restarting more than once every 10 seconds.

diff --git a/lib/daemon.c b/lib/daemon.c
index 292be14..78f88a0 100644 (file)
--- a/lib/daemon.c
+++ b/lib/daemon.c
@@ -323,14 +323,15 @@ should_restart(int status)
 static void
 monitor_daemon(pid_t daemon_pid)
 {
-    /* XXX Should limit the rate at which we restart the daemon. */
     /* XXX Should log daemon's stderr output at startup time. */
     const char *saved_program_name;
+    time_t last_restart;
     char *status_msg;
 
     saved_program_name = program_name;
     program_name = xasprintf("monitor(%s)", program_name);
     status_msg = xstrdup("healthy");
+    last_restart = TIME_MIN;
     for (;;) {
         int retval;
         int status;
@@ -365,6 +366,21 @@ monitor_daemon(pid_t daemon_pid)
                     }
                 }
 
+                /* Throttle restarts to no more than once every 10 seconds. */
+                if (time(NULL) < last_restart + 10) {
+                    VLOG_WARN("%s, waiting until 10 seconds since last "
+                              "restart", status_msg);
+                    for (;;) {
+                        time_t now = time(NULL);
+                        time_t wakeup = last_restart + 10;
+                        if (now >= wakeup) {
+                            break;
+                        }
+                        sleep(wakeup - now);
+                    }
+                }
+                last_restart = time(NULL);
+
                 VLOG_ERR("%s, restarting", status_msg);
                 daemon_pid = fork_and_wait_for_startup(&daemonize_fd);
                 if (!daemon_pid) {