timeval: Take a backtrace on each SIGALRM.
[sliver-openvswitch.git] / lib / timeval.c
index ea05017..e507140 100644 (file)
@@ -75,9 +75,8 @@ struct trace {
 };
 
 #define MAX_TRACES 50
-static struct unixctl_conn *backtrace_conn = NULL;
-static struct trace *traces = NULL;
-static size_t n_traces = 0;
+static struct trace traces[MAX_TRACES];
+static size_t trace_head = 0;
 
 static void set_up_timer(void);
 static void set_up_signal(int flags);
@@ -91,7 +90,6 @@ static struct rusage *get_recent_rusage(void);
 static void refresh_rusage(void);
 static void timespec_add(struct timespec *sum,
                          const struct timespec *a, const struct timespec *b);
-static void trace_run(void);
 static unixctl_cb_func backtrace_cb;
 
 /* Initializes the timetracking module, if not already initialized. */
@@ -100,16 +98,13 @@ time_init(void)
 {
     static bool inited;
 
-    /* The best place to do this is probably a timeval_run() function.
-     * However, none exists and this function is usually so fast that doing it
-     * here seems fine for now. */
-    trace_run();
-
     if (inited) {
         return;
     }
     inited = true;
 
+    memset(traces, 0, sizeof traces);
+
     if (HAVE_EXECINFO_H && CACHE_TIME) {
         unixctl_command_register("backtrace", "", 0, 0, backtrace_cb, NULL);
     }
@@ -331,14 +326,16 @@ int
 time_poll(struct pollfd *pollfds, int n_pollfds, long long int timeout_when,
           int *elapsed)
 {
-    static long long int last_wakeup;
+    static long long int last_wakeup = 0;
     long long int start;
     sigset_t oldsigs;
     bool blocked;
     int retval;
 
     time_refresh();
-    log_poll_interval(last_wakeup);
+    if (last_wakeup) {
+        log_poll_interval(last_wakeup);
+    }
     coverage_clear();
     start = time_msec();
     blocked = false;
@@ -375,7 +372,7 @@ time_poll(struct pollfd *pollfds, int n_pollfds, long long int timeout_when,
             break;
         }
 
-        if (!blocked && !CACHE_TIME) {
+        if (!blocked && CACHE_TIME) {
             block_sigalrm(&oldsigs);
             blocked = true;
         }
@@ -396,10 +393,12 @@ sigalrm_handler(int sig_nr OVS_UNUSED)
     monotonic_tick = true;
 
 #if HAVE_EXECINFO_H
-    if (backtrace_conn && n_traces < MAX_TRACES) {
-        struct trace *trace = &traces[n_traces++];
+    if (CACHE_TIME) {
+        struct trace *trace = &traces[trace_head];
+
         trace->n_frames = backtrace(trace->backtrace,
                                     ARRAY_SIZE(trace->backtrace));
+        trace_head = (trace_head + 1) % MAX_TRACES;
     }
 #endif
 }
@@ -490,66 +489,40 @@ timespec_add(struct timespec *sum,
 static void
 log_poll_interval(long long int last_wakeup)
 {
-    static unsigned int mean_interval; /* In 16ths of a millisecond. */
-    static unsigned int n_samples;
-
-    long long int now;
-    unsigned int interval;      /* In 16ths of a millisecond. */
-
-    /* Compute interval from last wakeup to now in 16ths of a millisecond,
-     * capped at 10 seconds (16000 in this unit). */
-    now = time_msec();
-    interval = MIN(10000, now - last_wakeup) << 4;
-
-    /* Warn if we took too much time between polls: at least 50 ms and at least
-     * 8X the mean interval. */
-    if (n_samples > 10 && interval > mean_interval * 8 && interval > 50 * 16) {
-        static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 3);
-
-        if (!VLOG_DROP_WARN(&rl)) {
-            const struct rusage *last_rusage = get_recent_rusage();
-            struct rusage rusage;
-
-            getrusage(RUSAGE_SELF, &rusage);
-            VLOG_WARN("%lld ms poll interval (%lld ms user, %lld ms system) "
-                      "is over %u times the weighted mean interval %u ms "
-                      "(%u samples)",
-                      now - last_wakeup,
-                      timeval_diff_msec(&rusage.ru_utime,
-                                        &last_rusage->ru_utime),
-                      timeval_diff_msec(&rusage.ru_stime,
-                                        &last_rusage->ru_stime),
-                      interval / mean_interval,
-                      (mean_interval + 8) / 16, n_samples);
-            if (rusage.ru_minflt > last_rusage->ru_minflt
-                || rusage.ru_majflt > last_rusage->ru_majflt) {
-                VLOG_WARN("faults: %ld minor, %ld major",
-                          rusage.ru_minflt - last_rusage->ru_minflt,
-                          rusage.ru_majflt - last_rusage->ru_majflt);
-            }
-            if (rusage.ru_inblock > last_rusage->ru_inblock
-                || rusage.ru_oublock > last_rusage->ru_oublock) {
-                VLOG_WARN("disk: %ld reads, %ld writes",
-                          rusage.ru_inblock - last_rusage->ru_inblock,
-                          rusage.ru_oublock - last_rusage->ru_oublock);
-            }
-            if (rusage.ru_nvcsw > last_rusage->ru_nvcsw
-                || rusage.ru_nivcsw > last_rusage->ru_nivcsw) {
-                VLOG_WARN("context switches: %ld voluntary, %ld involuntary",
-                          rusage.ru_nvcsw - last_rusage->ru_nvcsw,
-                          rusage.ru_nivcsw - last_rusage->ru_nivcsw);
-            }
+    long long int interval = time_msec() - last_wakeup;
+
+    if (interval >= 1000) {
+        const struct rusage *last_rusage = get_recent_rusage();
+        struct rusage rusage;
+
+        getrusage(RUSAGE_SELF, &rusage);
+        VLOG_WARN("Unreasonably long %lldms poll interval"
+                  " (%lldms user, %lldms system)",
+                  interval,
+                  timeval_diff_msec(&rusage.ru_utime,
+                                    &last_rusage->ru_utime),
+                  timeval_diff_msec(&rusage.ru_stime,
+                                    &last_rusage->ru_stime));
+        if (rusage.ru_minflt > last_rusage->ru_minflt
+            || rusage.ru_majflt > last_rusage->ru_majflt) {
+            VLOG_WARN("faults: %ld minor, %ld major",
+                      rusage.ru_minflt - last_rusage->ru_minflt,
+                      rusage.ru_majflt - last_rusage->ru_majflt);
+        }
+        if (rusage.ru_inblock > last_rusage->ru_inblock
+            || rusage.ru_oublock > last_rusage->ru_oublock) {
+            VLOG_WARN("disk: %ld reads, %ld writes",
+                      rusage.ru_inblock - last_rusage->ru_inblock,
+                      rusage.ru_oublock - last_rusage->ru_oublock);
+        }
+        if (rusage.ru_nvcsw > last_rusage->ru_nvcsw
+            || rusage.ru_nivcsw > last_rusage->ru_nivcsw) {
+            VLOG_WARN("context switches: %ld voluntary, %ld involuntary",
+                      rusage.ru_nvcsw - last_rusage->ru_nvcsw,
+                      rusage.ru_nivcsw - last_rusage->ru_nivcsw);
         }
         coverage_log();
     }
-
-    /* Update exponentially weighted moving average.  With these parameters, a
-     * given value decays to 1% of its value in about 100 time steps.  */
-    if (n_samples++) {
-        mean_interval = (mean_interval * 122 + interval * 6 + 64) / 128;
-    } else {
-        mean_interval = interval;
-    }
 }
 \f
 /* CPU usage tracking. */
@@ -605,42 +578,38 @@ get_cpu_usage(void)
 }
 
 static void
-trace_run(void)
+format_backtraces(struct ds *ds)
 {
+    time_init();
+
 #if HAVE_EXECINFO_H
-    if (backtrace_conn && n_traces >= MAX_TRACES) {
-        struct unixctl_conn *reply_conn = backtrace_conn;
-        struct ds ds = DS_EMPTY_INITIALIZER;
+    if (CACHE_TIME) {
         sigset_t oldsigs;
         size_t i;
 
         block_sigalrm(&oldsigs);
 
-        for (i = 0; i < n_traces; i++) {
+        for (i = 0; i < MAX_TRACES; i++) {
             struct trace *trace = &traces[i];
             char **frame_strs;
             size_t j;
 
+            if (!trace->n_frames) {
+                continue;
+            }
+
             frame_strs = backtrace_symbols(trace->backtrace, trace->n_frames);
 
-            ds_put_format(&ds, "Backtrace %zu\n", i + 1);
+            ds_put_format(ds, "Backtrace %zu\n", i + 1);
             for (j = 0; j < trace->n_frames; j++) {
-                ds_put_format(&ds, "%s\n", frame_strs[j]);
+                ds_put_format(ds, "%s\n", frame_strs[j]);
             }
-            ds_put_cstr(&ds, "\n");
+            ds_put_cstr(ds, "\n");
 
             free(frame_strs);
         }
-
-        free(traces);
-        traces = NULL;
-        n_traces = 0;
-        backtrace_conn = NULL;
-
+        ds_chomp(ds, '\n');
         unblock_sigalrm(&oldsigs);
-
-        unixctl_command_reply(reply_conn, ds_cstr(&ds));
-        ds_destroy(&ds);
     }
 #endif
 }
@@ -688,21 +657,12 @@ backtrace_cb(struct unixctl_conn *conn,
              int argc OVS_UNUSED, const char *argv[] OVS_UNUSED,
              void *aux OVS_UNUSED)
 {
-    sigset_t oldsigs;
+    struct ds ds = DS_EMPTY_INITIALIZER;
 
     assert(HAVE_EXECINFO_H && CACHE_TIME);
-
-    if (backtrace_conn) {
-        unixctl_command_reply_error(conn, "In Use");
-        return;
-    }
-    assert(!traces);
-
-    block_sigalrm(&oldsigs);
-    backtrace_conn = conn;
-    traces = xmalloc(MAX_TRACES * sizeof *traces);
-    n_traces = 0;
-    unblock_sigalrm(&oldsigs);
+    format_backtraces(&ds);
+    unixctl_command_reply(conn, ds_cstr(&ds));
+    ds_destroy(&ds);
 }
 
 void