When profiling CPU usage in situations involving high numbers of ports,
coverage_clear() was highlighted as a commonly called function. It
appears that it can be quite expensive to access all of the per-thread
coverage counters when threads are constantly waking up.
This patch makes each thread only do coverage_clear() logic roughly once
per second by introducing per-thread timers. Upcall handler counters may
become less accurate, as these threads may sleep without synchronising
and not wake up for some time. When the main thread is under load at
~90% CPU, this drops to ~85%. Upcall handler threads sitting at ~2.5%
drop to ~1.5%.
Signed-off-by: Joe Stringer <joestringer@nicira.com>
Signed-off-by: Ben Pfaff <blp@nicira.com>
static struct ovs_mutex coverage_mutex = OVS_MUTEX_INITIALIZER;
static struct ovs_mutex coverage_mutex = OVS_MUTEX_INITIALIZER;
+DEFINE_STATIC_PER_THREAD_DATA(long long int, coverage_clear_time, LLONG_MIN);
static long long int coverage_run_time = LLONG_MIN;
/* Index counter used to compute the moving average array's index. */
static long long int coverage_run_time = LLONG_MIN;
/* Index counter used to compute the moving average array's index. */
+/* Runs approximately every COVERAGE_CLEAR_INTERVAL amount of time to
+ * synchronize per-thread counters with global counters. Every thread maintains
+ * a separate timer to ensure all counters are periodically aggregated. */
void
coverage_clear(void)
{
void
coverage_clear(void)
{
+ long long int now, *thread_time;
- ovs_mutex_lock(&coverage_mutex);
- for (i = 0; i < n_coverage_counters; i++) {
- struct coverage_counter *c = coverage_counters[i];
- c->total += c->count();
+ now = time_msec();
+ thread_time = coverage_clear_time_get();
+
+ /* Initialize the coverage_clear_time. */
+ if (*thread_time == LLONG_MIN) {
+ *thread_time = now + COVERAGE_CLEAR_INTERVAL;
+ }
+
+ if (now >= *thread_time) {
+ size_t i;
+
+ ovs_mutex_lock(&coverage_mutex);
+ for (i = 0; i < n_coverage_counters; i++) {
+ struct coverage_counter *c = coverage_counters[i];
+ c->total += c->count();
+ }
+ ovs_mutex_unlock(&coverage_mutex);
+ *thread_time = now + COVERAGE_CLEAR_INTERVAL;
- ovs_mutex_unlock(&coverage_mutex);
}
/* Runs approximately every COVERAGE_RUN_INTERVAL amount of time to update the
}
/* Runs approximately every COVERAGE_RUN_INTERVAL amount of time to update the
#define COVERAGE_RUN_INTERVAL 5000
BUILD_ASSERT_DECL(60000 % COVERAGE_RUN_INTERVAL == 0);
#define COVERAGE_RUN_INTERVAL 5000
BUILD_ASSERT_DECL(60000 % COVERAGE_RUN_INTERVAL == 0);
+#define COVERAGE_CLEAR_INTERVAL 1000
+BUILD_ASSERT_DECL(COVERAGE_RUN_INTERVAL % COVERAGE_CLEAR_INTERVAL == 0);
+
/* Defines the moving average array length. */
#define MIN_AVG_LEN (60000/COVERAGE_RUN_INTERVAL)
#define HR_AVG_LEN 60
/* Defines the moving average array length. */
#define MIN_AVG_LEN (60000/COVERAGE_RUN_INTERVAL)
#define HR_AVG_LEN 60
arp,metadata=0,in_port=0,vlan_tci=0x0000,dl_src=80:88:88:88:88:88,dl_dst=ff:ff:ff:ff:ff:ff,arp_spa=192.168.128.1,arp_tpa=192.168.0.2,arp_op=2,arp_sha=50:54:00:00:00:05,arp_tha=40:44:44:44:44:41
])
arp,metadata=0,in_port=0,vlan_tci=0x0000,dl_src=80:88:88:88:88:88,dl_dst=ff:ff:ff:ff:ff:ff,arp_spa=192.168.128.1,arp_tpa=192.168.0.2,arp_op=2,arp_sha=50:54:00:00:00:05,arp_tha=40:44:44:44:44:41
])
-# Check that each of the packets actually passed through the slow-path.
-AT_CHECK([ovs-appctl coverage/show], [0], [stdout])
-AT_CHECK([sed -n 's/[[ ]]\{2,\}/ /g
-s/^dpif_execute_with_help.*total: //p' stdout], [0], [3
-])
-
OVS_VSWITCHD_STOP
AT_CLEANUP
OVS_VSWITCHD_STOP
AT_CLEANUP