ofproto: Make number of packet handler threads runtime configurable.

This commit adds a new column "n-handler-threads" to the Open_vSwitch
table. This is used to set the number of upcall handler threads created by
the ofproto-dpif-upcall module.

Signed-off-by: Alex Wang <alexw@nicira.com>
Signed-off-by: Ben Pfaff <blp@nicira.com>

diff --git a/ofproto/ofproto-dpif.c b/ofproto/ofproto-dpif.c
index 3bc0e98..4e953f4 100644 (file)
--- a/ofproto/ofproto-dpif.c
+++ b/ofproto/ofproto-dpif.c
@@ -75,8 +75,6 @@ COVERAGE_DEFINE(subfacet_install_fail);
 COVERAGE_DEFINE(packet_in_overflow);
 COVERAGE_DEFINE(flow_mod_overflow);
 
-#define N_THREADS 16
-
 /* Number of implemented OpenFlow tables. */
 enum { N_TABLES = 255 };
 enum { TBL_INTERNAL = N_TABLES - 1 };    /* Used for internal hidden rules. */
@@ -431,6 +429,9 @@ struct dpif_backer {
     /* Number of subfacets added or deleted from 'created' to 'last_minute.' */
     unsigned long long int total_subfacet_add_count;
     unsigned long long int total_subfacet_del_count;
+
+    /* Number of upcall handling threads. */
+    unsigned int n_handler_threads;
 };
 
 /* All existing ofproto_backer instances, indexed by ofproto->up.type. */
@@ -700,11 +701,20 @@ type_run(const char *type)
             VLOG_ERR("Failed to enable receiving packets in dpif.");
             return error;
         }
-        udpif_recv_set(backer->udpif, N_THREADS, backer->recv_set_enable);
+        udpif_recv_set(backer->udpif, n_handler_threads,
+                       backer->recv_set_enable);
         dpif_flow_flush(backer->dpif);
         backer->need_revalidate = REV_RECONFIGURE;
     }
 
+    /* If the n_handler_threads is reconfigured, call udpif_recv_set()
+     * to reset the handler threads. */
+    if (backer->n_handler_threads != n_handler_threads) {
+        udpif_recv_set(backer->udpif, n_handler_threads,
+                       backer->recv_set_enable);
+        backer->n_handler_threads = n_handler_threads;
+    }
+
     if (backer->need_revalidate) {
         struct ofproto_dpif *ofproto;
         struct simap_node *node;
@@ -1211,7 +1221,9 @@ open_dpif_backer(const char *type, struct dpif_backer **backerp)
         close_dpif_backer(backer);
         return error;
     }
-    udpif_recv_set(backer->udpif, N_THREADS, backer->recv_set_enable);
+    udpif_recv_set(backer->udpif, n_handler_threads,
+                   backer->recv_set_enable);
+    backer->n_handler_threads = n_handler_threads;
 
     backer->max_n_subfacet = 0;
     backer->created = time_msec();

diff --git a/ofproto/ofproto-provider.h b/ofproto/ofproto-provider.h
index aa262bc..ef4d588 100644 (file)
--- a/ofproto/ofproto-provider.h
+++ b/ofproto/ofproto-provider.h
@@ -262,6 +262,10 @@ struct rule {
  * ofproto-dpif implementation */
 extern unsigned flow_eviction_threshold;
 
+/* Number of upcall handler threads. Only affects the ofproto-dpif
+ * implementation. */
+extern unsigned n_handler_threads;
+
 /* Determines which model to use for handling misses in the ofproto-dpif
  * implementation */
 extern enum ofproto_flow_miss_model flow_miss_model;

diff --git a/ofproto/ofproto.c b/ofproto/ofproto.c
index bbdb2d2..bead3c4 100644 (file)
--- a/ofproto/ofproto.c
+++ b/ofproto/ofproto.c
@@ -21,6 +21,7 @@
 #include <inttypes.h>
 #include <stdbool.h>
 #include <stdlib.h>
+#include <unistd.h>
 #include "bitmap.h"
 #include "byte-order.h"
 #include "classifier.h"
@@ -229,6 +230,7 @@ static size_t n_ofproto_classes;
 static size_t allocated_ofproto_classes;
 
 unsigned flow_eviction_threshold = OFPROTO_FLOW_EVICTION_THRESHOLD_DEFAULT;
+unsigned n_handler_threads;
 enum ofproto_flow_miss_model flow_miss_model = OFPROTO_HANDLE_MISS_AUTO;
 
 /* Map from datapath name to struct ofproto, for use by unixctl commands. */
@@ -628,6 +630,18 @@ ofproto_set_mac_table_config(struct ofproto *ofproto, unsigned idle_time,
     }
 }
 
+/* Sets number of upcall handler threads.  The default is
+ * (number of online cores - 1). */
+void
+ofproto_set_n_handler_threads(unsigned limit)
+{
+    if (limit) {
+        n_handler_threads = limit;
+    } else {
+        n_handler_threads = MAX(1, sysconf(_SC_NPROCESSORS_ONLN) - 1);
+    }
+}
+
 void
 ofproto_set_dp_desc(struct ofproto *p, const char *dp_desc)
 {

diff --git a/ofproto/ofproto.h b/ofproto/ofproto.h
index 1bde385..516bbad 100644 (file)
--- a/ofproto/ofproto.h
+++ b/ofproto/ofproto.h
@@ -247,6 +247,7 @@ void ofproto_set_flow_miss_model(unsigned model);
 void ofproto_set_forward_bpdu(struct ofproto *, bool forward_bpdu);
 void ofproto_set_mac_table_config(struct ofproto *, unsigned idle_time,
                                   size_t max_entries);
+void ofproto_set_n_handler_threads(unsigned limit);
 void ofproto_set_dp_desc(struct ofproto *, const char *dp_desc);
 int ofproto_set_snoops(struct ofproto *, const struct sset *snoops);
 int ofproto_set_netflow(struct ofproto *,

diff --git a/vswitchd/bridge.c b/vswitchd/bridge.c
index abbda56..3d63125 100644 (file)
--- a/vswitchd/bridge.c
+++ b/vswitchd/bridge.c
@@ -502,6 +502,9 @@ bridge_reconfigure(const struct ovsrec_open_vswitch *ovs_cfg)
         smap_get_int(&ovs_cfg->other_config, "flow-eviction-threshold",
                      OFPROTO_FLOW_EVICTION_THRESHOLD_DEFAULT));
 
+    ofproto_set_n_handler_threads(
+        smap_get_int(&ovs_cfg->other_config, "n-handler-threads", 0));
+
     bridge_configure_flow_miss_model(smap_get(&ovs_cfg->other_config,
                                               "force-miss-model"));
 

diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml
index b89d58c..5bbe943 100644 (file)
--- a/vswitchd/vswitch.xml
+++ b/vswitchd/vswitch.xml
@@ -158,6 +158,22 @@
           </dl>
         </p>
       </column>
+
+      <column name="other_config" key="n-handler-threads"
+              type='{"type": "integer", "minInteger": 1}'>
+        <p>
+          Specifies the number of threads for software datapaths to use for
+          handling new flows.  The default is one less than the number of
+          online CPU cores (but at least 1).
+        </p>
+        <p>
+          This configuration is per datapath.  If you have more than one
+          software datapath (e.g. some <code>system</code> bridges and some
+          <code>netdev</code> bridges), then the total number of threads is
+          <code>n-handler-threads</code> times the number of software
+          datapaths.
+        </p>
+      </column>
     </group>
 
     <group title="Status">