X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=lib%2Fdpif-netdev.c;h=55712dde1674beb117e1ea70db81d3aec82a2a07;hb=27bbe15dec4e1862396b5c4d265f0ced71b49930;hp=d6bf59d58a172bee2798ded50707689d68d66905;hpb=61e7deb14330d107464c2ea70ed2a9f2f1d27af6;p=sliver-openvswitch.git

diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
index d6bf59d58..55712dde1 100644
--- a/lib/dpif-netdev.c
+++ b/lib/dpif-netdev.c
@@ -43,6 +43,7 @@
 #include "list.h"
 #include "meta-flow.h"
 #include "netdev.h"
+#include "netdev-dpdk.h"
 #include "netdev-vport.h"
 #include "netlink.h"
 #include "odp-execute.h"
@@ -66,15 +67,15 @@ VLOG_DEFINE_THIS_MODULE(dpif_netdev);
 /* By default, choose a priority in the middle. */
 #define NETDEV_RULE_PRIORITY 0x8000
 
+#define NR_THREADS 1
+/* Use per thread recirc_depth to prevent recirculation loop. */
+#define MAX_RECIRC_DEPTH 5
+DEFINE_STATIC_PER_THREAD_DATA(uint32_t, recirc_depth, 0)
+
 /* Configuration parameters. */
 enum { MAX_FLOWS = 65536 };     /* Maximum number of flows in flow table. */
 
-/* Enough headroom to add a vlan tag, plus an extra 2 bytes to allow IP
- * headers to be aligned on a 4-byte boundary.  */
-enum { DP_NETDEV_HEADROOM = 2 + VLAN_HEADER_LEN };
-
 /* Queues. */
-enum { N_QUEUES = 2 };          /* Number of queues for dpif_recv(). */
 enum { MAX_QUEUE_LEN = 128 };   /* Maximum number of packets per queue. */
 enum { QUEUE_MASK = MAX_QUEUE_LEN - 1 };
 BUILD_ASSERT_DECL(IS_POW2(MAX_QUEUE_LEN));
@@ -91,14 +92,17 @@ struct dp_netdev_upcall {
     struct ofpbuf buf;          /* ofpbuf instance for upcall.packet. */
 };
 
-/* A queue passing packets from a struct dp_netdev to its clients.
+/* A queue passing packets from a struct dp_netdev to its clients (handlers).
  *
  *
  * Thread-safety
  * =============
  *
- * Any access at all requires the owning 'dp_netdev''s queue_mutex. */
+ * Any access at all requires the owning 'dp_netdev''s queue_rwlock and
+ * its own mutex. */
 struct dp_netdev_queue {
+    struct ovs_mutex mutex;
+    struct seq *seq;      /* Incremented whenever a packet is queued. */
     struct dp_netdev_upcall upcalls[MAX_QUEUE_LEN] OVS_GUARDED;
     unsigned int head OVS_GUARDED;
     unsigned int tail OVS_GUARDED;
@@ -119,7 +123,7 @@ struct dp_netdev_queue {
  *    port_rwlock
  *    flow_mutex
  *    cls.rwlock
- *    queue_mutex
+ *    queue_rwlock
  */
 struct dp_netdev {
     const struct dpif_class *const class;
@@ -141,10 +145,12 @@ struct dp_netdev {
 
     /* Queues.
      *
-     * Everything in 'queues' is protected by 'queue_mutex'. */
-    struct ovs_mutex queue_mutex;
-    struct dp_netdev_queue queues[N_QUEUES];
-    struct seq *queue_seq;      /* Incremented whenever a packet is queued. */
+     * 'queue_rwlock' protects the modification of 'handler_queues' and
+     * 'n_handlers'.  The queue elements are protected by its
+     * 'handler_queues''s mutex. */
+    struct fat_rwlock queue_rwlock;
+    struct dp_netdev_queue *handler_queues;
+    uint32_t n_handlers;
 
     /* Statistics.
      *
@@ -161,8 +167,9 @@ struct dp_netdev {
 
     /* Forwarding threads. */
     struct latch exit_latch;
-    struct dp_forwarder *forwarders;
-    size_t n_forwarders;
+    struct pmd_thread *pmd_threads;
+    size_t n_pmd_threads;
+    int pmd_count;
 };
 
 static struct dp_netdev_port *dp_netdev_lookup_port(const struct dp_netdev *dp,
@@ -191,7 +198,8 @@ struct dp_netdev_port {
     odp_port_t port_no;
     struct netdev *netdev;
     struct netdev_saved_flags *sf;
-    struct netdev_rx *rx;
+    struct netdev_rxq **rxq;
+    struct ovs_refcount ref_cnt;
     char *type;                 /* Port type as requested by user. */
 };
 
@@ -299,13 +307,22 @@ struct dp_netdev_actions *dp_netdev_flow_get_actions(
     const struct dp_netdev_flow *);
 static void dp_netdev_actions_free(struct dp_netdev_actions *);
 
-/* A thread that receives packets from some ports, looks them up in the flow
- * table, and executes the actions it finds. */
-struct dp_forwarder {
+/* PMD: Poll modes drivers.  PMD accesses devices via polling to eliminate
+ * the performance overhead of interrupt processing.  Therefore netdev can
+ * not implement rx-wait for these devices.  dpif-netdev needs to poll
+ * these device to check for recv buffer.  pmd-thread does polling for
+ * devices assigned to itself thread.
+ *
+ * DPDK used PMD for accessing NIC.
+ *
+ * A thread that receives packets from PMD ports, looks them up in the flow
+ * table, and executes the actions it finds.
+ **/
+struct pmd_thread {
     struct dp_netdev *dp;
     pthread_t thread;
-    char *name;
-    uint32_t min_hash, max_hash;
+    int id;
+    atomic_uint change_seq;
 };
 
 /* Interface to netdev-based datapath. */
@@ -329,22 +346,24 @@ static int do_add_port(struct dp_netdev *dp, const char *devname,
     OVS_REQ_WRLOCK(dp->port_rwlock);
 static int do_del_port(struct dp_netdev *dp, odp_port_t port_no)
     OVS_REQ_WRLOCK(dp->port_rwlock);
+static void dp_netdev_destroy_all_queues(struct dp_netdev *dp)
+    OVS_REQ_WRLOCK(dp->queue_rwlock);
 static int dpif_netdev_open(const struct dpif_class *, const char *name,
                             bool create, struct dpif **);
 static int dp_netdev_output_userspace(struct dp_netdev *dp, struct ofpbuf *,
-                                    int queue_no, const struct flow *,
-                                    const struct nlattr *userdata)
-    OVS_EXCLUDED(dp->queue_mutex);
+                                      int queue_no, int type,
+                                      const struct miniflow *,
+                                      const struct nlattr *userdata);
 static void dp_netdev_execute_actions(struct dp_netdev *dp,
-                                      const struct flow *, struct ofpbuf *,
+                                      const struct miniflow *,
+                                      struct ofpbuf *, bool may_steal,
                                       struct pkt_metadata *,
                                       const struct nlattr *actions,
-                                      size_t actions_len)
-    OVS_REQ_RDLOCK(dp->port_rwlock);
+                                      size_t actions_len);
 static void dp_netdev_port_input(struct dp_netdev *dp, struct ofpbuf *packet,
-                                 struct pkt_metadata *)
-    OVS_REQ_RDLOCK(dp->port_rwlock);
-static void dp_netdev_set_threads(struct dp_netdev *, int n);
+                                 struct pkt_metadata *);
+
+static void dp_netdev_set_pmd_threads(struct dp_netdev *, int n);
 
 static struct dpif_netdev *
 dpif_netdev_cast(const struct dpif *dpif)
@@ -452,7 +471,6 @@ create_dp_netdev(const char *name, const struct dpif_class *class,
 {
     struct dp_netdev *dp;
     int error;
-    int i;
 
     dp = xzalloc(sizeof *dp);
     shash_add(&dp_netdevs, name, dp);
@@ -466,13 +484,7 @@ create_dp_netdev(const char *name, const struct dpif_class *class,
     classifier_init(&dp->cls, NULL);
     hmap_init(&dp->flow_table);
 
-    ovs_mutex_init(&dp->queue_mutex);
-    ovs_mutex_lock(&dp->queue_mutex);
-    for (i = 0; i < N_QUEUES; i++) {
-        dp->queues[i].head = dp->queues[i].tail = 0;
-    }
-    ovs_mutex_unlock(&dp->queue_mutex);
-    dp->queue_seq = seq_create();
+    fat_rwlock_init(&dp->queue_rwlock);
 
     ovsthread_stats_init(&dp->stats);
 
@@ -488,7 +500,6 @@ create_dp_netdev(const char *name, const struct dpif_class *class,
         dp_netdev_free(dp);
         return error;
     }
-    dp_netdev_set_threads(dp, 2);
 
     *dpp = dp;
     return 0;
@@ -520,20 +531,21 @@ dpif_netdev_open(const struct dpif_class *class, const char *name,
 
 static void
 dp_netdev_purge_queues(struct dp_netdev *dp)
+    OVS_REQ_WRLOCK(dp->queue_rwlock)
 {
     int i;
 
-    ovs_mutex_lock(&dp->queue_mutex);
-    for (i = 0; i < N_QUEUES; i++) {
-        struct dp_netdev_queue *q = &dp->queues[i];
+    for (i = 0; i < dp->n_handlers; i++) {
+        struct dp_netdev_queue *q = &dp->handler_queues[i];
 
+        ovs_mutex_lock(&q->mutex);
         while (q->tail != q->head) {
             struct dp_netdev_upcall *u = &q->upcalls[q->tail++ & QUEUE_MASK];
             ofpbuf_uninit(&u->upcall.packet);
             ofpbuf_uninit(&u->buf);
         }
+        ovs_mutex_unlock(&q->mutex);
     }
-    ovs_mutex_unlock(&dp->queue_mutex);
 }
 
 /* Requires dp_netdev_mutex so that we can't get a new reference to 'dp'
@@ -548,8 +560,8 @@ dp_netdev_free(struct dp_netdev *dp)
 
     shash_find_and_delete(&dp_netdevs, dp->name);
 
-    dp_netdev_set_threads(dp, 0);
-    free(dp->forwarders);
+    dp_netdev_set_pmd_threads(dp, 0);
+    free(dp->pmd_threads);
 
     dp_netdev_flow_flush(dp);
     ovs_rwlock_wrlock(&dp->port_rwlock);
@@ -564,9 +576,11 @@ dp_netdev_free(struct dp_netdev *dp)
     }
     ovsthread_stats_destroy(&dp->stats);
 
-    dp_netdev_purge_queues(dp);
-    seq_destroy(dp->queue_seq);
-    ovs_mutex_destroy(&dp->queue_mutex);
+    fat_rwlock_wrlock(&dp->queue_rwlock);
+    dp_netdev_destroy_all_queues(dp);
+    fat_rwlock_unlock(&dp->queue_rwlock);
+
+    fat_rwlock_destroy(&dp->queue_rwlock);
 
     classifier_destroy(&dp->cls);
     hmap_destroy(&dp->flow_table);
@@ -641,6 +655,19 @@ dpif_netdev_get_stats(const struct dpif *dpif, struct dpif_dp_stats *stats)
     return 0;
 }
 
+static void
+dp_netdev_reload_pmd_threads(struct dp_netdev *dp)
+{
+    int i;
+
+    for (i = 0; i < dp->n_pmd_threads; i++) {
+        struct pmd_thread *f = &dp->pmd_threads[i];
+        int id;
+
+        atomic_add(&f->change_seq, 1, &id);
+   }
+}
+
 static int
 do_add_port(struct dp_netdev *dp, const char *devname, const char *type,
             odp_port_t port_no)
@@ -649,10 +676,10 @@ do_add_port(struct dp_netdev *dp, const char *devname, const char *type,
     struct netdev_saved_flags *sf;
     struct dp_netdev_port *port;
     struct netdev *netdev;
-    struct netdev_rx *rx;
     enum netdev_flags flags;
     const char *open_type;
     int error;
+    int i;
 
     /* XXX reject devices already in some dp_netdev. */
 
@@ -671,28 +698,40 @@ do_add_port(struct dp_netdev *dp, const char *devname, const char *type,
         return EINVAL;
     }
 
-    error = netdev_rx_open(netdev, &rx);
-    if (error
-        && !(error == EOPNOTSUPP && dpif_netdev_class_is_dummy(dp->class))) {
-        VLOG_ERR("%s: cannot receive packets on this network device (%s)",
-                 devname, ovs_strerror(errno));
-        netdev_close(netdev);
-        return error;
+    port = xzalloc(sizeof *port);
+    port->port_no = port_no;
+    port->netdev = netdev;
+    port->rxq = xmalloc(sizeof *port->rxq * netdev_n_rxq(netdev));
+    port->type = xstrdup(type);
+    for (i = 0; i < netdev_n_rxq(netdev); i++) {
+        error = netdev_rxq_open(netdev, &port->rxq[i], i);
+        if (error
+            && !(error == EOPNOTSUPP && dpif_netdev_class_is_dummy(dp->class))) {
+            VLOG_ERR("%s: cannot receive packets on this network device (%s)",
+                     devname, ovs_strerror(errno));
+            netdev_close(netdev);
+            return error;
+        }
     }
 
     error = netdev_turn_flags_on(netdev, NETDEV_PROMISC, &sf);
     if (error) {
-        netdev_rx_close(rx);
+        for (i = 0; i < netdev_n_rxq(netdev); i++) {
+            netdev_rxq_close(port->rxq[i]);
+        }
         netdev_close(netdev);
+        free(port->rxq);
+        free(port);
         return error;
     }
-
-    port = xmalloc(sizeof *port);
-    port->port_no = port_no;
-    port->netdev = netdev;
     port->sf = sf;
-    port->rx = rx;
-    port->type = xstrdup(type);
+
+    if (netdev_is_pmd(netdev)) {
+        dp->pmd_count++;
+        dp_netdev_set_pmd_threads(dp, NR_THREADS);
+        dp_netdev_reload_pmd_threads(dp);
+    }
+    ovs_refcount_init(&port->ref_cnt);
 
     hmap_insert(&dp->ports, &port->node, hash_int(odp_to_u32(port_no), 0));
     seq_change(dp->port_seq);
@@ -776,6 +815,31 @@ get_port_by_number(struct dp_netdev *dp,
     }
 }
 
+static void
+port_ref(struct dp_netdev_port *port)
+{
+    if (port) {
+        ovs_refcount_ref(&port->ref_cnt);
+    }
+}
+
+static void
+port_unref(struct dp_netdev_port *port)
+{
+    if (port && ovs_refcount_unref(&port->ref_cnt) == 1) {
+        int i;
+
+        netdev_close(port->netdev);
+        netdev_restore_flags(port->sf);
+
+        for (i = 0; i < netdev_n_rxq(port->netdev); i++) {
+            netdev_rxq_close(port->rxq[i]);
+        }
+        free(port->type);
+        free(port);
+    }
+}
+
 static int
 get_port_by_name(struct dp_netdev *dp,
                  const char *devname, struct dp_netdev_port **portp)
@@ -806,13 +870,11 @@ do_del_port(struct dp_netdev *dp, odp_port_t port_no)
 
     hmap_remove(&dp->ports, &port->node);
     seq_change(dp->port_seq);
+    if (netdev_is_pmd(port->netdev)) {
+        dp_netdev_reload_pmd_threads(dp);
+    }
 
-    netdev_close(port->netdev);
-    netdev_restore_flags(port->sf);
-    netdev_rx_close(port->rx);
-    free(port->type);
-    free(port);
-
+    port_unref(port);
     return 0;
 }
 
@@ -1001,13 +1063,15 @@ dp_netdev_flow_cast(const struct cls_rule *cr)
 }
 
 static struct dp_netdev_flow *
-dp_netdev_lookup_flow(const struct dp_netdev *dp, const struct flow *flow)
+dp_netdev_lookup_flow(const struct dp_netdev *dp, const struct miniflow *key)
     OVS_EXCLUDED(dp->cls.rwlock)
 {
     struct dp_netdev_flow *netdev_flow;
+    struct cls_rule *rule;
 
     fat_rwlock_rdlock(&dp->cls.rwlock);
-    netdev_flow = dp_netdev_flow_cast(classifier_lookup(&dp->cls, flow, NULL));
+    rule = classifier_lookup_miniflow_first(&dp->cls, key);
+    netdev_flow = dp_netdev_flow_cast(rule);
     fat_rwlock_unlock(&dp->cls.rwlock);
 
     return netdev_flow;
@@ -1077,8 +1141,6 @@ dpif_netdev_mask_from_nlattrs(const struct nlattr *key, uint32_t key_len,
 
             return EINVAL;
         }
-        /* Force unwildcard the in_port. */
-        mask->in_port.odp_port = u32_to_odp(UINT32_MAX);
     } else {
         enum mf_field_id id;
         /* No mask key, unwildcard everything except fields whose
@@ -1097,6 +1159,14 @@ dpif_netdev_mask_from_nlattrs(const struct nlattr *key, uint32_t key_len,
         }
     }
 
+    /* Force unwildcard the in_port.
+     *
+     * We need to do this even in the case where we unwildcard "everything"
+     * above because "everything" only includes the 16-bit OpenFlow port number
+     * mask->in_port.ofp_port, which only covers half of the 32-bit datapath
+     * port number mask->in_port.odp_port. */
+    mask->in_port.odp_port = u32_to_odp(UINT32_MAX);
+
     return 0;
 }
 
@@ -1226,6 +1296,7 @@ dpif_netdev_flow_put(struct dpif *dpif, const struct dpif_flow_put *put)
     struct dp_netdev *dp = get_dp_netdev(dpif);
     struct dp_netdev_flow *netdev_flow;
     struct flow flow;
+    struct miniflow miniflow;
     struct flow_wildcards wc;
     int error;
 
@@ -1239,9 +1310,10 @@ dpif_netdev_flow_put(struct dpif *dpif, const struct dpif_flow_put *put)
     if (error) {
         return error;
     }
+    miniflow_init(&miniflow, &flow);
 
     ovs_mutex_lock(&dp->flow_mutex);
-    netdev_flow = dp_netdev_lookup_flow(dp, &flow);
+    netdev_flow = dp_netdev_lookup_flow(dp, &miniflow);
     if (!netdev_flow) {
         if (put->flags & DPIF_FP_CREATE) {
             if (hmap_count(&dp->flow_table) < MAX_FLOWS) {
@@ -1374,6 +1446,7 @@ dpif_netdev_flow_dump_next(const struct dpif *dpif, void *iter_, void *state_,
     struct dp_netdev_flow_state *state = state_;
     struct dp_netdev *dp = get_dp_netdev(dpif);
     struct dp_netdev_flow *netdev_flow;
+    struct flow_wildcards wc;
     int error;
 
     ovs_mutex_lock(&iter->mutex);
@@ -1396,29 +1469,29 @@ dpif_netdev_flow_dump_next(const struct dpif *dpif, void *iter_, void *state_,
         return error;
     }
 
+    minimask_expand(&netdev_flow->cr.match.mask, &wc);
+
     if (key) {
         struct ofpbuf buf;
 
         ofpbuf_use_stack(&buf, &state->keybuf, sizeof state->keybuf);
-        odp_flow_key_from_flow(&buf, &netdev_flow->flow,
+        odp_flow_key_from_flow(&buf, &netdev_flow->flow, &wc.masks,
                                netdev_flow->flow.in_port.odp_port);
 
-        *key = buf.data;
-        *key_len = buf.size;
+        *key = ofpbuf_data(&buf);
+        *key_len = ofpbuf_size(&buf);
     }
 
     if (key && mask) {
         struct ofpbuf buf;
-        struct flow_wildcards wc;
 
         ofpbuf_use_stack(&buf, &state->maskbuf, sizeof state->maskbuf);
-        minimask_expand(&netdev_flow->cr.match.mask, &wc);
         odp_flow_key_from_mask(&buf, &wc.masks, &netdev_flow->flow,
                                odp_to_u32(wc.masks.in_port.odp_port),
                                SIZE_MAX);
 
-        *mask = buf.data;
-        *mask_len = buf.size;
+        *mask = ofpbuf_data(&buf);
+        *mask_len = ofpbuf_size(&buf);
     }
 
     if (actions || stats) {
@@ -1454,27 +1527,99 @@ dpif_netdev_execute(struct dpif *dpif, struct dpif_execute *execute)
 {
     struct dp_netdev *dp = get_dp_netdev(dpif);
     struct pkt_metadata *md = &execute->md;
-    struct flow key;
+    struct {
+        struct miniflow flow;
+        uint32_t buf[FLOW_U32S];
+    } key;
 
-    if (execute->packet->size < ETH_HEADER_LEN ||
-        execute->packet->size > UINT16_MAX) {
+    if (ofpbuf_size(execute->packet) < ETH_HEADER_LEN ||
+        ofpbuf_size(execute->packet) > UINT16_MAX) {
         return EINVAL;
     }
 
     /* Extract flow key. */
-    flow_extract(execute->packet, md, &key);
+    miniflow_initialize(&key.flow, key.buf);
+    miniflow_extract(execute->packet, md, &key.flow);
 
     ovs_rwlock_rdlock(&dp->port_rwlock);
-    dp_netdev_execute_actions(dp, &key, execute->packet, md, execute->actions,
-                              execute->actions_len);
+    dp_netdev_execute_actions(dp, &key.flow, execute->packet, false, md,
+                              execute->actions, execute->actions_len);
     ovs_rwlock_unlock(&dp->port_rwlock);
 
     return 0;
 }
 
+static void
+dp_netdev_destroy_all_queues(struct dp_netdev *dp)
+    OVS_REQ_WRLOCK(dp->queue_rwlock)
+{
+    size_t i;
+
+    dp_netdev_purge_queues(dp);
+
+    for (i = 0; i < dp->n_handlers; i++) {
+        struct dp_netdev_queue *q = &dp->handler_queues[i];
+
+        ovs_mutex_destroy(&q->mutex);
+        seq_destroy(q->seq);
+    }
+    free(dp->handler_queues);
+    dp->handler_queues = NULL;
+    dp->n_handlers = 0;
+}
+
+static void
+dp_netdev_refresh_queues(struct dp_netdev *dp, uint32_t n_handlers)
+    OVS_REQ_WRLOCK(dp->queue_rwlock)
+{
+    if (dp->n_handlers != n_handlers) {
+        size_t i;
+
+        dp_netdev_destroy_all_queues(dp);
+
+        dp->n_handlers = n_handlers;
+        dp->handler_queues = xzalloc(n_handlers * sizeof *dp->handler_queues);
+
+        for (i = 0; i < n_handlers; i++) {
+            struct dp_netdev_queue *q = &dp->handler_queues[i];
+
+            ovs_mutex_init(&q->mutex);
+            q->seq = seq_create();
+        }
+    }
+}
+
+static int
+dpif_netdev_recv_set(struct dpif *dpif, bool enable)
+{
+    struct dp_netdev *dp = get_dp_netdev(dpif);
+
+    if ((dp->handler_queues != NULL) == enable) {
+        return 0;
+    }
+
+    fat_rwlock_wrlock(&dp->queue_rwlock);
+    if (!enable) {
+        dp_netdev_destroy_all_queues(dp);
+    } else {
+        dp_netdev_refresh_queues(dp, 1);
+    }
+    fat_rwlock_unlock(&dp->queue_rwlock);
+
+    return 0;
+}
+
 static int
-dpif_netdev_recv_set(struct dpif *dpif OVS_UNUSED, bool enable OVS_UNUSED)
+dpif_netdev_handlers_set(struct dpif *dpif, uint32_t n_handlers)
 {
+    struct dp_netdev *dp = get_dp_netdev(dpif);
+
+    fat_rwlock_wrlock(&dp->queue_rwlock);
+    if (dp->handler_queues) {
+        dp_netdev_refresh_queues(dp, n_handlers);
+    }
+    fat_rwlock_unlock(&dp->queue_rwlock);
+
     return 0;
 }
 
@@ -1486,62 +1631,86 @@ dpif_netdev_queue_to_priority(const struct dpif *dpif OVS_UNUSED,
     return 0;
 }
 
-static struct dp_netdev_queue *
-find_nonempty_queue(struct dp_netdev *dp)
-    OVS_REQUIRES(dp->queue_mutex)
+static bool
+dp_netdev_recv_check(const struct dp_netdev *dp, const uint32_t handler_id)
+    OVS_REQ_RDLOCK(dp->queue_rwlock)
 {
-    int i;
+    static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
 
-    for (i = 0; i < N_QUEUES; i++) {
-        struct dp_netdev_queue *q = &dp->queues[i];
-        if (q->head != q->tail) {
-            return q;
-        }
+    if (!dp->handler_queues) {
+        VLOG_WARN_RL(&rl, "receiving upcall disabled");
+        return false;
     }
-    return NULL;
+
+    if (handler_id >= dp->n_handlers) {
+        VLOG_WARN_RL(&rl, "handler index out of bound");
+        return false;
+    }
+
+    return true;
 }
 
 static int
-dpif_netdev_recv(struct dpif *dpif, struct dpif_upcall *upcall,
-                 struct ofpbuf *buf)
+dpif_netdev_recv(struct dpif *dpif, uint32_t handler_id,
+                 struct dpif_upcall *upcall, struct ofpbuf *buf)
 {
     struct dp_netdev *dp = get_dp_netdev(dpif);
     struct dp_netdev_queue *q;
-    int error;
+    int error = 0;
 
-    ovs_mutex_lock(&dp->queue_mutex);
-    q = find_nonempty_queue(dp);
-    if (q) {
+    fat_rwlock_rdlock(&dp->queue_rwlock);
+
+    if (!dp_netdev_recv_check(dp, handler_id)) {
+        error = EAGAIN;
+        goto out;
+    }
+
+    q = &dp->handler_queues[handler_id];
+    ovs_mutex_lock(&q->mutex);
+    if (q->head != q->tail) {
         struct dp_netdev_upcall *u = &q->upcalls[q->tail++ & QUEUE_MASK];
 
         *upcall = u->upcall;
 
         ofpbuf_uninit(buf);
         *buf = u->buf;
-
-        error = 0;
     } else {
         error = EAGAIN;
     }
-    ovs_mutex_unlock(&dp->queue_mutex);
+    ovs_mutex_unlock(&q->mutex);
+
+out:
+    fat_rwlock_unlock(&dp->queue_rwlock);
 
     return error;
 }
 
 static void
-dpif_netdev_recv_wait(struct dpif *dpif)
+dpif_netdev_recv_wait(struct dpif *dpif, uint32_t handler_id)
 {
     struct dp_netdev *dp = get_dp_netdev(dpif);
+    struct dp_netdev_queue *q;
     uint64_t seq;
 
-    ovs_mutex_lock(&dp->queue_mutex);
-    seq = seq_read(dp->queue_seq);
-    if (find_nonempty_queue(dp)) {
+    fat_rwlock_rdlock(&dp->queue_rwlock);
+
+    if (!dp_netdev_recv_check(dp, handler_id)) {
+        goto out;
+    }
+
+    q = &dp->handler_queues[handler_id];
+    ovs_mutex_lock(&q->mutex);
+    seq = seq_read(q->seq);
+    if (q->head != q->tail) {
         poll_immediate_wake();
     } else {
-        seq_wait(dp->queue_seq, seq);
+        seq_wait(q->seq, seq);
     }
-    ovs_mutex_unlock(&dp->queue_mutex);
+
+    ovs_mutex_unlock(&q->mutex);
+
+out:
+    fat_rwlock_unlock(&dp->queue_rwlock);
 }
 
 static void
@@ -1549,7 +1718,9 @@ dpif_netdev_recv_purge(struct dpif *dpif)
 {
     struct dpif_netdev *dpif_netdev = dpif_netdev_cast(dpif);
 
+    fat_rwlock_wrlock(&dpif_netdev->dp->queue_rwlock);
     dp_netdev_purge_queues(dpif_netdev->dp);
+    fat_rwlock_unlock(&dpif_netdev->dp->queue_rwlock);
 }
 
 /* Creates and returns a new 'struct dp_netdev_actions', with a reference count
@@ -1580,125 +1751,212 @@ dp_netdev_actions_free(struct dp_netdev_actions *actions)
     free(actions);
 }
 
-static void *
-dp_forwarder_main(void *f_)
-{
-    struct dp_forwarder *f = f_;
-    struct dp_netdev *dp = f->dp;
-    struct ofpbuf packet;
 
-    f->name = xasprintf("forwarder_%u", ovsthread_id_self());
-    set_subprogram_name("%s", f->name);
+static void
+dp_netdev_process_rxq_port(struct dp_netdev *dp,
+                          struct dp_netdev_port *port,
+                          struct netdev_rxq *rxq)
+{
+    struct ofpbuf *packet[NETDEV_MAX_RX_BATCH];
+    int error, c;
 
-    ofpbuf_init(&packet, 0);
-    while (!latch_is_set(&dp->exit_latch)) {
-        bool received_anything;
+    error = netdev_rxq_recv(rxq, packet, &c);
+    if (!error) {
+        struct pkt_metadata md = PKT_METADATA_INITIALIZER(port->port_no);
         int i;
 
-        ovs_rwlock_rdlock(&dp->port_rwlock);
-        for (i = 0; i < 50; i++) {
-            struct dp_netdev_port *port;
-
-            received_anything = false;
-            HMAP_FOR_EACH (port, node, &f->dp->ports) {
-                if (port->rx
-                    && port->node.hash >= f->min_hash
-                    && port->node.hash <= f->max_hash) {
-                    int buf_size;
-                    int error;
-                    int mtu;
-
-                    if (netdev_get_mtu(port->netdev, &mtu)) {
-                        mtu = ETH_PAYLOAD_MAX;
-                    }
-                    buf_size = DP_NETDEV_HEADROOM + VLAN_ETH_HEADER_LEN + mtu;
-
-                    ofpbuf_clear(&packet);
-                    ofpbuf_reserve_with_tailroom(&packet, DP_NETDEV_HEADROOM,
-                                                 buf_size);
-
-                    error = netdev_rx_recv(port->rx, &packet);
-                    if (!error) {
-                        struct pkt_metadata md
-                            = PKT_METADATA_INITIALIZER(port->port_no);
-
-                        dp_netdev_port_input(dp, &packet, &md);
-                        received_anything = true;
-                    } else if (error != EAGAIN && error != EOPNOTSUPP) {
-                        static struct vlog_rate_limit rl
-                            = VLOG_RATE_LIMIT_INIT(1, 5);
-
-                        VLOG_ERR_RL(&rl, "error receiving data from %s: %s",
-                                    netdev_get_name(port->netdev),
-                                    ovs_strerror(error));
-                    }
-                }
+        for (i = 0; i < c; i++) {
+            dp_netdev_port_input(dp, packet[i], &md);
+        }
+    } else if (error != EAGAIN && error != EOPNOTSUPP) {
+        static struct vlog_rate_limit rl
+            = VLOG_RATE_LIMIT_INIT(1, 5);
+
+        VLOG_ERR_RL(&rl, "error receiving data from %s: %s",
+                    netdev_get_name(port->netdev),
+                    ovs_strerror(error));
+    }
+}
+
+static void
+dpif_netdev_run(struct dpif *dpif)
+{
+    struct dp_netdev_port *port;
+    struct dp_netdev *dp = get_dp_netdev(dpif);
+
+    ovs_rwlock_rdlock(&dp->port_rwlock);
+
+    HMAP_FOR_EACH (port, node, &dp->ports) {
+        if (!netdev_is_pmd(port->netdev)) {
+            int i;
+
+            for (i = 0; i < netdev_n_rxq(port->netdev); i++) {
+                dp_netdev_process_rxq_port(dp, port, port->rxq[i]);
             }
+        }
+    }
 
-            if (!received_anything) {
-                break;
+    ovs_rwlock_unlock(&dp->port_rwlock);
+}
+
+static void
+dpif_netdev_wait(struct dpif *dpif)
+{
+    struct dp_netdev_port *port;
+    struct dp_netdev *dp = get_dp_netdev(dpif);
+
+    ovs_rwlock_rdlock(&dp->port_rwlock);
+
+    HMAP_FOR_EACH (port, node, &dp->ports) {
+        if (!netdev_is_pmd(port->netdev)) {
+            int i;
+
+            for (i = 0; i < netdev_n_rxq(port->netdev); i++) {
+                netdev_rxq_wait(port->rxq[i]);
             }
         }
+    }
+    ovs_rwlock_unlock(&dp->port_rwlock);
+}
 
-        if (received_anything) {
-            poll_immediate_wake();
-        } else {
-            struct dp_netdev_port *port;
+struct rxq_poll {
+    struct dp_netdev_port *port;
+    struct netdev_rxq *rx;
+};
+
+static int
+pmd_load_queues(struct pmd_thread *f,
+                struct rxq_poll **ppoll_list, int poll_cnt)
+{
+    struct dp_netdev *dp = f->dp;
+    struct rxq_poll *poll_list = *ppoll_list;
+    struct dp_netdev_port *port;
+    int id = f->id;
+    int index;
+    int i;
+
+    /* Simple scheduler for netdev rx polling. */
+    ovs_rwlock_rdlock(&dp->port_rwlock);
+    for (i = 0; i < poll_cnt; i++) {
+         port_unref(poll_list[i].port);
+    }
 
-            HMAP_FOR_EACH (port, node, &f->dp->ports)
-                if (port->rx
-                    && port->node.hash >= f->min_hash
-                    && port->node.hash <= f->max_hash) {
-                    netdev_rx_wait(port->rx);
+    poll_cnt = 0;
+    index = 0;
+
+    HMAP_FOR_EACH (port, node, &f->dp->ports) {
+        if (netdev_is_pmd(port->netdev)) {
+            int i;
+
+            for (i = 0; i < netdev_n_rxq(port->netdev); i++) {
+                if ((index % dp->n_pmd_threads) == id) {
+                    poll_list = xrealloc(poll_list, sizeof *poll_list * (poll_cnt + 1));
+
+                    port_ref(port);
+                    poll_list[poll_cnt].port = port;
+                    poll_list[poll_cnt].rx = port->rxq[i];
+                    poll_cnt++;
                 }
-            seq_wait(dp->port_seq, seq_read(dp->port_seq));
-            latch_wait(&dp->exit_latch);
+                index++;
+            }
+        }
+    }
+
+    ovs_rwlock_unlock(&dp->port_rwlock);
+    *ppoll_list = poll_list;
+    return poll_cnt;
+}
+
+static void *
+pmd_thread_main(void *f_)
+{
+    struct pmd_thread *f = f_;
+    struct dp_netdev *dp = f->dp;
+    unsigned int lc = 0;
+    struct rxq_poll *poll_list;
+    unsigned int port_seq;
+    int poll_cnt;
+    int i;
+
+    poll_cnt = 0;
+    poll_list = NULL;
+
+    pmd_thread_setaffinity_cpu(f->id);
+reload:
+    poll_cnt = pmd_load_queues(f, &poll_list, poll_cnt);
+    atomic_read(&f->change_seq, &port_seq);
+
+    for (;;) {
+        unsigned int c_port_seq;
+        int i;
+
+        for (i = 0; i < poll_cnt; i++) {
+            dp_netdev_process_rxq_port(dp,  poll_list[i].port, poll_list[i].rx);
         }
-        ovs_rwlock_unlock(&dp->port_rwlock);
 
-        poll_block();
+        if (lc++ > 1024) {
+            ovsrcu_quiesce();
+
+            /* TODO: need completely userspace based signaling method.
+             * to keep this thread entirely in userspace.
+             * For now using atomic counter. */
+            lc = 0;
+            atomic_read_explicit(&f->change_seq, &c_port_seq, memory_order_consume);
+            if (c_port_seq != port_seq) {
+                break;
+            }
+        }
+    }
+
+    if (!latch_is_set(&f->dp->exit_latch)){
+        goto reload;
     }
-    ofpbuf_uninit(&packet);
 
-    free(f->name);
+    for (i = 0; i < poll_cnt; i++) {
+         port_unref(poll_list[i].port);
+    }
 
+    free(poll_list);
     return NULL;
 }
 
 static void
-dp_netdev_set_threads(struct dp_netdev *dp, int n)
+dp_netdev_set_pmd_threads(struct dp_netdev *dp, int n)
 {
     int i;
 
-    if (n == dp->n_forwarders) {
+    if (n == dp->n_pmd_threads) {
         return;
     }
 
     /* Stop existing threads. */
     latch_set(&dp->exit_latch);
-    for (i = 0; i < dp->n_forwarders; i++) {
-        struct dp_forwarder *f = &dp->forwarders[i];
+    dp_netdev_reload_pmd_threads(dp);
+    for (i = 0; i < dp->n_pmd_threads; i++) {
+        struct pmd_thread *f = &dp->pmd_threads[i];
 
         xpthread_join(f->thread, NULL);
     }
     latch_poll(&dp->exit_latch);
-    free(dp->forwarders);
+    free(dp->pmd_threads);
 
     /* Start new threads. */
-    dp->forwarders = xmalloc(n * sizeof *dp->forwarders);
-    dp->n_forwarders = n;
+    dp->pmd_threads = xmalloc(n * sizeof *dp->pmd_threads);
+    dp->n_pmd_threads = n;
+
     for (i = 0; i < n; i++) {
-        struct dp_forwarder *f = &dp->forwarders[i];
+        struct pmd_thread *f = &dp->pmd_threads[i];
 
         f->dp = dp;
-        f->min_hash = UINT32_MAX / n * i;
-        f->max_hash = UINT32_MAX / n * (i + 1) - 1;
-        if (i == n - 1) {
-            f->max_hash = UINT32_MAX;
-        }
-        xpthread_create(&f->thread, NULL, dp_forwarder_main, f);
+        f->id = i;
+        atomic_store(&f->change_seq, 1);
+
+        /* Each thread will distribute all devices rx-queues among
+         * themselves. */
+        f->thread = ovs_thread_create("pmd", pmd_thread_main, f);
     }
 }
+
 
 static void *
 dp_netdev_flow_stats_new_cb(void)
@@ -1710,9 +1968,10 @@ dp_netdev_flow_stats_new_cb(void)
 
 static void
 dp_netdev_flow_used(struct dp_netdev_flow *netdev_flow,
-                    const struct ofpbuf *packet)
+                    const struct ofpbuf *packet,
+                    const struct miniflow *key)
 {
-    uint16_t tcp_flags = packet_get_tcp_flags(packet, &netdev_flow->flow);
+    uint16_t tcp_flags = miniflow_get_tcp_flags(key);
     long long int now = time_msec();
     struct dp_netdev_flow_stats *bucket;
 
@@ -1722,7 +1981,7 @@ dp_netdev_flow_used(struct dp_netdev_flow *netdev_flow,
     ovs_mutex_lock(&bucket->mutex);
     bucket->used = MAX(now, bucket->used);
     bucket->packet_count++;
-    bucket->byte_count += packet->size;
+    bucket->byte_count += ofpbuf_size(packet);
     bucket->tcp_flags |= tcp_flags;
     ovs_mutex_unlock(&bucket->mutex);
 }
@@ -1747,62 +2006,87 @@ dp_netdev_count_packet(struct dp_netdev *dp, enum dp_stat_type type)
 }
 
 static void
-dp_netdev_port_input(struct dp_netdev *dp, struct ofpbuf *packet,
-                     struct pkt_metadata *md)
+dp_netdev_input(struct dp_netdev *dp, struct ofpbuf *packet,
+                struct pkt_metadata *md)
     OVS_REQ_RDLOCK(dp->port_rwlock)
 {
     struct dp_netdev_flow *netdev_flow;
-    struct flow key;
+    struct {
+        struct miniflow flow;
+        uint32_t buf[FLOW_U32S];
+    } key;
 
-    if (packet->size < ETH_HEADER_LEN) {
+    if (ofpbuf_size(packet) < ETH_HEADER_LEN) {
+        ofpbuf_delete(packet);
         return;
     }
-    flow_extract(packet, md, &key);
-    netdev_flow = dp_netdev_lookup_flow(dp, &key);
+    miniflow_initialize(&key.flow, key.buf);
+    miniflow_extract(packet, md, &key.flow);
+
+    netdev_flow = dp_netdev_lookup_flow(dp, &key.flow);
     if (netdev_flow) {
         struct dp_netdev_actions *actions;
 
-        dp_netdev_flow_used(netdev_flow, packet);
+        dp_netdev_flow_used(netdev_flow, packet, &key.flow);
 
         actions = dp_netdev_flow_get_actions(netdev_flow);
-        dp_netdev_execute_actions(dp, &key, packet, md,
+        dp_netdev_execute_actions(dp, &key.flow, packet, true, md,
                                   actions->actions, actions->size);
         dp_netdev_count_packet(dp, DP_STAT_HIT);
-    } else {
+    } else if (dp->handler_queues) {
         dp_netdev_count_packet(dp, DP_STAT_MISS);
-        dp_netdev_output_userspace(dp, packet, DPIF_UC_MISS, &key, NULL);
+        dp_netdev_output_userspace(dp, packet,
+                                   miniflow_hash_5tuple(&key.flow, 0)
+                                   % dp->n_handlers,
+                                   DPIF_UC_MISS, &key.flow, NULL);
+        ofpbuf_delete(packet);
     }
 }
 
+static void
+dp_netdev_port_input(struct dp_netdev *dp, struct ofpbuf *packet,
+                     struct pkt_metadata *md)
+    OVS_REQ_RDLOCK(dp->port_rwlock)
+{
+    uint32_t *recirc_depth = recirc_depth_get();
+
+    *recirc_depth = 0;
+    dp_netdev_input(dp, packet, md);
+}
+
 static int
 dp_netdev_output_userspace(struct dp_netdev *dp, struct ofpbuf *packet,
-                           int queue_no, const struct flow *flow,
+                           int queue_no, int type, const struct miniflow *key,
                            const struct nlattr *userdata)
-    OVS_EXCLUDED(dp->queue_mutex)
 {
-    struct dp_netdev_queue *q = &dp->queues[queue_no];
+    struct dp_netdev_queue *q;
     int error;
 
-    ovs_mutex_lock(&dp->queue_mutex);
+    fat_rwlock_rdlock(&dp->queue_rwlock);
+    q = &dp->handler_queues[queue_no];
+    ovs_mutex_lock(&q->mutex);
     if (q->head - q->tail < MAX_QUEUE_LEN) {
         struct dp_netdev_upcall *u = &q->upcalls[q->head++ & QUEUE_MASK];
         struct dpif_upcall *upcall = &u->upcall;
         struct ofpbuf *buf = &u->buf;
         size_t buf_size;
+        struct flow flow;
 
-        upcall->type = queue_no;
+        upcall->type = type;
 
         /* Allocate buffer big enough for everything. */
         buf_size = ODPUTIL_FLOW_KEY_BYTES;
         if (userdata) {
             buf_size += NLA_ALIGN(userdata->nla_len);
         }
+        buf_size += ofpbuf_size(packet);
         ofpbuf_init(buf, buf_size);
 
         /* Put ODP flow. */
-        odp_flow_key_from_flow(buf, flow, flow->in_port.odp_port);
-        upcall->key = buf->data;
-        upcall->key_len = buf->size;
+        miniflow_expand(key, &flow);
+        odp_flow_key_from_flow(buf, &flow, NULL, flow.in_port.odp_port);
+        upcall->key = ofpbuf_data(buf);
+        upcall->key_len = ofpbuf_size(buf);
 
         /* Put userdata. */
         if (userdata) {
@@ -1810,43 +2094,44 @@ dp_netdev_output_userspace(struct dp_netdev *dp, struct ofpbuf *packet,
                                           NLA_ALIGN(userdata->nla_len));
         }
 
-        /* Steal packet data. */
-        ovs_assert(packet->source == OFPBUF_MALLOC);
-        upcall->packet = *packet;
-        ofpbuf_use(packet, NULL, 0);
+        ofpbuf_set_data(&upcall->packet,
+                        ofpbuf_put(buf, ofpbuf_data(packet), ofpbuf_size(packet)));
+        ofpbuf_set_size(&upcall->packet, ofpbuf_size(packet));
 
-        seq_change(dp->queue_seq);
+        seq_change(q->seq);
 
         error = 0;
     } else {
         dp_netdev_count_packet(dp, DP_STAT_LOST);
         error = ENOBUFS;
     }
-    ovs_mutex_unlock(&dp->queue_mutex);
+    ovs_mutex_unlock(&q->mutex);
+    fat_rwlock_unlock(&dp->queue_rwlock);
 
     return error;
 }
 
 struct dp_netdev_execute_aux {
     struct dp_netdev *dp;
-    const struct flow *key;
+    const struct miniflow *key;
 };
 
 static void
 dp_execute_cb(void *aux_, struct ofpbuf *packet,
-              const struct pkt_metadata *md OVS_UNUSED,
+              struct pkt_metadata *md,
               const struct nlattr *a, bool may_steal)
     OVS_NO_THREAD_SAFETY_ANALYSIS
 {
     struct dp_netdev_execute_aux *aux = aux_;
     int type = nl_attr_type(a);
     struct dp_netdev_port *p;
+    uint32_t *depth = recirc_depth_get();
 
     switch ((enum ovs_action_attr)type) {
     case OVS_ACTION_ATTR_OUTPUT:
         p = dp_netdev_lookup_port(aux->dp, u32_to_odp(nl_attr_get_u32(a)));
         if (p) {
-            netdev_send(p->netdev, packet);
+            netdev_send(p->netdev, packet, may_steal);
         }
         break;
 
@@ -1855,17 +2140,58 @@ dp_execute_cb(void *aux_, struct ofpbuf *packet,
 
         userdata = nl_attr_find_nested(a, OVS_USERSPACE_ATTR_USERDATA);
 
-        /* Make a copy if we are not allowed to steal the packet's data. */
-        if (!may_steal) {
-            packet = ofpbuf_clone_with_headroom(packet, DP_NETDEV_HEADROOM);
-        }
-        dp_netdev_output_userspace(aux->dp, packet, DPIF_UC_ACTION, aux->key,
+        dp_netdev_output_userspace(aux->dp, packet,
+                                   miniflow_hash_5tuple(aux->key, 0)
+                                       % aux->dp->n_handlers,
+                                   DPIF_UC_ACTION, aux->key,
                                    userdata);
-        if (!may_steal) {
-            ofpbuf_uninit(packet);
+
+        if (may_steal) {
+            ofpbuf_delete(packet);
+        }
+        break;
+    }
+
+    case OVS_ACTION_ATTR_HASH: {
+        const struct ovs_action_hash *hash_act;
+        uint32_t hash;
+
+        hash_act = nl_attr_get(a);
+        if (hash_act->hash_alg == OVS_HASH_ALG_L4) {
+            /* Hash need not be symmetric, nor does it need to include
+             * L2 fields. */
+            hash = miniflow_hash_5tuple(aux->key, hash_act->hash_basis);
+            if (!hash) {
+                hash = 1; /* 0 is not valid */
+            }
+
+        } else {
+            VLOG_WARN("Unknown hash algorithm specified for the hash action.");
+            hash = 2;
         }
+
+        md->dp_hash = hash;
         break;
     }
+
+    case OVS_ACTION_ATTR_RECIRC:
+        if (*depth < MAX_RECIRC_DEPTH) {
+            struct pkt_metadata recirc_md = *md;
+            struct ofpbuf *recirc_packet;
+
+            recirc_packet = may_steal ? packet : ofpbuf_clone(packet);
+            recirc_md.recirc_id = nl_attr_get_u32(a);
+
+            (*depth)++;
+            dp_netdev_input(aux->dp, recirc_packet, &recirc_md);
+            (*depth)--;
+
+            break;
+        } else {
+            VLOG_WARN("Packet dropped. Max recirculation depth exceeded.");
+        }
+        break;
+
     case OVS_ACTION_ATTR_PUSH_VLAN:
     case OVS_ACTION_ATTR_POP_VLAN:
     case OVS_ACTION_ATTR_PUSH_MPLS:
@@ -1879,14 +2205,15 @@ dp_execute_cb(void *aux_, struct ofpbuf *packet,
 }
 
 static void
-dp_netdev_execute_actions(struct dp_netdev *dp, const struct flow *key,
-                          struct ofpbuf *packet, struct pkt_metadata *md,
+dp_netdev_execute_actions(struct dp_netdev *dp, const struct miniflow *key,
+                          struct ofpbuf *packet, bool may_steal,
+                          struct pkt_metadata *md,
                           const struct nlattr *actions, size_t actions_len)
-    OVS_REQ_RDLOCK(dp->port_rwlock)
 {
     struct dp_netdev_execute_aux aux = {dp, key};
 
-    odp_execute_actions(&aux, packet, md, actions, actions_len, dp_execute_cb);
+    odp_execute_actions(&aux, packet, may_steal, md,
+                        actions, actions_len, dp_execute_cb);
 }
 
 const struct dpif_class dpif_netdev_class = {
@@ -1896,8 +2223,8 @@ const struct dpif_class dpif_netdev_class = {
     dpif_netdev_open,
     dpif_netdev_close,
     dpif_netdev_destroy,
-    NULL,                       /* run */
-    NULL,                       /* wait */
+    dpif_netdev_run,
+    dpif_netdev_wait,
     dpif_netdev_get_stats,
     dpif_netdev_port_add,
     dpif_netdev_port_del,
@@ -1922,6 +2249,7 @@ const struct dpif_class dpif_netdev_class = {
     dpif_netdev_execute,
     NULL,                       /* operate */
     dpif_netdev_recv_set,
+    dpif_netdev_handlers_set,
     dpif_netdev_queue_to_priority,
     dpif_netdev_recv,
     dpif_netdev_recv_wait,