+ ofpbuf_use_stack(&xout_actions, slow_path_buf, sizeof slow_path_buf);
+ compose_slow_path(udpif, &xout, &flow, odp_in_port, &xout_actions);
+ }
+
+ if (!ofpbuf_equal(&xout_actions, actions)) {
+ goto exit;
+ }
+
+ if (odp_flow_key_to_mask(udump->mask, udump->mask_len, &udump_mask, &flow)
+ == ODP_FIT_ERROR) {
+ goto exit;
+ }
+
+ /* Since the kernel is free to ignore wildcarded bits in the mask, we can't
+ * directly check that the masks are the same. Instead we check that the
+ * mask in the kernel is more specific i.e. less wildcarded, than what
+ * we've calculated here. This guarantees we don't catch any packets we
+ * shouldn't with the megaflow. */
+ udump32 = (uint32_t *) &udump_mask;
+ xout32 = (uint32_t *) &xout.wc.masks;
+ for (i = 0; i < FLOW_U32S; i++) {
+ if ((udump32[i] | xout32[i]) != udump32[i]) {
+ goto exit;
+ }
+ }
+ ok = true;
+
+exit:
+ if (netflow) {
+ if (!ok) {
+ netflow_expire(netflow, &flow);
+ netflow_flow_clear(netflow, &flow);
+ }
+ netflow_unref(netflow);
+ }
+ ofpbuf_delete(actions);
+ xlate_out_uninit(xoutp);
+ return ok;
+}
+
+struct dump_op {
+ struct udpif_key *ukey;
+ struct udpif_flow_dump *udump;
+ struct dpif_flow_stats stats; /* Stats for 'op'. */
+ struct dpif_op op; /* Flow del operation. */
+};
+
+static void
+dump_op_init(struct dump_op *op, const struct nlattr *key, size_t key_len,
+ struct udpif_key *ukey, struct udpif_flow_dump *udump)
+{
+ op->ukey = ukey;
+ op->udump = udump;
+ op->op.type = DPIF_OP_FLOW_DEL;
+ op->op.u.flow_del.key = key;
+ op->op.u.flow_del.key_len = key_len;
+ op->op.u.flow_del.stats = &op->stats;
+}
+
+static void
+push_dump_ops(struct revalidator *revalidator,
+ struct dump_op *ops, size_t n_ops)
+{
+ struct udpif *udpif = revalidator->udpif;
+ struct dpif_op *opsp[REVALIDATE_MAX_BATCH];
+ size_t i;
+
+ ovs_assert(n_ops <= REVALIDATE_MAX_BATCH);
+ for (i = 0; i < n_ops; i++) {
+ opsp[i] = &ops[i].op;
+ }
+ dpif_operate(udpif->dpif, opsp, n_ops);
+
+ for (i = 0; i < n_ops; i++) {
+ struct dump_op *op = &ops[i];
+ struct dpif_flow_stats *push, *stats, push_buf;
+
+ stats = op->op.u.flow_del.stats;
+ if (op->ukey) {
+ push = &push_buf;
+ push->used = MAX(stats->used, op->ukey->stats.used);
+ push->tcp_flags = stats->tcp_flags | op->ukey->stats.tcp_flags;
+ push->n_packets = stats->n_packets - op->ukey->stats.n_packets;
+ push->n_bytes = stats->n_bytes - op->ukey->stats.n_bytes;
+ } else {
+ push = stats;
+ }
+
+ if (push->n_packets || netflow_exists()) {
+ struct ofproto_dpif *ofproto;
+ struct netflow *netflow;
+ struct flow flow;
+ bool may_learn;
+
+ may_learn = push->n_packets > 0;
+ if (op->ukey && op->ukey->xcache) {
+ xlate_push_stats(op->ukey->xcache, may_learn, push);
+ continue;
+ }
+
+ if (!xlate_receive(udpif->backer, NULL, op->op.u.flow_del.key,
+ op->op.u.flow_del.key_len, &flow, &ofproto,
+ NULL, NULL, &netflow, NULL)) {
+ struct xlate_in xin;
+
+ xlate_in_init(&xin, ofproto, &flow, NULL, push->tcp_flags,
+ NULL);
+ xin.resubmit_stats = push->n_packets ? push : NULL;
+ xin.may_learn = may_learn;
+ xin.skip_wildcards = true;
+ xlate_actions_for_side_effects(&xin);
+
+ if (netflow) {
+ netflow_expire(netflow, &flow);
+ netflow_flow_clear(netflow, &flow);
+ netflow_unref(netflow);
+ }
+ }
+ }
+ }
+
+ for (i = 0; i < n_ops; i++) {
+ struct udpif_key *ukey;
+
+ /* If there's a udump, this ukey came directly from a datapath flow
+ * dump. Sometimes a datapath can send duplicates in flow dumps, in
+ * which case we wouldn't want to double-free a ukey, so avoid that by
+ * looking up the ukey again.
+ *
+ * If there's no udump then we know what we're doing. */
+ ukey = (ops[i].udump
+ ? ukey_lookup(revalidator, ops[i].udump)
+ : ops[i].ukey);
+ if (ukey) {
+ ukey_delete(revalidator, ukey);
+ }
+ }
+}
+
+static void
+revalidate_udumps(struct revalidator *revalidator, struct list *udumps)
+{
+ struct udpif *udpif = revalidator->udpif;
+
+ struct dump_op ops[REVALIDATE_MAX_BATCH];
+ struct udpif_flow_dump *udump, *next_udump;
+ size_t n_ops, n_flows;
+ unsigned int flow_limit;
+ long long int max_idle;
+ bool must_del;
+
+ atomic_read(&udpif->flow_limit, &flow_limit);
+
+ n_flows = udpif_get_n_flows(udpif);
+
+ must_del = false;
+ max_idle = ofproto_max_idle;
+ if (n_flows > flow_limit) {
+ must_del = n_flows > 2 * flow_limit;
+ max_idle = 100;
+ }
+
+ n_ops = 0;
+ LIST_FOR_EACH_SAFE (udump, next_udump, list_node, udumps) {
+ long long int used, now;
+ struct udpif_key *ukey;
+
+ now = time_msec();
+ ukey = ukey_lookup(revalidator, udump);
+
+ used = udump->stats.used;
+ if (!used && ukey) {
+ used = ukey->created;
+ }
+
+ if (ukey && (ukey->mark || !ukey->flow_exists)) {
+ /* The flow has already been dumped. This can occasionally occur
+ * if the datapath is changed in the middle of a flow dump. Rather
+ * than perform the same work twice, skip the flow this time. */
+ COVERAGE_INC(upcall_duplicate_flow);
+ continue;
+ }
+
+ if (must_del || (used && used < now - max_idle)) {
+ struct dump_op *dop = &ops[n_ops++];
+
+ if (ukey) {
+ ukey->flow_exists = false;
+ }
+ dump_op_init(dop, udump->key, udump->key_len, ukey, udump);
+ continue;
+ }
+
+ if (!ukey) {
+ ukey = ukey_create(udump->key, udump->key_len, used);
+ hmap_insert(&revalidator->ukeys, &ukey->hmap_node,
+ udump->key_hash);
+ }
+ ukey->mark = true;
+
+ if (!revalidate_ukey(udpif, udump, ukey)) {
+ ukey->flow_exists = false;
+ dpif_flow_del(udpif->dpif, udump->key, udump->key_len, NULL);
+ /* The ukey will be cleaned up by revalidator_sweep().
+ * This helps to avoid deleting the same flow twice. */
+ }
+
+ list_remove(&udump->list_node);
+ free(udump);
+ }
+
+ push_dump_ops(revalidator, ops, n_ops);
+
+ LIST_FOR_EACH_SAFE (udump, next_udump, list_node, udumps) {
+ list_remove(&udump->list_node);
+ free(udump);
+ }
+}
+
+static void
+revalidator_sweep__(struct revalidator *revalidator, bool purge)
+{
+ struct dump_op ops[REVALIDATE_MAX_BATCH];
+ struct udpif_key *ukey, *next;
+ size_t n_ops;
+
+ n_ops = 0;
+
+ HMAP_FOR_EACH_SAFE (ukey, next, hmap_node, &revalidator->ukeys) {
+ if (!purge && ukey->mark) {
+ ukey->mark = false;
+ } else if (!ukey->flow_exists) {
+ ukey_delete(revalidator, ukey);
+ } else {
+ struct dump_op *op = &ops[n_ops++];
+
+ /* If we have previously seen a flow in the datapath, but didn't
+ * see it during the most recent dump, delete it. This allows us
+ * to clean up the ukey and keep the statistics consistent. */
+ dump_op_init(op, ukey->key, ukey->key_len, ukey, NULL);
+ if (n_ops == REVALIDATE_MAX_BATCH) {
+ push_dump_ops(revalidator, ops, n_ops);
+ n_ops = 0;
+ }
+ }
+ }
+
+ if (n_ops) {
+ push_dump_ops(revalidator, ops, n_ops);
+ }
+}
+
+static void
+revalidator_sweep(struct revalidator *revalidator)
+{
+ revalidator_sweep__(revalidator, false);
+}
+
+static void
+revalidator_purge(struct revalidator *revalidator)
+{
+ revalidator_sweep__(revalidator, true);
+}
+\f
+static void
+upcall_unixctl_show(struct unixctl_conn *conn, int argc OVS_UNUSED,
+ const char *argv[] OVS_UNUSED, void *aux OVS_UNUSED)
+{
+ struct ds ds = DS_EMPTY_INITIALIZER;
+ struct udpif *udpif;
+
+ LIST_FOR_EACH (udpif, list_node, &all_udpifs) {
+ unsigned int flow_limit;
+ size_t i;
+
+ atomic_read(&udpif->flow_limit, &flow_limit);
+
+ ds_put_format(&ds, "%s:\n", dpif_name(udpif->dpif));
+ ds_put_format(&ds, "\tflows : (current %"PRIu64")"
+ " (avg %u) (max %u) (limit %u)\n", udpif_get_n_flows(udpif),
+ udpif->avg_n_flows, udpif->max_n_flows, flow_limit);
+ ds_put_format(&ds, "\tdump duration : %lldms\n", udpif->dump_duration);
+
+ ds_put_char(&ds, '\n');
+ for (i = 0; i < n_revalidators; i++) {
+ struct revalidator *revalidator = &udpif->revalidators[i];
+
+ /* XXX: The result of hmap_count(&revalidator->ukeys) may not be
+ * accurate because it's not protected by the revalidator mutex. */
+ ovs_mutex_lock(&revalidator->mutex);
+ ds_put_format(&ds, "\t%s: (dump queue %"PRIuSIZE") (keys %"PRIuSIZE
+ ")\n", revalidator->name, revalidator->n_udumps,
+ hmap_count(&revalidator->ukeys));
+ ovs_mutex_unlock(&revalidator->mutex);
+ }
+ }
+
+ unixctl_command_reply(conn, ds_cstr(&ds));
+ ds_destroy(&ds);
+}
+
+/* Disable using the megaflows.
+ *
+ * This command is only needed for advanced debugging, so it's not
+ * documented in the man page. */
+static void
+upcall_unixctl_disable_megaflows(struct unixctl_conn *conn,
+ int argc OVS_UNUSED,
+ const char *argv[] OVS_UNUSED,
+ void *aux OVS_UNUSED)
+{
+ atomic_store(&enable_megaflows, false);
+ udpif_flush_all_datapaths();
+ unixctl_command_reply(conn, "megaflows disabled");
+}
+
+/* Re-enable using megaflows.
+ *
+ * This command is only needed for advanced debugging, so it's not
+ * documented in the man page. */
+static void
+upcall_unixctl_enable_megaflows(struct unixctl_conn *conn,
+ int argc OVS_UNUSED,
+ const char *argv[] OVS_UNUSED,
+ void *aux OVS_UNUSED)
+{
+ atomic_store(&enable_megaflows, true);
+ udpif_flush_all_datapaths();
+ unixctl_command_reply(conn, "megaflows enabled");
+}
+
+/* Set the flow limit.
+ *
+ * This command is only needed for advanced debugging, so it's not
+ * documented in the man page. */
+static void
+upcall_unixctl_set_flow_limit(struct unixctl_conn *conn,
+ int argc OVS_UNUSED,
+ const char *argv[] OVS_UNUSED,
+ void *aux OVS_UNUSED)
+{
+ struct ds ds = DS_EMPTY_INITIALIZER;
+ struct udpif *udpif;
+ unsigned int flow_limit = atoi(argv[1]);
+
+ LIST_FOR_EACH (udpif, list_node, &all_udpifs) {
+ atomic_store(&udpif->flow_limit, flow_limit);