- VLOG_WARN_RL(&rl, "unexpected byte count from datapath");
- diff.n_bytes = 0;
- }
-
- facet->ofproto->n_hit += diff.n_packets;
- subfacet->dp_packet_count = stats->n_packets;
- subfacet->dp_byte_count = stats->n_bytes;
- subfacet_update_stats(subfacet, &diff);
-
- if (facet->accounted_bytes < facet->byte_count) {
- facet_learn(facet);
- facet_account(facet);
- facet->accounted_bytes = facet->byte_count;
- }
-}
-
-/* 'key' with length 'key_len' bytes is a flow in 'dpif' that we know nothing
- * about, or a flow that shouldn't be installed but was anyway. Delete it. */
-static void
-delete_unexpected_flow(struct dpif_backer *backer,
- const struct nlattr *key, size_t key_len)
-{
- if (!VLOG_DROP_WARN(&rl)) {
- struct ds s;
-
- ds_init(&s);
- odp_flow_key_format(key, key_len, &s);
- VLOG_WARN("unexpected flow: %s", ds_cstr(&s));
- ds_destroy(&s);
- }
-
- COVERAGE_INC(facet_unexpected);
- dpif_flow_del(backer->dpif, key, key_len, NULL);
-}
-
-/* Update 'packet_count', 'byte_count', and 'used' members of installed facets.
- *
- * This function also pushes statistics updates to rules which each facet
- * resubmits into. Generally these statistics will be accurate. However, if a
- * facet changes the rule it resubmits into at some time in between
- * update_stats() runs, it is possible that statistics accrued to the
- * old rule will be incorrectly attributed to the new rule. This could be
- * avoided by calling update_stats() whenever rules are created or
- * deleted. However, the performance impact of making so many calls to the
- * datapath do not justify the benefit of having perfectly accurate statistics.
- *
- * In addition, this function maintains per ofproto flow hit counts. The patch
- * port is not treated specially. e.g. A packet ingress from br0 patched into
- * br1 will increase the hit count of br0 by 1, however, does not affect
- * the hit or miss counts of br1.
- */
-static void
-update_stats(struct dpif_backer *backer)
-{
- const struct dpif_flow_stats *stats;
- struct dpif_flow_dump dump;
- const struct nlattr *key, *mask;
- size_t key_len, mask_len;
-
- dpif_flow_dump_start(&dump, backer->dpif);
- while (dpif_flow_dump_next(&dump, &key, &key_len,
- &mask, &mask_len, NULL, NULL, &stats)) {
- struct subfacet *subfacet;
- uint32_t key_hash;
-
- key_hash = odp_flow_key_hash(key, key_len);
- subfacet = subfacet_find(backer, key, key_len, key_hash);
- switch (subfacet ? subfacet->path : SF_NOT_INSTALLED) {
- case SF_FAST_PATH:
- update_subfacet_stats(subfacet, stats);
- break;
-
- case SF_SLOW_PATH:
- /* Stats are updated per-packet. */
- break;
-
- case SF_NOT_INSTALLED:
- default:
- delete_unexpected_flow(backer, key, key_len);
- break;
- }
- run_fast_rl();
- }
- dpif_flow_dump_done(&dump);
-}
-
-/* Calculates and returns the number of milliseconds of idle time after which
- * subfacets should expire from the datapath. When a subfacet expires, we fold
- * its statistics into its facet, and when a facet's last subfacet expires, we
- * fold its statistic into its rule. */
-static int
-subfacet_max_idle(const struct dpif_backer *backer)
-{
- /*
- * Idle time histogram.
- *
- * Most of the time a switch has a relatively small number of subfacets.
- * When this is the case we might as well keep statistics for all of them
- * in userspace and to cache them in the kernel datapath for performance as
- * well.
- *
- * As the number of subfacets increases, the memory required to maintain
- * statistics about them in userspace and in the kernel becomes
- * significant. However, with a large number of subfacets it is likely
- * that only a few of them are "heavy hitters" that consume a large amount
- * of bandwidth. At this point, only heavy hitters are worth caching in
- * the kernel and maintaining in userspaces; other subfacets we can
- * discard.
- *
- * The technique used to compute the idle time is to build a histogram with
- * N_BUCKETS buckets whose width is BUCKET_WIDTH msecs each. Each subfacet
- * that is installed in the kernel gets dropped in the appropriate bucket.
- * After the histogram has been built, we compute the cutoff so that only
- * the most-recently-used 1% of subfacets (but at least
- * flow_eviction_threshold flows) are kept cached. At least
- * the most-recently-used bucket of subfacets is kept, so actually an
- * arbitrary number of subfacets can be kept in any given expiration run
- * (though the next run will delete most of those unless they receive
- * additional data).
- *
- * This requires a second pass through the subfacets, in addition to the
- * pass made by update_stats(), because the former function never looks at
- * uninstallable subfacets.
- */
- enum { BUCKET_WIDTH = 100 };
- enum { N_BUCKETS = 5000 / BUCKET_WIDTH };
- int buckets[N_BUCKETS] = { 0 };
- int total, subtotal, bucket;
- struct subfacet *subfacet;
- long long int now;
- int i;
-
- total = hmap_count(&backer->subfacets);
- if (total <= flow_eviction_threshold) {
- return N_BUCKETS * BUCKET_WIDTH;
- }
-
- /* Build histogram. */
- now = time_msec();
- HMAP_FOR_EACH (subfacet, hmap_node, &backer->subfacets) {
- long long int idle = now - subfacet->used;
- int bucket = (idle <= 0 ? 0
- : idle >= BUCKET_WIDTH * N_BUCKETS ? N_BUCKETS - 1
- : (unsigned int) idle / BUCKET_WIDTH);
- buckets[bucket]++;
- }
-
- /* Find the first bucket whose flows should be expired. */
- subtotal = bucket = 0;
- do {
- subtotal += buckets[bucket++];
- } while (bucket < N_BUCKETS &&
- subtotal < MAX(flow_eviction_threshold, total / 100));
-
- if (VLOG_IS_DBG_ENABLED()) {
- struct ds s;
-
- ds_init(&s);
- ds_put_cstr(&s, "keep");
- for (i = 0; i < N_BUCKETS; i++) {
- if (i == bucket) {
- ds_put_cstr(&s, ", drop");
- }
- if (buckets[i]) {
- ds_put_format(&s, " %d:%d", i * BUCKET_WIDTH, buckets[i]);
- }
- }
- VLOG_INFO("%s (msec:count)", ds_cstr(&s));
- ds_destroy(&s);
- }
-
- return bucket * BUCKET_WIDTH;
-}
-
-static void
-expire_subfacets(struct dpif_backer *backer, int dp_max_idle)
-{
- /* Cutoff time for most flows. */
- long long int normal_cutoff = time_msec() - dp_max_idle;
-
- /* We really want to keep flows for special protocols around, so use a more
- * conservative cutoff. */
- long long int special_cutoff = time_msec() - 10000;
-
- struct subfacet *subfacet, *next_subfacet;
- struct subfacet *batch[SUBFACET_DESTROY_MAX_BATCH];
- int n_batch;
-
- n_batch = 0;
- HMAP_FOR_EACH_SAFE (subfacet, next_subfacet, hmap_node,
- &backer->subfacets) {
- long long int cutoff;
-
- cutoff = (subfacet->facet->xout.slow & (SLOW_CFM | SLOW_BFD | SLOW_LACP
- | SLOW_STP)
- ? special_cutoff
- : normal_cutoff);
- if (subfacet->used < cutoff) {
- if (subfacet->path != SF_NOT_INSTALLED) {
- batch[n_batch++] = subfacet;
- if (n_batch >= SUBFACET_DESTROY_MAX_BATCH) {
- subfacet_destroy_batch(backer, batch, n_batch);
- n_batch = 0;
- }
- } else {
- subfacet_destroy(subfacet);
- }
- }
- }
-
- if (n_batch > 0) {
- subfacet_destroy_batch(backer, batch, n_batch);
- }
-}
-
-/* If 'rule' is an OpenFlow rule, that has expired according to OpenFlow rules,
- * then delete it entirely. */
-static void
-rule_expire(struct rule_dpif *rule)
- OVS_REQUIRES(ofproto_mutex)
-{
- uint16_t idle_timeout, hard_timeout;
- long long int now = time_msec();
- int reason;
-
- ovs_assert(!rule->up.pending);
-
- /* Has 'rule' expired? */
- ovs_mutex_lock(&rule->up.mutex);
- hard_timeout = rule->up.hard_timeout;
- idle_timeout = rule->up.idle_timeout;
- if (hard_timeout && now > rule->up.modified + hard_timeout * 1000) {
- reason = OFPRR_HARD_TIMEOUT;
- } else if (idle_timeout && now > rule->up.used + idle_timeout * 1000) {
- reason = OFPRR_IDLE_TIMEOUT;
- } else {
- reason = -1;
- }
- ovs_mutex_unlock(&rule->up.mutex);
-
- if (reason >= 0) {
- COVERAGE_INC(ofproto_dpif_expired);
- ofproto_rule_expire(&rule->up, reason);
- }
-}
-\f
-/* Facets. */
-
-/* Creates and returns a new facet based on 'miss'.
- *
- * The caller must already have determined that no facet with an identical
- * 'miss->flow' exists in 'miss->ofproto'.
- *
- * 'rule' and 'xout' must have been created based on 'miss'.
- *
- * 'facet'' statistics are initialized based on 'stats'.
- *
- * The facet will initially have no subfacets. The caller should create (at
- * least) one subfacet with subfacet_create(). */
-static struct facet *
-facet_create(const struct flow_miss *miss)
-{
- struct ofproto_dpif *ofproto = miss->ofproto;
- struct facet *facet;
- struct match match;
-
- COVERAGE_INC(facet_create);
- facet = xzalloc(sizeof *facet);
- facet->ofproto = miss->ofproto;
- facet->used = miss->stats.used;
- facet->flow = miss->flow;
- facet->learn_rl = time_msec() + 500;
-
- list_init(&facet->subfacets);
-
- xlate_out_copy(&facet->xout, &miss->xout);
-
- match_init(&match, &facet->flow, &facet->xout.wc);
- cls_rule_init(&facet->cr, &match, OFP_DEFAULT_PRIORITY);
- ovs_rwlock_wrlock(&ofproto->facets.rwlock);
- classifier_insert(&ofproto->facets, &facet->cr);
- ovs_rwlock_unlock(&ofproto->facets.rwlock);
-
- if (ofproto->netflow && !facet_is_controller_flow(facet)) {
- netflow_flow_update(ofproto->netflow, &facet->flow,
- facet->xout.nf_output_iface, &miss->stats);
- }
-
- return facet;
-}
-
-static void
-facet_free(struct facet *facet)
-{
- if (facet) {
- xlate_out_uninit(&facet->xout);
- free(facet);
- }
-}
-
-/* Executes, within 'ofproto', the actions in 'rule' or 'ofpacts' on 'packet'.
- * 'flow' must reflect the data in 'packet'. */
-int
-ofproto_dpif_execute_actions(struct ofproto_dpif *ofproto,
- const struct flow *flow,
- struct rule_dpif *rule,
- const struct ofpact *ofpacts, size_t ofpacts_len,
- struct ofpbuf *packet)
-{
- struct odputil_keybuf keybuf;
- struct dpif_flow_stats stats;
- struct xlate_out xout;
- struct xlate_in xin;
- ofp_port_t in_port;
- struct ofpbuf key;
- int error;
-
- ovs_assert((rule != NULL) != (ofpacts != NULL));
-
- dpif_flow_stats_extract(flow, packet, time_msec(), &stats);
- if (rule) {
- rule_dpif_credit_stats(rule, &stats);
- }
-
- xlate_in_init(&xin, ofproto, flow, rule, stats.tcp_flags, packet);
- xin.ofpacts = ofpacts;
- xin.ofpacts_len = ofpacts_len;
- xin.resubmit_stats = &stats;
- xlate_actions(&xin, &xout);
-
- ofpbuf_use_stack(&key, &keybuf, sizeof keybuf);
- in_port = flow->in_port.ofp_port;
- if (in_port == OFPP_NONE) {
- in_port = OFPP_LOCAL;
- }
- odp_flow_key_from_flow(&key, flow, ofp_port_to_odp_port(ofproto, in_port));
-
- error = dpif_execute(ofproto->backer->dpif, key.data, key.size,
- xout.odp_actions.data, xout.odp_actions.size, packet,
- (xout.slow & SLOW_ACTION) != 0);
- xlate_out_uninit(&xout);
-
- return error;
-}
-
-/* Remove 'facet' from its ofproto and free up the associated memory:
- *
- * - If 'facet' was installed in the datapath, uninstalls it and updates its
- * rule's statistics, via subfacet_uninstall().
- *
- * - Removes 'facet' from its rule and from ofproto->facets.
- */
-static void
-facet_remove(struct facet *facet)
-{
- struct subfacet *subfacet, *next_subfacet;
-
- COVERAGE_INC(facet_remove);
- ovs_assert(!list_is_empty(&facet->subfacets));
-
- /* First uninstall all of the subfacets to get final statistics. */
- LIST_FOR_EACH (subfacet, list_node, &facet->subfacets) {
- subfacet_uninstall(subfacet);
- }
-
- /* Flush the final stats to the rule.
- *
- * This might require us to have at least one subfacet around so that we
- * can use its actions for accounting in facet_account(), which is why we
- * have uninstalled but not yet destroyed the subfacets. */
- facet_flush_stats(facet);
-
- /* Now we're really all done so destroy everything. */
- LIST_FOR_EACH_SAFE (subfacet, next_subfacet, list_node,
- &facet->subfacets) {
- subfacet_destroy__(subfacet);
- }
- ovs_rwlock_wrlock(&facet->ofproto->facets.rwlock);
- classifier_remove(&facet->ofproto->facets, &facet->cr);
- ovs_rwlock_unlock(&facet->ofproto->facets.rwlock);
- cls_rule_destroy(&facet->cr);
- facet_free(facet);
-}
-
-/* Feed information from 'facet' back into the learning table to keep it in
- * sync with what is actually flowing through the datapath. */
-static void
-facet_learn(struct facet *facet)
-{
- long long int now = time_msec();
-
- if (!facet->xout.has_fin_timeout && now < facet->learn_rl) {
- return;
- }
-
- facet->learn_rl = now + 500;
-
- if (!facet->xout.has_learn
- && !facet->xout.has_normal
- && (!facet->xout.has_fin_timeout
- || !(facet->tcp_flags & (TCP_FIN | TCP_RST)))) {
- return;
- }
-
- facet_push_stats(facet, true);
-}
-
-static void
-facet_account(struct facet *facet)
-{
- const struct nlattr *a;
- unsigned int left;
- ovs_be16 vlan_tci;
- uint64_t n_bytes;
-
- if (!facet->xout.has_normal || !facet->ofproto->has_bonded_bundles) {
- return;
- }
- n_bytes = facet->byte_count - facet->accounted_bytes;
-
- /* This loop feeds byte counters to bond_account() for rebalancing to use
- * as a basis. We also need to track the actual VLAN on which the packet
- * is going to be sent to ensure that it matches the one passed to
- * bond_choose_output_slave(). (Otherwise, we will account to the wrong
- * hash bucket.)
- *
- * We use the actions from an arbitrary subfacet because they should all
- * be equally valid for our purpose. */
- vlan_tci = facet->flow.vlan_tci;
- NL_ATTR_FOR_EACH_UNSAFE (a, left, facet->xout.odp_actions.data,
- facet->xout.odp_actions.size) {
- const struct ovs_action_push_vlan *vlan;
- struct ofport_dpif *port;
-
- switch (nl_attr_type(a)) {
- case OVS_ACTION_ATTR_OUTPUT:
- port = get_odp_port(facet->ofproto, nl_attr_get_odp_port(a));
- if (port && port->bundle && port->bundle->bond) {
- bond_account(port->bundle->bond, &facet->flow,
- vlan_tci_to_vid(vlan_tci), n_bytes);
- }
- break;
-
- case OVS_ACTION_ATTR_POP_VLAN:
- vlan_tci = htons(0);
- break;
-
- case OVS_ACTION_ATTR_PUSH_VLAN:
- vlan = nl_attr_get(a);
- vlan_tci = vlan->vlan_tci;
- break;
- }
- }
-}
-
-/* Returns true if the only action for 'facet' is to send to the controller.
- * (We don't report NetFlow expiration messages for such facets because they
- * are just part of the control logic for the network, not real traffic). */
-static bool
-facet_is_controller_flow(struct facet *facet)
-{
- if (facet) {
- struct ofproto_dpif *ofproto = facet->ofproto;
- const struct ofpact *ofpacts;
- struct rule_actions *actions;
- struct rule_dpif *rule;
- size_t ofpacts_len;
- bool is_controller;
-
- rule_dpif_lookup(ofproto, &facet->flow, NULL, &rule);
- actions = rule_dpif_get_actions(rule);
- rule_dpif_unref(rule);
-
- ofpacts_len = actions->ofpacts_len;
- ofpacts = actions->ofpacts;
- is_controller = ofpacts_len > 0
- && ofpacts->type == OFPACT_CONTROLLER
- && ofpact_next(ofpacts) >= ofpact_end(ofpacts, ofpacts_len);
- rule_actions_unref(actions);
-
- return is_controller;
- }
- return false;
-}
-
-/* Folds all of 'facet''s statistics into its rule. Also updates the
- * accounting ofhook and emits a NetFlow expiration if appropriate. All of
- * 'facet''s statistics in the datapath should have been zeroed and folded into
- * its packet and byte counts before this function is called. */
-static void
-facet_flush_stats(struct facet *facet)
-{
- struct ofproto_dpif *ofproto = facet->ofproto;
- struct subfacet *subfacet;
-
- LIST_FOR_EACH (subfacet, list_node, &facet->subfacets) {
- ovs_assert(!subfacet->dp_byte_count);
- ovs_assert(!subfacet->dp_packet_count);
- }
-
- facet_push_stats(facet, false);
- if (facet->accounted_bytes < facet->byte_count) {
- facet_account(facet);
- facet->accounted_bytes = facet->byte_count;
- }
-
- if (ofproto->netflow && !facet_is_controller_flow(facet)) {
- netflow_expire(ofproto->netflow, &facet->flow);
- netflow_flow_clear(ofproto->netflow, &facet->flow);
- }
-
- /* Reset counters to prevent double counting if 'facet' ever gets
- * reinstalled. */
- facet_reset_counters(facet);
- facet->tcp_flags = 0;
-}
-
-/* Searches 'ofproto''s table of facets for one which would be responsible for
- * 'flow'. Returns it if found, otherwise a null pointer.
- *
- * The returned facet might need revalidation; use facet_lookup_valid()
- * instead if that is important. */
-static struct facet *
-facet_find(struct ofproto_dpif *ofproto, const struct flow *flow)
-{
- struct cls_rule *cr;
-
- ovs_rwlock_rdlock(&ofproto->facets.rwlock);
- cr = classifier_lookup(&ofproto->facets, flow, NULL);
- ovs_rwlock_unlock(&ofproto->facets.rwlock);
- return cr ? CONTAINER_OF(cr, struct facet, cr) : NULL;
-}
-
-/* Searches 'ofproto''s table of facets for one capable that covers
- * 'flow'. Returns it if found, otherwise a null pointer.
- *
- * The returned facet is guaranteed to be valid. */
-static struct facet *
-facet_lookup_valid(struct ofproto_dpif *ofproto, const struct flow *flow)
-{
- struct facet *facet;
-
- facet = facet_find(ofproto, flow);
- if (facet
- && ofproto->backer->need_revalidate
- && !facet_revalidate(facet)) {
- return NULL;
- }
-
- return facet;
-}
-
-static bool
-facet_check_consistency(struct facet *facet)
-{
- static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 15);
-
- struct xlate_out xout;
- struct xlate_in xin;
- bool ok;
-
- /* Check the datapath actions for consistency. */
- xlate_in_init(&xin, facet->ofproto, &facet->flow, NULL, 0, NULL);
- xlate_actions(&xin, &xout);
-
- ok = ofpbuf_equal(&facet->xout.odp_actions, &xout.odp_actions)
- && facet->xout.slow == xout.slow;
- if (!ok && !VLOG_DROP_WARN(&rl)) {
- struct ds s = DS_EMPTY_INITIALIZER;
-
- flow_format(&s, &facet->flow);
- ds_put_cstr(&s, ": inconsistency in facet");
-
- if (!ofpbuf_equal(&facet->xout.odp_actions, &xout.odp_actions)) {
- ds_put_cstr(&s, " (actions were: ");
- format_odp_actions(&s, facet->xout.odp_actions.data,
- facet->xout.odp_actions.size);
- ds_put_cstr(&s, ") (correct actions: ");
- format_odp_actions(&s, xout.odp_actions.data,
- xout.odp_actions.size);
- ds_put_char(&s, ')');
- }
-
- if (facet->xout.slow != xout.slow) {
- ds_put_format(&s, " slow path incorrect. should be %d", xout.slow);
- }
-
- ds_destroy(&s);
- }
- xlate_out_uninit(&xout);
-
- return ok;
-}
-
-/* Re-searches the classifier for 'facet':
- *
- * - If the rule found is different from 'facet''s current rule, moves
- * 'facet' to the new rule and recompiles its actions.
- *
- * - If the rule found is the same as 'facet''s current rule, leaves 'facet'
- * where it is and recompiles its actions anyway.
- *
- * - If any of 'facet''s subfacets correspond to a new flow according to
- * xlate_receive(), 'facet' is removed.
- *
- * Returns true if 'facet' is still valid. False if 'facet' was removed. */
-static bool
-facet_revalidate(struct facet *facet)
-{
- struct ofproto_dpif *ofproto = facet->ofproto;
- struct rule_dpif *new_rule;
- struct subfacet *subfacet;
- struct flow_wildcards wc;
- struct xlate_out xout;
- struct xlate_in xin;
-
- COVERAGE_INC(facet_revalidate);
-
- /* Check that child subfacets still correspond to this facet. Tunnel
- * configuration changes could cause a subfacet's OpenFlow in_port to
- * change. */
- LIST_FOR_EACH (subfacet, list_node, &facet->subfacets) {
- struct ofproto_dpif *recv_ofproto;
- struct flow recv_flow;
- int error;
-
- error = xlate_receive(ofproto->backer, NULL, subfacet->key,
- subfacet->key_len, &recv_flow, NULL,
- &recv_ofproto, NULL);
- if (error
- || recv_ofproto != ofproto
- || facet != facet_find(ofproto, &recv_flow)) {
- facet_remove(facet);
- return false;
- }
- }
-
- flow_wildcards_init_catchall(&wc);
- rule_dpif_lookup(ofproto, &facet->flow, &wc, &new_rule);
-
- /* Calculate new datapath actions.
- *
- * We do not modify any 'facet' state yet, because we might need to, e.g.,
- * emit a NetFlow expiration and, if so, we need to have the old state
- * around to properly compose it. */
- xlate_in_init(&xin, ofproto, &facet->flow, new_rule, 0, NULL);
- xlate_actions(&xin, &xout);
- flow_wildcards_or(&xout.wc, &xout.wc, &wc);
- /* Make sure non -packet fields are not masked. If not cleared,
- * the memcmp() below may fail, causing an otherwise valid facet
- * to be removed. */
- flow_wildcards_clear_non_packet_fields(&xout.wc);
-
- /* A facet's slow path reason should only change under dramatic
- * circumstances. Rather than try to update everything, it's simpler to
- * remove the facet and start over.
- *
- * More importantly, if a facet's wildcards change, it will be relatively
- * difficult to figure out if its subfacets still belong to it, and if not
- * which facet they may belong to. Again, to avoid the complexity, we
- * simply give up instead. */
- if (facet->xout.slow != xout.slow
- || memcmp(&facet->xout.wc, &xout.wc, sizeof xout.wc)) {
- facet_remove(facet);
- xlate_out_uninit(&xout);
- rule_dpif_unref(new_rule);
- return false;
- }
-
- if (!ofpbuf_equal(&facet->xout.odp_actions, &xout.odp_actions)) {
- LIST_FOR_EACH(subfacet, list_node, &facet->subfacets) {
- if (subfacet->path == SF_FAST_PATH) {
- struct dpif_flow_stats stats;
-
- subfacet_install(subfacet, &xout.odp_actions, &stats);
- subfacet_update_stats(subfacet, &stats);
- }
- }
-
- facet_flush_stats(facet);
-
- ofpbuf_clear(&facet->xout.odp_actions);
- ofpbuf_put(&facet->xout.odp_actions, xout.odp_actions.data,
- xout.odp_actions.size);
- }
-
- /* Update 'facet' now that we've taken care of all the old state. */
- facet->xout.slow = xout.slow;
- facet->xout.has_learn = xout.has_learn;
- facet->xout.has_normal = xout.has_normal;
- facet->xout.has_fin_timeout = xout.has_fin_timeout;
- facet->xout.nf_output_iface = xout.nf_output_iface;
- facet->xout.mirrors = xout.mirrors;
-
- ovs_mutex_lock(&new_rule->up.mutex);
- facet->used = MAX(facet->used, new_rule->up.created);
- ovs_mutex_unlock(&new_rule->up.mutex);
-
- xlate_out_uninit(&xout);
- rule_dpif_unref(new_rule);
- return true;
-}
-
-static void
-facet_reset_counters(struct facet *facet)
-{
- facet->packet_count = 0;
- facet->byte_count = 0;
- facet->prev_packet_count = 0;
- facet->prev_byte_count = 0;
- facet->accounted_bytes = 0;
-}
-
-static void
-flow_push_stats(struct ofproto_dpif *ofproto, struct flow *flow,
- struct dpif_flow_stats *stats, bool may_learn)
-{
- struct ofport_dpif *in_port;
- struct xlate_in xin;
-
- in_port = get_ofp_port(ofproto, flow->in_port.ofp_port);
- if (in_port && in_port->is_tunnel) {
- netdev_vport_inc_rx(in_port->up.netdev, stats);
- if (in_port->bfd) {
- bfd_account_rx(in_port->bfd, stats);
- }
- }
-
- xlate_in_init(&xin, ofproto, flow, NULL, stats->tcp_flags, NULL);
- xin.resubmit_stats = stats;
- xin.may_learn = may_learn;
- xlate_actions_for_side_effects(&xin);
-}
-
-static void
-facet_push_stats(struct facet *facet, bool may_learn)
-{
- struct dpif_flow_stats stats;
-
- ovs_assert(facet->packet_count >= facet->prev_packet_count);
- ovs_assert(facet->byte_count >= facet->prev_byte_count);
- ovs_assert(facet->used >= facet->prev_used);
-
- stats.n_packets = facet->packet_count - facet->prev_packet_count;
- stats.n_bytes = facet->byte_count - facet->prev_byte_count;
- stats.used = facet->used;
- stats.tcp_flags = facet->tcp_flags;
-
- if (may_learn || stats.n_packets || facet->used > facet->prev_used) {
- facet->prev_packet_count = facet->packet_count;
- facet->prev_byte_count = facet->byte_count;
- facet->prev_used = facet->used;
-
- if (facet->ofproto->netflow && !facet_is_controller_flow(facet)) {
- netflow_flow_update(facet->ofproto->netflow, &facet->flow,
- facet->xout.nf_output_iface, &stats);
- }
- mirror_update_stats(facet->ofproto->mbridge, facet->xout.mirrors,
- stats.n_packets, stats.n_bytes);
- flow_push_stats(facet->ofproto, &facet->flow, &stats, may_learn);
- }
-}
-
-static void
-push_all_stats__(bool run_fast)
-{
- static long long int rl = LLONG_MIN;
- struct ofproto_dpif *ofproto;
-
- if (time_msec() < rl) {
- return;
- }
-
- HMAP_FOR_EACH (ofproto, all_ofproto_dpifs_node, &all_ofproto_dpifs) {
- struct cls_cursor cursor;
- struct facet *facet;
-
- ovs_rwlock_rdlock(&ofproto->facets.rwlock);
- cls_cursor_init(&cursor, &ofproto->facets, NULL);
- CLS_CURSOR_FOR_EACH (facet, cr, &cursor) {
- facet_push_stats(facet, false);
- if (run_fast) {
- run_fast_rl();
- }
- }
- ovs_rwlock_unlock(&ofproto->facets.rwlock);
- }
-
- rl = time_msec() + 100;
-}
-
-static void
-push_all_stats(void)
-{
- push_all_stats__(true);
-}
-
-void
-rule_dpif_credit_stats(struct rule_dpif *rule,
- const struct dpif_flow_stats *stats)
-{
- ovs_mutex_lock(&rule->stats_mutex);
- rule->packet_count += stats->n_packets;
- rule->byte_count += stats->n_bytes;
- rule->up.used = MAX(rule->up.used, stats->used);
- ovs_mutex_unlock(&rule->stats_mutex);
-}
-
-bool
-rule_dpif_is_fail_open(const struct rule_dpif *rule)
-{
- return is_fail_open_rule(&rule->up);
-}
-
-bool
-rule_dpif_is_table_miss(const struct rule_dpif *rule)
-{
- return rule_is_table_miss(&rule->up);
-}
-
-ovs_be64
-rule_dpif_get_flow_cookie(const struct rule_dpif *rule)
- OVS_REQUIRES(rule->up.mutex)
-{
- return rule->up.flow_cookie;
-}
-
-void
-rule_dpif_reduce_timeouts(struct rule_dpif *rule, uint16_t idle_timeout,
- uint16_t hard_timeout)
-{
- ofproto_rule_reduce_timeouts(&rule->up, idle_timeout, hard_timeout);
-}
-
-/* Returns 'rule''s actions. The caller owns a reference on the returned
- * actions and must eventually release it (with rule_actions_unref()) to avoid
- * a memory leak. */
-struct rule_actions *
-rule_dpif_get_actions(const struct rule_dpif *rule)
-{
- return rule_get_actions(&rule->up);
-}
-\f
-/* Subfacets. */
-
-static struct subfacet *
-subfacet_find(struct dpif_backer *backer, const struct nlattr *key,
- size_t key_len, uint32_t key_hash)
-{
- struct subfacet *subfacet;
-
- HMAP_FOR_EACH_WITH_HASH (subfacet, hmap_node, key_hash,
- &backer->subfacets) {
- if (subfacet->key_len == key_len
- && !memcmp(key, subfacet->key, key_len)) {
- return subfacet;
- }
- }
-
- return NULL;
-}
-
-/* Creates and returns a new subfacet within 'facet' for the flow in 'miss'.
- * 'key_hash' must be a hash over miss->key. The caller must have already
- * ensured that no subfacet subfacet already exists. */
-static struct subfacet *
-subfacet_create(struct facet *facet, struct flow_miss *miss, uint32_t key_hash)
-{
- struct dpif_backer *backer = miss->ofproto->backer;
- const struct nlattr *key = miss->key;
- size_t key_len = miss->key_len;
- struct subfacet *subfacet;
-
- subfacet = (list_is_empty(&facet->subfacets)
- ? &facet->one_subfacet
- : xmalloc(sizeof *subfacet));
-
- COVERAGE_INC(subfacet_create);
- hmap_insert(&backer->subfacets, &subfacet->hmap_node, key_hash);
- list_push_back(&facet->subfacets, &subfacet->list_node);
- subfacet->facet = facet;
- subfacet->key = xmemdup(key, key_len);
- subfacet->key_len = key_len;
- subfacet->used = miss->stats.used;
- subfacet->created = subfacet->used;
- subfacet->dp_packet_count = 0;
- subfacet->dp_byte_count = 0;
- subfacet->path = SF_NOT_INSTALLED;
- subfacet->backer = backer;
-
- return subfacet;
-}
-
-/* Uninstalls 'subfacet' from the datapath, if it is installed, removes it from
- * its facet within 'ofproto', and frees it. */
-static void
-subfacet_destroy__(struct subfacet *subfacet)
-{
- struct facet *facet = subfacet->facet;
-
- COVERAGE_INC(subfacet_destroy);
- subfacet_uninstall(subfacet);
- hmap_remove(&subfacet->backer->subfacets, &subfacet->hmap_node);
- list_remove(&subfacet->list_node);
- free(subfacet->key);
- if (subfacet != &facet->one_subfacet) {
- free(subfacet);
- }
-}
-
-/* Destroys 'subfacet', as with subfacet_destroy__(), and then if this was the
- * last remaining subfacet in its facet destroys the facet too. */
-static void
-subfacet_destroy(struct subfacet *subfacet)
-{
- struct facet *facet = subfacet->facet;
-
- if (list_is_singleton(&facet->subfacets)) {
- /* facet_remove() needs at least one subfacet (it will remove it). */
- facet_remove(facet);
- } else {
- subfacet_destroy__(subfacet);
- }
-}
-
-static void
-subfacet_destroy_batch(struct dpif_backer *backer,
- struct subfacet **subfacets, int n)
-{
- struct dpif_op ops[SUBFACET_DESTROY_MAX_BATCH];
- struct dpif_op *opsp[SUBFACET_DESTROY_MAX_BATCH];
- struct dpif_flow_stats stats[SUBFACET_DESTROY_MAX_BATCH];
- int i;
-
- for (i = 0; i < n; i++) {
- ops[i].type = DPIF_OP_FLOW_DEL;
- ops[i].u.flow_del.key = subfacets[i]->key;
- ops[i].u.flow_del.key_len = subfacets[i]->key_len;
- ops[i].u.flow_del.stats = &stats[i];
- opsp[i] = &ops[i];
- }
-
- dpif_operate(backer->dpif, opsp, n);
- for (i = 0; i < n; i++) {
- subfacet_reset_dp_stats(subfacets[i], &stats[i]);
- subfacets[i]->path = SF_NOT_INSTALLED;
- subfacet_destroy(subfacets[i]);
- run_fast_rl();
- }
-}
-
-/* Updates 'subfacet''s datapath flow, setting its actions to 'actions_len'
- * bytes of actions in 'actions'. If 'stats' is non-null, statistics counters
- * in the datapath will be zeroed and 'stats' will be updated with traffic new
- * since 'subfacet' was last updated.
- *
- * Returns 0 if successful, otherwise a positive errno value. */
-static int
-subfacet_install(struct subfacet *subfacet, const struct ofpbuf *odp_actions,
- struct dpif_flow_stats *stats)
-{
- struct facet *facet = subfacet->facet;
- enum subfacet_path path = facet->xout.slow ? SF_SLOW_PATH : SF_FAST_PATH;
- const struct nlattr *actions = odp_actions->data;
- size_t actions_len = odp_actions->size;
- struct odputil_keybuf maskbuf;
- struct ofpbuf mask;
-
- uint64_t slow_path_stub[128 / 8];
- enum dpif_flow_put_flags flags;
- int ret;
-
- flags = subfacet->path == SF_NOT_INSTALLED ? DPIF_FP_CREATE
- : DPIF_FP_MODIFY;
- if (stats) {
- flags |= DPIF_FP_ZERO_STATS;
- }
-
- if (path == SF_SLOW_PATH) {
- compose_slow_path(facet->ofproto, &facet->flow, facet->xout.slow,
- slow_path_stub, sizeof slow_path_stub,
- &actions, &actions_len);
- }
-
- ofpbuf_use_stack(&mask, &maskbuf, sizeof maskbuf);
- if (enable_megaflows) {
- odp_flow_key_from_mask(&mask, &facet->xout.wc.masks,
- &facet->flow, UINT32_MAX);
- }
-
- ret = dpif_flow_put(subfacet->backer->dpif, flags, subfacet->key,
- subfacet->key_len, mask.data, mask.size,
- actions, actions_len, stats);
-
- if (stats) {
- subfacet_reset_dp_stats(subfacet, stats);
- }
-
- if (ret) {
- COVERAGE_INC(subfacet_install_fail);
- } else {
- subfacet->path = path;
- }
- return ret;
-}
-
-/* If 'subfacet' is installed in the datapath, uninstalls it. */
-static void
-subfacet_uninstall(struct subfacet *subfacet)
-{
- if (subfacet->path != SF_NOT_INSTALLED) {
- struct ofproto_dpif *ofproto = subfacet->facet->ofproto;
- struct dpif_flow_stats stats;
- int error;
-
- error = dpif_flow_del(ofproto->backer->dpif, subfacet->key,
- subfacet->key_len, &stats);
- subfacet_reset_dp_stats(subfacet, &stats);
- if (!error) {
- subfacet_update_stats(subfacet, &stats);
- }
- subfacet->path = SF_NOT_INSTALLED;
- } else {
- ovs_assert(subfacet->dp_packet_count == 0);
- ovs_assert(subfacet->dp_byte_count == 0);
- }
-}
-
-/* Resets 'subfacet''s datapath statistics counters. This should be called
- * when 'subfacet''s statistics are cleared in the datapath. If 'stats' is
- * non-null, it should contain the statistics returned by dpif when 'subfacet'
- * was reset in the datapath. 'stats' will be modified to include only
- * statistics new since 'subfacet' was last updated. */
-static void
-subfacet_reset_dp_stats(struct subfacet *subfacet,
- struct dpif_flow_stats *stats)
-{
- if (stats
- && subfacet->dp_packet_count <= stats->n_packets
- && subfacet->dp_byte_count <= stats->n_bytes) {
- stats->n_packets -= subfacet->dp_packet_count;
- stats->n_bytes -= subfacet->dp_byte_count;
- }
-
- subfacet->dp_packet_count = 0;
- subfacet->dp_byte_count = 0;
-}
-
-/* Folds the statistics from 'stats' into the counters in 'subfacet'.
- *
- * Because of the meaning of a subfacet's counters, it only makes sense to do
- * this if 'stats' are not tracked in the datapath, that is, if 'stats'
- * represents a packet that was sent by hand or if it represents statistics
- * that have been cleared out of the datapath. */
-static void
-subfacet_update_stats(struct subfacet *subfacet,
- const struct dpif_flow_stats *stats)
-{
- if (stats->n_packets || stats->used > subfacet->used) {
- struct facet *facet = subfacet->facet;
-
- subfacet->used = MAX(subfacet->used, stats->used);
- facet->used = MAX(facet->used, stats->used);
- facet->packet_count += stats->n_packets;
- facet->byte_count += stats->n_bytes;
- facet->tcp_flags |= stats->tcp_flags;
- }
-}
-\f
-/* Rules. */
-
-/* Lookup 'flow' in 'ofproto''s classifier. If 'wc' is non-null, sets
- * the fields that were relevant as part of the lookup. */
-void
-rule_dpif_lookup(struct ofproto_dpif *ofproto, const struct flow *flow,
- struct flow_wildcards *wc, struct rule_dpif **rule)
-{
- struct ofport_dpif *port;
-
- if (rule_dpif_lookup_in_table(ofproto, flow, wc, 0, rule)) {
- return;
- }
- port = get_ofp_port(ofproto, flow->in_port.ofp_port);
- if (!port) {
- VLOG_WARN_RL(&rl, "packet-in on unknown OpenFlow port %"PRIu16,
- flow->in_port.ofp_port);
- }
-
- choose_miss_rule(port ? port->up.pp.config : 0, ofproto->miss_rule,
- ofproto->no_packet_in_rule, rule);
-}
-
-bool
-rule_dpif_lookup_in_table(struct ofproto_dpif *ofproto,
- const struct flow *flow, struct flow_wildcards *wc,
- uint8_t table_id, struct rule_dpif **rule)
-{
- const struct cls_rule *cls_rule;
- struct classifier *cls;
- bool frag;
-
- *rule = NULL;
- if (table_id >= N_TABLES) {
- return false;