datapath: Move table destroy to dp-rcu callback.

[sliver-openvswitch.git] / datapath / flow_table.c

diff --git a/datapath/flow_table.c b/datapath/flow_table.c
index b7007ee..be2d7d5 100644 (file)
--- a/datapath/flow_table.c
+++ b/datapath/flow_table.c
@@ -50,6 +50,7 @@
 #define REHASH_INTERVAL                (10 * 60 * HZ)
 
 static struct kmem_cache *flow_cache;
+struct kmem_cache *flow_stats_cache __read_mostly;
 
 static u16 range_n_bytes(const struct sw_flow_key_range *range)
 {
@@ -77,7 +78,8 @@ void ovs_flow_mask_key(struct sw_flow_key *dst, const struct sw_flow_key *src,
 struct sw_flow *ovs_flow_alloc(void)
 {
        struct sw_flow *flow;
-       int cpu;
+       struct flow_stats *stats;
+       int node;
 
        flow = kmem_cache_alloc(flow_cache, GFP_KERNEL);
        if (!flow)
@@ -85,17 +87,22 @@ struct sw_flow *ovs_flow_alloc(void)
 
        flow->sf_acts = NULL;
        flow->mask = NULL;
+       flow->stats_last_writer = NUMA_NO_NODE;
 
-       flow->stats = alloc_percpu(struct flow_stats);
-       if (!flow->stats)
+       /* Initialize the default stat node. */
+       stats = kmem_cache_alloc_node(flow_stats_cache,
+                                     GFP_KERNEL | __GFP_ZERO, 0);
+       if (!stats)
                goto err;
 
-       for_each_possible_cpu(cpu) {
-               struct flow_stats *cpu_stats;
+       spin_lock_init(&stats->lock);
+
+       RCU_INIT_POINTER(flow->stats[0], stats);
+
+       for_each_node(node)
+               if (node != 0)
+                       RCU_INIT_POINTER(flow->stats[node], NULL);
 
-               cpu_stats = per_cpu_ptr(flow->stats, cpu);
-               spin_lock_init(&cpu_stats->lock);
-       }
        return flow;
 err:
        kmem_cache_free(flow_cache, flow);
@@ -132,8 +139,13 @@ static struct flex_array *alloc_buckets(unsigned int n_buckets)
 
 static void flow_free(struct sw_flow *flow)
 {
-       kfree((struct sf_flow_acts __force *)flow->sf_acts);
-       free_percpu(flow->stats);
+       int node;
+
+       kfree((struct sw_flow_actions __force *)flow->sf_acts);
+       for_each_node(node)
+               if (flow->stats[node])
+                       kmem_cache_free(flow_stats_cache,
+                                       (struct flow_stats __force *)flow->stats[node]);
        kmem_cache_free(flow_cache, flow);
 }
 
@@ -156,25 +168,6 @@ void ovs_flow_free(struct sw_flow *flow, bool deferred)
        if (!flow)
                return;
 
-       if (flow->mask) {
-               struct sw_flow_mask *mask = flow->mask;
-
-               /* ovs-lock is required to protect mask-refcount and
-                * mask list.
-                */
-               ASSERT_OVSL();
-               BUG_ON(!mask->ref_count);
-               mask->ref_count--;
-
-               if (!mask->ref_count) {
-                       list_del_rcu(&mask->list);
-                       if (deferred)
-                               call_rcu(&mask->rcu, rcu_free_sw_flow_mask_cb);
-                       else
-                               kfree(mask);
-               }
-       }
-
        if (deferred)
                call_rcu(&flow->rcu, rcu_free_flow_callback);
        else
@@ -266,11 +259,13 @@ skip_flows:
                __table_instance_destroy(ti);
 }
 
-void ovs_flow_tbl_destroy(struct flow_table *table, bool deferred)
+/* No need for locking this function is called from RCU callback or
+ * error path. */
+void ovs_flow_tbl_destroy(struct flow_table *table)
 {
-       struct table_instance *ti = ovsl_dereference(table->ti);
+       struct table_instance *ti = (struct table_instance __force *)table->ti;
 
-       table_instance_destroy(ti, deferred);
+       table_instance_destroy(ti, false);
 }
 
 struct sw_flow *ovs_flow_tbl_dump_next(struct table_instance *ti,
@@ -488,6 +483,25 @@ static struct table_instance *table_instance_expand(struct table_instance *ti)
        return table_instance_rehash(ti, ti->n_buckets * 2);
 }
 
+/* Remove 'mask' from the mask list, if it is not needed any more. */
+static void flow_mask_remove(struct flow_table *tbl, struct sw_flow_mask *mask)
+{
+       if (mask) {
+               /* ovs-lock is required to protect mask-refcount and
+                * mask list.
+                */
+               ASSERT_OVSL();
+               BUG_ON(!mask->ref_count);
+               mask->ref_count--;
+
+               if (!mask->ref_count) {
+                       list_del_rcu(&mask->list);
+                       call_rcu(&mask->rcu, rcu_free_sw_flow_mask_cb);
+               }
+       }
+}
+
+/* Must be called with OVS mutex held. */
 void ovs_flow_tbl_remove(struct flow_table *table, struct sw_flow *flow)
 {
        struct table_instance *ti = ovsl_dereference(table->ti);
@@ -495,6 +509,10 @@ void ovs_flow_tbl_remove(struct flow_table *table, struct sw_flow *flow)
        BUG_ON(table->count == 0);
        hlist_del_rcu(&flow->hash_node[ti->node_ver]);
        table->count--;
+
+       /* RCU delete the mask. 'flow->mask' is not NULLed, as it should be
+        * accessible as long as the RCU read lock is held. */
+       flow_mask_remove(table, flow->mask);
 }
 
 static struct sw_flow_mask *mask_alloc(void)
@@ -557,6 +575,7 @@ static int flow_mask_insert(struct flow_table *tbl, struct sw_flow *flow,
        return 0;
 }
 
+/* Must be called with OVS mutex held. */
 int ovs_flow_tbl_insert(struct flow_table *table, struct sw_flow *flow,
                        struct sw_flow_mask *mask)
 {
@@ -595,16 +614,28 @@ int ovs_flow_init(void)
        BUILD_BUG_ON(__alignof__(struct sw_flow_key) % __alignof__(long));
        BUILD_BUG_ON(sizeof(struct sw_flow_key) % sizeof(long));
 
-       flow_cache = kmem_cache_create("sw_flow", sizeof(struct sw_flow), 0,
-                                       0, NULL);
+       flow_cache = kmem_cache_create("sw_flow", sizeof(struct sw_flow)
+                                      + (num_possible_nodes()
+                                         * sizeof(struct flow_stats *)),
+                                      0, 0, NULL);
        if (flow_cache == NULL)
                return -ENOMEM;
 
+       flow_stats_cache
+               = kmem_cache_create("sw_flow_stats", sizeof(struct flow_stats),
+                                   0, SLAB_HWCACHE_ALIGN, NULL);
+       if (flow_stats_cache == NULL) {
+               kmem_cache_destroy(flow_cache);
+               flow_cache = NULL;
+               return -ENOMEM;
+       }
+
        return 0;
 }
 
 /* Uninitializes the flow module. */
 void ovs_flow_exit(void)
 {
+       kmem_cache_destroy(flow_stats_cache);
        kmem_cache_destroy(flow_cache);
 }