datapath: Per NUMA node flow stats.

[sliver-openvswitch.git] / datapath / flow_table.c

diff --git a/datapath/flow_table.c b/datapath/flow_table.c
index b7007ee..54a2e25 100644 (file)
--- a/datapath/flow_table.c
+++ b/datapath/flow_table.c
@@ -50,6 +50,7 @@
 #define REHASH_INTERVAL                (10 * 60 * HZ)
 
 static struct kmem_cache *flow_cache;
+struct kmem_cache *flow_stats_cache __read_mostly;
 
 static u16 range_n_bytes(const struct sw_flow_key_range *range)
 {
@@ -77,7 +78,8 @@ void ovs_flow_mask_key(struct sw_flow_key *dst, const struct sw_flow_key *src,
 struct sw_flow *ovs_flow_alloc(void)
 {
        struct sw_flow *flow;
-       int cpu;
+       struct flow_stats *stats;
+       int node;
 
        flow = kmem_cache_alloc(flow_cache, GFP_KERNEL);
        if (!flow)
@@ -85,17 +87,22 @@ struct sw_flow *ovs_flow_alloc(void)
 
        flow->sf_acts = NULL;
        flow->mask = NULL;
+       flow->stats_last_writer = NUMA_NO_NODE;
 
-       flow->stats = alloc_percpu(struct flow_stats);
-       if (!flow->stats)
+       /* Initialize the default stat node. */
+       stats = kmem_cache_alloc_node(flow_stats_cache,
+                                     GFP_KERNEL | __GFP_ZERO, 0);
+       if (!stats)
                goto err;
 
-       for_each_possible_cpu(cpu) {
-               struct flow_stats *cpu_stats;
+       spin_lock_init(&stats->lock);
+
+       RCU_INIT_POINTER(flow->stats[0], stats);
+
+       for_each_node(node)
+               if (node != 0)
+                       RCU_INIT_POINTER(flow->stats[node], NULL);
 
-               cpu_stats = per_cpu_ptr(flow->stats, cpu);
-               spin_lock_init(&cpu_stats->lock);
-       }
        return flow;
 err:
        kmem_cache_free(flow_cache, flow);
@@ -132,8 +139,13 @@ static struct flex_array *alloc_buckets(unsigned int n_buckets)
 
 static void flow_free(struct sw_flow *flow)
 {
+       int node;
+
        kfree((struct sf_flow_acts __force *)flow->sf_acts);
-       free_percpu(flow->stats);
+       for_each_node(node)
+               if (flow->stats[node])
+                       kmem_cache_free(flow_stats_cache,
+                                       (struct flow_stats __force *)flow->stats[node]);
        kmem_cache_free(flow_cache, flow);
 }
 
@@ -595,16 +607,28 @@ int ovs_flow_init(void)
        BUILD_BUG_ON(__alignof__(struct sw_flow_key) % __alignof__(long));
        BUILD_BUG_ON(sizeof(struct sw_flow_key) % sizeof(long));
 
-       flow_cache = kmem_cache_create("sw_flow", sizeof(struct sw_flow), 0,
-                                       0, NULL);
+       flow_cache = kmem_cache_create("sw_flow", sizeof(struct sw_flow)
+                                      + (num_possible_nodes()
+                                         * sizeof(struct flow_stats *)),
+                                      0, 0, NULL);
        if (flow_cache == NULL)
                return -ENOMEM;
 
+       flow_stats_cache
+               = kmem_cache_create("sw_flow_stats", sizeof(struct flow_stats),
+                                   0, SLAB_HWCACHE_ALIGN, NULL);
+       if (flow_stats_cache == NULL) {
+               kmem_cache_destroy(flow_cache);
+               flow_cache = NULL;
+               return -ENOMEM;
+       }
+
        return 0;
 }
 
 /* Uninitializes the flow module. */
 void ovs_flow_exit(void)
 {
+       kmem_cache_destroy(flow_stats_cache);
        kmem_cache_destroy(flow_cache);
 }