datapath: Don't query time for every packet.

Rather than actually query the time every time a packet comes through,
just store the current jiffies and convert it to actual time when
requested.  GRE is the primary beneficiary of this because the traffic
travels through the datapath twice.  This change reduces CPU utilization
3-4% with GRE.

diff --git a/datapath/datapath.c b/datapath/datapath.c
index eb260e3..54c9fa3 100644 (file)
--- a/datapath/datapath.c
+++ b/datapath/datapath.c
@@ -903,15 +903,32 @@ error:
        return ERR_PTR(error);
 }
 
-static void get_stats(struct sw_flow *flow, struct odp_flow_stats *stats)
+static struct timespec get_time_offset(void)
 {
-       if (flow->used.tv_sec) {
-               stats->used_sec = flow->used.tv_sec;
-               stats->used_nsec = flow->used.tv_nsec;
+       struct timespec now_mono, now_jiffies;
+
+       ktime_get_ts(&now_mono);
+       jiffies_to_timespec(jiffies, &now_jiffies);
+       return timespec_sub(now_mono, now_jiffies);
+}
+
+static void get_stats(struct sw_flow *flow, struct odp_flow_stats *stats,
+                     struct timespec time_offset)
+{
+       if (flow->used) {
+               struct timespec flow_ts, used;
+
+               jiffies_to_timespec(flow->used, &flow_ts);
+               set_normalized_timespec(&used, flow_ts.tv_sec + time_offset.tv_sec,
+                                       flow_ts.tv_nsec + time_offset.tv_nsec);
+
+               stats->used_sec = used.tv_sec;
+               stats->used_nsec = used.tv_nsec;
        } else {
                stats->used_sec = 0;
                stats->used_nsec = 0;
        }
+
        stats->n_packets = flow->packet_count;
        stats->n_bytes = flow->byte_count;
        stats->ip_tos = flow->ip_tos;
@@ -921,7 +938,7 @@ static void get_stats(struct sw_flow *flow, struct odp_flow_stats *stats)
 
 static void clear_stats(struct sw_flow *flow)
 {
-       flow->used.tv_sec = flow->used.tv_nsec = 0;
+       flow->used = 0;
        flow->tcp_flags = 0;
        flow->ip_tos = 0;
        flow->packet_count = 0;
@@ -1021,7 +1038,7 @@ static int do_put_flow(struct datapath *dp, struct odp_flow_put *uf,
 
                /* Fetch stats, then clear them if necessary. */
                spin_lock_bh(&flow->lock);
-               get_stats(flow, stats);
+               get_stats(flow, stats, get_time_offset());
                if (uf->flags & ODPPF_ZERO_STATS)
                        clear_stats(flow);
                spin_unlock_bh(&flow->lock);
@@ -1058,6 +1075,7 @@ static int put_flow(struct datapath *dp, struct odp_flow_put __user *ufp)
 }
 
 static int do_answer_query(struct sw_flow *flow, u32 query_flags,
+                          struct timespec time_offset,
                           struct odp_flow_stats __user *ustats,
                           union odp_action __user *actions,
                           u32 __user *n_actionsp)
@@ -1067,7 +1085,7 @@ static int do_answer_query(struct sw_flow *flow, u32 query_flags,
        u32 n_actions;
 
        spin_lock_bh(&flow->lock);
-       get_stats(flow, &stats);
+       get_stats(flow, &stats, time_offset);
        if (query_flags & ODPFF_ZERO_TCP_FLAGS)
                flow->tcp_flags = 0;
 
@@ -1091,6 +1109,7 @@ static int do_answer_query(struct sw_flow *flow, u32 query_flags,
 }
 
 static int answer_query(struct sw_flow *flow, u32 query_flags,
+                       struct timespec time_offset,
                        struct odp_flow __user *ufp)
 {
        union odp_action *actions;
@@ -1098,7 +1117,7 @@ static int answer_query(struct sw_flow *flow, u32 query_flags,
        if (get_user(actions, &ufp->actions))
                return -EFAULT;
 
-       return do_answer_query(flow, query_flags,
+       return do_answer_query(flow, query_flags, time_offset,
                               &ufp->stats, actions, &ufp->n_actions);
 }
 
@@ -1137,7 +1156,7 @@ static int del_flow(struct datapath *dp, struct odp_flow __user *ufp)
        if (IS_ERR(flow))
                return PTR_ERR(flow);
 
-       error = answer_query(flow, 0, ufp);
+       error = answer_query(flow, 0, get_time_offset(), ufp);
        flow_deferred_free(flow);
        return error;
 }
@@ -1145,8 +1164,11 @@ static int del_flow(struct datapath *dp, struct odp_flow __user *ufp)
 static int do_query_flows(struct datapath *dp, const struct odp_flowvec *flowvec)
 {
        struct tbl *table = rcu_dereference(dp->table);
+       struct timespec time_offset;
        u32 i;
 
+       time_offset = get_time_offset();
+
        for (i = 0; i < flowvec->n_flows; i++) {
                struct odp_flow __user *ufp = &flowvec->flows[i];
                struct odp_flow uf;
@@ -1161,7 +1183,7 @@ static int do_query_flows(struct datapath *dp, const struct odp_flowvec *flowvec
                if (!flow_node)
                        error = put_user(ENOENT, &ufp->stats.error);
                else
-                       error = answer_query(flow_cast(flow_node), uf.flags, ufp);
+                       error = answer_query(flow_cast(flow_node), uf.flags, time_offset, ufp);
                if (error)
                        return -EFAULT;
        }
@@ -1172,6 +1194,7 @@ struct list_flows_cbdata {
        struct odp_flow __user *uflows;
        u32 n_flows;
        u32 listed_flows;
+       struct timespec time_offset;
 };
 
 static int list_flow(struct tbl_node *node, void *cbdata_)
@@ -1183,7 +1206,7 @@ static int list_flow(struct tbl_node *node, void *cbdata_)
 
        if (copy_to_user(&ufp->key, &flow->key, sizeof flow->key))
                return -EFAULT;
-       error = answer_query(flow, 0, ufp);
+       error = answer_query(flow, 0, cbdata->time_offset, ufp);
        if (error)
                return error;
 
@@ -1203,6 +1226,8 @@ static int do_list_flows(struct datapath *dp, const struct odp_flowvec *flowvec)
        cbdata.uflows = flowvec->flows;
        cbdata.n_flows = flowvec->n_flows;
        cbdata.listed_flows = 0;
+       cbdata.time_offset = get_time_offset();
+
        error = tbl_foreach(rcu_dereference(dp->table), list_flow, &cbdata);
        return error ? error : cbdata.listed_flows;
 }
@@ -1811,6 +1836,7 @@ static int compat_put_flow(struct datapath *dp, struct compat_odp_flow_put __use
 }
 
 static int compat_answer_query(struct sw_flow *flow, u32 query_flags,
+                              struct timespec time_offset,
                               struct compat_odp_flow __user *ufp)
 {
        compat_uptr_t actions;
@@ -1818,7 +1844,7 @@ static int compat_answer_query(struct sw_flow *flow, u32 query_flags,
        if (get_user(actions, &ufp->actions))
                return -EFAULT;
 
-       return do_answer_query(flow, query_flags, &ufp->stats,
+       return do_answer_query(flow, query_flags, time_offset, &ufp->stats,
                               compat_ptr(actions), &ufp->n_actions);
 }
 
@@ -1835,7 +1861,7 @@ static int compat_del_flow(struct datapath *dp, struct compat_odp_flow __user *u
        if (IS_ERR(flow))
                return PTR_ERR(flow);
 
-       error = compat_answer_query(flow, 0, ufp);
+       error = compat_answer_query(flow, 0, get_time_offset(), ufp);
        flow_deferred_free(flow);
        return error;
 }
@@ -1843,8 +1869,11 @@ static int compat_del_flow(struct datapath *dp, struct compat_odp_flow __user *u
 static int compat_query_flows(struct datapath *dp, struct compat_odp_flow *flows, u32 n_flows)
 {
        struct tbl *table = rcu_dereference(dp->table);
+       struct timespec time_offset;
        u32 i;
 
+       time_offset = get_time_offset();
+
        for (i = 0; i < n_flows; i++) {
                struct compat_odp_flow __user *ufp = &flows[i];
                struct odp_flow uf;
@@ -1859,7 +1888,7 @@ static int compat_query_flows(struct datapath *dp, struct compat_odp_flow *flows
                if (!flow_node)
                        error = put_user(ENOENT, &ufp->stats.error);
                else
-                       error = compat_answer_query(flow_cast(flow_node), uf.flags, ufp);
+                       error = compat_answer_query(flow_cast(flow_node), uf.flags, time_offset, ufp);
                if (error)
                        return -EFAULT;
        }
@@ -1870,6 +1899,7 @@ struct compat_list_flows_cbdata {
        struct compat_odp_flow __user *uflows;
        u32 n_flows;
        u32 listed_flows;
+       struct timespec time_offset;
 };
 
 static int compat_list_flow(struct tbl_node *node, void *cbdata_)
@@ -1881,7 +1911,7 @@ static int compat_list_flow(struct tbl_node *node, void *cbdata_)
 
        if (copy_to_user(&ufp->key, &flow->key, sizeof flow->key))
                return -EFAULT;
-       error = compat_answer_query(flow, 0, ufp);
+       error = compat_answer_query(flow, 0, cbdata->time_offset, ufp);
        if (error)
                return error;
 
@@ -1901,6 +1931,8 @@ static int compat_list_flows(struct datapath *dp, struct compat_odp_flow *flows,
        cbdata.uflows = flows;
        cbdata.n_flows = n_flows;
        cbdata.listed_flows = 0;
+       cbdata.time_offset = get_time_offset();
+
        error = tbl_foreach(rcu_dereference(dp->table), compat_list_flow, &cbdata);
        return error ? error : cbdata.listed_flows;
 }

diff --git a/datapath/flow.c b/datapath/flow.c
index bf50e94..95cc042 100644 (file)
--- a/datapath/flow.c
+++ b/datapath/flow.c
@@ -111,7 +111,7 @@ void flow_used(struct sw_flow *flow, struct sk_buff *skb)
        }
 
        spin_lock_bh(&flow->lock);
-       ktime_get_ts(&flow->used);
+       flow->used = jiffies;
        flow->packet_count++;
        flow->byte_count += skb->len;
        flow->tcp_flags |= tcp_flags;

diff --git a/datapath/flow.h b/datapath/flow.h
index 9704489..29a1ac4 100644 (file)
--- a/datapath/flow.h
+++ b/datapath/flow.h
@@ -14,6 +14,7 @@
 #include <linux/types.h>
 #include <linux/rcupdate.h>
 #include <linux/gfp.h>
+#include <linux/jiffies.h>
 #include <linux/time.h>
 
 #include "openvswitch/datapath-protocol.h"
@@ -34,11 +35,10 @@ struct sw_flow {
        struct odp_flow_key key;
        struct sw_flow_actions *sf_acts;
 
-       struct timespec used;   /* Last used time. */
-
        u8 ip_tos;              /* IP TOS value. */
 
        spinlock_t lock;        /* Lock for values below. */
+       unsigned long used;     /* Last used time (in jiffies). */
        u64 packet_count;       /* Number of packets matched. */
        u64 byte_count;         /* Number of bytes matched. */
        u8 tcp_flags;           /* Union of seen TCP flags. */