classifier: Use OFP_DEFAULT_PRIORITY instead of literal 32768.

[sliver-openvswitch.git] / datapath / tunnel.c

diff --git a/datapath/tunnel.c b/datapath/tunnel.c
index c0e8b1e..bf66f4f 100644 (file)
--- a/datapath/tunnel.c
+++ b/datapath/tunnel.c
@@ -68,8 +68,7 @@
 
 #define CACHE_DATA_ALIGN 16
 
-/* Protected by RCU. */
-static struct tbl *port_table __read_mostly;
+static struct tbl __rcu *port_table __read_mostly;
 
 static void cache_cleaner(struct work_struct *work);
 static DECLARE_DELAYED_WORK(cache_cleaner_wq, cache_cleaner);
@@ -100,6 +99,15 @@ static inline struct tnl_vport *tnl_vport_table_cast(const struct tbl_node *node
        return container_of(node, struct tnl_vport, tbl_node);
 }
 
+/* This is analogous to rtnl_dereference for the tunnel cache.  It checks that
+ * cache_lock is held, so it is only for update side code.
+ */
+static inline struct tnl_cache *cache_dereference(struct tnl_vport *tnl_vport)
+{
+       return rcu_dereference_protected(tnl_vport->cache,
+                                        lockdep_is_held(&tnl_vport->cache_lock));
+}
+
 static inline void schedule_cache_cleaner(void)
 {
        schedule_delayed_work(&cache_cleaner_wq, CACHE_CLEANER_INTERVAL);
@@ -133,7 +141,7 @@ static void assign_config_rcu(struct vport *vport,
        struct tnl_vport *tnl_vport = tnl_vport_priv(vport);
        struct tnl_mutable_config *old_config;
 
-       old_config = tnl_vport->mutable;
+       old_config = rtnl_dereference(tnl_vport->mutable);
        rcu_assign_pointer(tnl_vport->mutable, new_config);
        call_rcu(&old_config->rcu, free_config_rcu);
 }
@@ -143,7 +151,7 @@ static void assign_cache_rcu(struct vport *vport, struct tnl_cache *new_cache)
        struct tnl_vport *tnl_vport = tnl_vport_priv(vport);
        struct tnl_cache *old_cache;
 
-       old_cache = tnl_vport->cache;
+       old_cache = cache_dereference(tnl_vport);
        rcu_assign_pointer(tnl_vport->cache, new_cache);
 
        if (old_cache)
@@ -166,11 +174,11 @@ static unsigned int *find_port_pool(const struct tnl_mutable_config *mutable)
 }
 
 struct port_lookup_key {
+       const struct tnl_mutable_config *mutable;
+       __be64 key;
        u32 tunnel_type;
        __be32 saddr;
        __be32 daddr;
-       __be32 key;
-       const struct tnl_mutable_config *mutable;
 };
 
 /*
@@ -182,7 +190,7 @@ static int port_cmp(const struct tbl_node *node, void *target)
        const struct tnl_vport *tnl_vport = tnl_vport_table_cast(node);
        struct port_lookup_key *lookup = target;
 
-       lookup->mutable = rcu_dereference(tnl_vport->mutable);
+       lookup->mutable = rcu_dereference_rtnl(tnl_vport->mutable);
 
        return (lookup->mutable->tunnel_type == lookup->tunnel_type &&
                lookup->mutable->port_config.daddr == lookup->daddr &&
@@ -192,7 +200,9 @@ static int port_cmp(const struct tbl_node *node, void *target)
 
 static u32 port_hash(struct port_lookup_key *k)
 {
-       return jhash_3words(k->key, k->saddr, k->daddr, k->tunnel_type);
+       u32 x = jhash_3words((__force u32)k->saddr, (__force u32)k->daddr,
+                            k->tunnel_type, 0);
+       return jhash_2words((__force u64)k->key >> 32, (__force u32)k->key, x);
 }
 
 static u32 mutable_hash(const struct tnl_mutable_config *mutable)
@@ -209,9 +219,9 @@ static u32 mutable_hash(const struct tnl_mutable_config *mutable)
 
 static void check_table_empty(void)
 {
-       if (tbl_count(port_table) == 0) {
-               struct tbl *old_table = port_table;
+       struct tbl *old_table = rtnl_dereference(port_table);
 
+       if (tbl_count(old_table) == 0) {
                cancel_delayed_work_sync(&cache_cleaner_wq);
                rcu_assign_pointer(port_table, NULL);
                tbl_deferred_destroy(old_table, NULL);
@@ -220,6 +230,7 @@ static void check_table_empty(void)
 
 static int add_port(struct vport *vport)
 {
+       struct tbl *cur_table = rtnl_dereference(port_table);
        struct tnl_vport *tnl_vport = tnl_vport_priv(vport);
        int err;
 
@@ -233,25 +244,25 @@ static int add_port(struct vport *vport)
                rcu_assign_pointer(port_table, new_table);
                schedule_cache_cleaner();
 
-       } else if (tbl_count(port_table) > tbl_n_buckets(port_table)) {
-               struct tbl *old_table = port_table;
+       } else if (tbl_count(cur_table) > tbl_n_buckets(cur_table)) {
                struct tbl *new_table;
 
-               new_table = tbl_expand(old_table);
+               new_table = tbl_expand(cur_table);
                if (IS_ERR(new_table))
                        return PTR_ERR(new_table);
 
                rcu_assign_pointer(port_table, new_table);
-               tbl_deferred_destroy(old_table, NULL);
+               tbl_deferred_destroy(cur_table, NULL);
        }
 
-       err = tbl_insert(port_table, &tnl_vport->tbl_node, mutable_hash(tnl_vport->mutable));
+       err = tbl_insert(rtnl_dereference(port_table), &tnl_vport->tbl_node,
+                        mutable_hash(rtnl_dereference(tnl_vport->mutable)));
        if (err) {
                check_table_empty();
                return err;
        }
 
-       (*find_port_pool(tnl_vport->mutable))++;
+       (*find_port_pool(rtnl_dereference(tnl_vport->mutable)))++;
 
        return 0;
 }
@@ -259,6 +270,7 @@ static int add_port(struct vport *vport)
 static int move_port(struct vport *vport, struct tnl_mutable_config *new_mutable)
 {
        int err;
+       struct tbl *cur_table = rtnl_dereference(port_table);
        struct tnl_vport *tnl_vport = tnl_vport_priv(vport);
        u32 hash;
 
@@ -271,21 +283,21 @@ static int move_port(struct vport *vport, struct tnl_mutable_config *new_mutable
         * finding tunnels or the possibility of failure.  However, if we do
         * find a tunnel it will always be consistent.
         */
-       err = tbl_remove(port_table, &tnl_vport->tbl_node);
+       err = tbl_remove(cur_table, &tnl_vport->tbl_node);
        if (err)
                return err;
 
-       err = tbl_insert(port_table, &tnl_vport->tbl_node, hash);
+       err = tbl_insert(cur_table, &tnl_vport->tbl_node, hash);
        if (err) {
-               (*find_port_pool(tnl_vport->mutable))--;
+               (*find_port_pool(rtnl_dereference(tnl_vport->mutable)))--;
                check_table_empty();
                return err;
        }
 
 table_updated:
-       (*find_port_pool(tnl_vport->mutable))--;
+       (*find_port_pool(rtnl_dereference(tnl_vport->mutable)))--;
        assign_config_rcu(vport, new_mutable);
-       (*find_port_pool(tnl_vport->mutable))++;
+       (*find_port_pool(rtnl_dereference(tnl_vport->mutable)))++;
 
        return 0;
 }
@@ -295,22 +307,22 @@ static int del_port(struct vport *vport)
        struct tnl_vport *tnl_vport = tnl_vport_priv(vport);
        int err;
 
-       err = tbl_remove(port_table, &tnl_vport->tbl_node);
+       err = tbl_remove(rtnl_dereference(port_table), &tnl_vport->tbl_node);
        if (err)
                return err;
 
        check_table_empty();
-       (*find_port_pool(tnl_vport->mutable))--;
+       (*find_port_pool(rtnl_dereference(tnl_vport->mutable)))--;
 
        return 0;
 }
 
-struct vport *tnl_find_port(__be32 saddr, __be32 daddr, __be32 key,
+struct vport *tnl_find_port(__be32 saddr, __be32 daddr, __be64 key,
                            int tunnel_type,
                            const struct tnl_mutable_config **mutable)
 {
        struct port_lookup_key lookup;
-       struct tbl *table = rcu_dereference(port_table);
+       struct tbl *table = rcu_dereference_rtnl(port_table);
        struct tbl_node *tbl_node;
 
        if (unlikely(!table))
@@ -598,7 +610,7 @@ static void ipv6_build_icmp(struct sk_buff *skb, struct sk_buff *nskb,
 #endif /* IPv6 */
 
 bool tnl_frag_needed(struct vport *vport, const struct tnl_mutable_config *mutable,
-                    struct sk_buff *skb, unsigned int mtu, __be32 flow_key)
+                    struct sk_buff *skb, unsigned int mtu, __be64 flow_key)
 {
        unsigned int eth_hdr_len = ETH_HLEN;
        unsigned int total_length = 0, header_length = 0, payload_length;
@@ -871,7 +883,7 @@ static struct tnl_cache *build_cache(struct vport *vport,
        if (!spin_trylock_bh(&tnl_vport->cache_lock))
                return NULL;
 
-       cache = tnl_vport->cache;
+       cache = cache_dereference(tnl_vport);
        if (check_cache_valid(cache, mutable))
                goto unlock;
        else
@@ -1044,7 +1056,7 @@ static struct sk_buff *handle_offloads(struct sk_buff *skb,
                 */
                if (skb_headroom(skb) < min_headroom) {
                        skb = check_headroom(skb, min_headroom);
-                       if (unlikely(IS_ERR(skb))) {
+                       if (IS_ERR(skb)) {
                                err = PTR_ERR(skb);
                                goto error;
                        }
@@ -1052,7 +1064,7 @@ static struct sk_buff *handle_offloads(struct sk_buff *skb,
 
                nskb = skb_gso_segment(skb, 0);
                kfree_skb(skb);
-               if (unlikely(IS_ERR(nskb))) {
+               if (IS_ERR(nskb)) {
                        err = PTR_ERR(nskb);
                        goto error;
                }
@@ -1060,7 +1072,7 @@ static struct sk_buff *handle_offloads(struct sk_buff *skb,
                skb = nskb;
        } else {
                skb = check_headroom(skb, min_headroom);
-               if (unlikely(IS_ERR(skb))) {
+               if (IS_ERR(skb)) {
                        err = PTR_ERR(skb);
                        goto error;
                }
@@ -1198,7 +1210,7 @@ int tnl_send(struct vport *vport, struct sk_buff *skb)
 
        /* Offloading */
        skb = handle_offloads(skb, mutable, rt);
-       if (unlikely(IS_ERR(skb)))
+       if (IS_ERR(skb))
                goto error;
 
        /* MTU */
@@ -1508,13 +1520,13 @@ const char *tnl_get_name(const struct vport *vport)
 const unsigned char *tnl_get_addr(const struct vport *vport)
 {
        const struct tnl_vport *tnl_vport = tnl_vport_priv(vport);
-       return rcu_dereference(tnl_vport->mutable)->eth_addr;
+       return rcu_dereference_rtnl(tnl_vport->mutable)->eth_addr;
 }
 
 int tnl_get_mtu(const struct vport *vport)
 {
        const struct tnl_vport *tnl_vport = tnl_vport_priv(vport);
-       return rcu_dereference(tnl_vport->mutable)->mtu;
+       return rcu_dereference_rtnl(tnl_vport->mutable)->mtu;
 }
 
 void tnl_free_linked_skbs(struct sk_buff *skb)