datapath/datapath.c

   1 /*
   2  * Copyright (c) 2007-2012 Nicira, Inc.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of version 2 of the GNU General Public
   6  * License as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  16  * 02110-1301, USA
  17  */
  18
  19 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  20
  21 #include <linux/init.h>
  22 #include <linux/module.h>
  23 #include <linux/if_arp.h>
  24 #include <linux/if_vlan.h>
  25 #include <linux/in.h>
  26 #include <linux/ip.h>
  27 #include <linux/jhash.h>
  28 #include <linux/delay.h>
  29 #include <linux/time.h>
  30 #include <linux/etherdevice.h>
  31 #include <linux/genetlink.h>
  32 #include <linux/kernel.h>
  33 #include <linux/kthread.h>
  34 #include <linux/mutex.h>
  35 #include <linux/percpu.h>
  36 #include <linux/rcupdate.h>
  37 #include <linux/tcp.h>
  38 #include <linux/udp.h>
  39 #include <linux/version.h>
  40 #include <linux/ethtool.h>
  41 #include <linux/wait.h>
  42 #include <asm/div64.h>
  43 #include <linux/highmem.h>
  44 #include <linux/netfilter_bridge.h>
  45 #include <linux/netfilter_ipv4.h>
  46 #include <linux/inetdevice.h>
  47 #include <linux/list.h>
  48 #include <linux/openvswitch.h>
  49 #include <linux/rculist.h>
  50 #include <linux/dmi.h>
  51 #include <net/genetlink.h>
  52 #include <net/net_namespace.h>
  53 #include <net/netns/generic.h>
  54
  55 #include "checksum.h"
  56 #include "datapath.h"
  57 #include "flow.h"
  58 #include "genl_exec.h"
  59 #include "vlan.h"
  60 #include "tunnel.h"
  61 #include "vport-internal_dev.h"
  62
  63 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) || \
  64     LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0)
  65 #error Kernels before 2.6.18 or after 3.8 are not supported by this version of Open vSwitch.
  66 #endif
  67
  68 #define REHASH_FLOW_INTERVAL (10 * 60 * HZ)
  69 static void rehash_flow_table(struct work_struct *work);
  70 static DECLARE_DELAYED_WORK(rehash_flow_wq, rehash_flow_table);
  71
  72 int ovs_net_id __read_mostly;
  73
  74 /**
  75  * DOC: Locking:
  76  *
  77  * Writes to device state (add/remove datapath, port, set operations on vports,
  78  * etc.) are protected by RTNL.
  79  *
  80  * Writes to other state (flow table modifications, set miscellaneous datapath
  81  * parameters, etc.) are protected by genl_mutex.  The RTNL lock nests inside
  82  * genl_mutex.
  83  *
  84  * Reads are protected by RCU.
  85  *
  86  * There are a few special cases (mostly stats) that have their own
  87  * synchronization but they nest under all of above and don't interact with
  88  * each other.
  89  */
  90
  91 static struct vport *new_vport(const struct vport_parms *);
  92 static int queue_gso_packets(struct net *, int dp_ifindex, struct sk_buff *,
  93                              const struct dp_upcall_info *);
  94 static int queue_userspace_packet(struct net *, int dp_ifindex,
  95                                   struct sk_buff *,
  96                                   const struct dp_upcall_info *);
  97
  98 /* Must be called with rcu_read_lock, genl_mutex, or RTNL lock. */
  99 static struct datapath *get_dp(struct net *net, int dp_ifindex)
 100 {
 101         struct datapath *dp = NULL;
 102         struct net_device *dev;
 103
 104         rcu_read_lock();
 105         dev = dev_get_by_index_rcu(net, dp_ifindex);
 106         if (dev) {
 107                 struct vport *vport = ovs_internal_dev_get_vport(dev);
 108                 if (vport)
 109                         dp = vport->dp;
 110         }
 111         rcu_read_unlock();
 112
 113         return dp;
 114 }
 115
 116 /* Must be called with rcu_read_lock or RTNL lock. */
 117 const char *ovs_dp_name(const struct datapath *dp)
 118 {
 119         struct vport *vport = ovs_vport_rtnl_rcu(dp, OVSP_LOCAL);
 120         return vport->ops->get_name(vport);
 121 }
 122
 123 static int get_dpifindex(struct datapath *dp)
 124 {
 125         struct vport *local;
 126         int ifindex;
 127
 128         rcu_read_lock();
 129
 130         local = ovs_vport_rcu(dp, OVSP_LOCAL);
 131         if (local)
 132                 ifindex = local->ops->get_ifindex(local);
 133         else
 134                 ifindex = 0;
 135
 136         rcu_read_unlock();
 137
 138         return ifindex;
 139 }
 140
 141 static void destroy_dp_rcu(struct rcu_head *rcu)
 142 {
 143         struct datapath *dp = container_of(rcu, struct datapath, rcu);
 144
 145         ovs_flow_tbl_destroy((__force struct flow_table *)dp->table);
 146         free_percpu(dp->stats_percpu);
 147         release_net(ovs_dp_get_net(dp));
 148         kfree(dp->ports);
 149         kfree(dp);
 150 }
 151
 152 static struct hlist_head *vport_hash_bucket(const struct datapath *dp,
 153                                             u16 port_no)
 154 {
 155         return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)];
 156 }
 157
 158 struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no)
 159 {
 160         struct vport *vport;
 161         struct hlist_head *head;
 162
 163         head = vport_hash_bucket(dp, port_no);
 164         hlist_for_each_entry_rcu(vport, head, dp_hash_node) {
 165                 if (vport->port_no == port_no)
 166                         return vport;
 167         }
 168         return NULL;
 169 }
 170
 171 /* Called with RTNL lock and genl_lock. */
 172 static struct vport *new_vport(const struct vport_parms *parms)
 173 {
 174         struct vport *vport;
 175
 176         vport = ovs_vport_add(parms);
 177         if (!IS_ERR(vport)) {
 178                 struct datapath *dp = parms->dp;
 179                 struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);
 180
 181                 hlist_add_head_rcu(&vport->dp_hash_node, head);
 182         }
 183         return vport;
 184 }
 185
 186 /* Called with RTNL lock. */
 187 void ovs_dp_detach_port(struct vport *p)
 188 {
 189         ASSERT_RTNL();
 190
 191         /* First drop references to device. */
 192         hlist_del_rcu(&p->dp_hash_node);
 193
 194         /* Then destroy it. */
 195         ovs_vport_del(p);
 196 }
 197
 198 /* Must be called with rcu_read_lock. */
 199 void ovs_dp_process_received_packet(struct vport *p, struct sk_buff *skb)
 200 {
 201         struct datapath *dp = p->dp;
 202         struct sw_flow *flow;
 203         struct dp_stats_percpu *stats;
 204         struct sw_flow_key key;
 205         u64 *stats_counter;
 206         int error;
 207         int key_len;
 208
 209         stats = this_cpu_ptr(dp->stats_percpu);
 210
 211         /* Extract flow from 'skb' into 'key'. */
 212         error = ovs_flow_extract(skb, p->port_no, &key, &key_len);
 213         if (unlikely(error)) {
 214                 kfree_skb(skb);
 215                 return;
 216         }
 217
 218         /* Look up flow. */
 219         flow = ovs_flow_tbl_lookup(rcu_dereference(dp->table), &key, key_len);
 220         if (unlikely(!flow)) {
 221                 struct dp_upcall_info upcall;
 222
 223                 upcall.cmd = OVS_PACKET_CMD_MISS;
 224                 upcall.key = &key;
 225                 upcall.userdata = NULL;
 226                 upcall.portid = p->upcall_portid;
 227                 ovs_dp_upcall(dp, skb, &upcall);
 228                 consume_skb(skb);
 229                 stats_counter = &stats->n_missed;
 230                 goto out;
 231         }
 232
 233         OVS_CB(skb)->flow = flow;
 234
 235         stats_counter = &stats->n_hit;
 236         ovs_flow_used(OVS_CB(skb)->flow, skb);
 237         ovs_execute_actions(dp, skb);
 238
 239 out:
 240         /* Update datapath statistics. */
 241         u64_stats_update_begin(&stats->sync);
 242         (*stats_counter)++;
 243         u64_stats_update_end(&stats->sync);
 244 }
 245
 246 static struct genl_family dp_packet_genl_family = {
 247         .id = GENL_ID_GENERATE,
 248         .hdrsize = sizeof(struct ovs_header),
 249         .name = OVS_PACKET_FAMILY,
 250         .version = OVS_PACKET_VERSION,
 251         .maxattr = OVS_PACKET_ATTR_MAX,
 252          SET_NETNSOK
 253 };
 254
 255 int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
 256                   const struct dp_upcall_info *upcall_info)
 257 {
 258         struct dp_stats_percpu *stats;
 259         int dp_ifindex;
 260         int err;
 261
 262         if (upcall_info->portid == 0) {
 263                 err = -ENOTCONN;
 264                 goto err;
 265         }
 266
 267         dp_ifindex = get_dpifindex(dp);
 268         if (!dp_ifindex) {
 269                 err = -ENODEV;
 270                 goto err;
 271         }
 272
 273         forward_ip_summed(skb, true);
 274
 275         if (!skb_is_gso(skb))
 276                 err = queue_userspace_packet(ovs_dp_get_net(dp), dp_ifindex, skb, upcall_info);
 277         else
 278                 err = queue_gso_packets(ovs_dp_get_net(dp), dp_ifindex, skb, upcall_info);
 279         if (err)
 280                 goto err;
 281
 282         return 0;
 283
 284 err:
 285         stats = this_cpu_ptr(dp->stats_percpu);
 286
 287         u64_stats_update_begin(&stats->sync);
 288         stats->n_lost++;
 289         u64_stats_update_end(&stats->sync);
 290
 291         return err;
 292 }
 293
 294 static int queue_gso_packets(struct net *net, int dp_ifindex,
 295                              struct sk_buff *skb,
 296                              const struct dp_upcall_info *upcall_info)
 297 {
 298         unsigned short gso_type = skb_shinfo(skb)->gso_type;
 299         struct dp_upcall_info later_info;
 300         struct sw_flow_key later_key;
 301         struct sk_buff *segs, *nskb;
 302         int err;
 303
 304         segs = __skb_gso_segment(skb, NETIF_F_SG | NETIF_F_HW_CSUM, false);
 305         if (IS_ERR(segs))
 306                 return PTR_ERR(segs);
 307
 308         /* Queue all of the segments. */
 309         skb = segs;
 310         do {
 311                 err = queue_userspace_packet(net, dp_ifindex, skb, upcall_info);
 312                 if (err)
 313                         break;
 314
 315                 if (skb == segs && gso_type & SKB_GSO_UDP) {
 316                         /* The initial flow key extracted by ovs_flow_extract()
 317                          * in this case is for a first fragment, so we need to
 318                          * properly mark later fragments.
 319                          */
 320                         later_key = *upcall_info->key;
 321                         later_key.ip.frag = OVS_FRAG_TYPE_LATER;
 322
 323                         later_info = *upcall_info;
 324                         later_info.key = &later_key;
 325                         upcall_info = &later_info;
 326                 }
 327         } while ((skb = skb->next));
 328
 329         /* Free all of the segments. */
 330         skb = segs;
 331         do {
 332                 nskb = skb->next;
 333                 if (err)
 334                         kfree_skb(skb);
 335                 else
 336                         consume_skb(skb);
 337         } while ((skb = nskb));
 338         return err;
 339 }
 340
 341 static size_t key_attr_size(void)
 342 {
 343         return    nla_total_size(4)   /* OVS_KEY_ATTR_PRIORITY */
 344                 + nla_total_size(0)   /* OVS_KEY_ATTR_TUNNEL */
 345                   + nla_total_size(8)   /* OVS_TUNNEL_KEY_ATTR_ID */
 346                   + nla_total_size(4)   /* OVS_TUNNEL_KEY_ATTR_IPV4_SRC */
 347                   + nla_total_size(4)   /* OVS_TUNNEL_KEY_ATTR_IPV4_DST */
 348                   + nla_total_size(1)   /* OVS_TUNNEL_KEY_ATTR_TOS */
 349                   + nla_total_size(1)   /* OVS_TUNNEL_KEY_ATTR_TTL */
 350                   + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT */
 351                   + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_CSUM */
 352                 + nla_total_size(4)   /* OVS_KEY_ATTR_IN_PORT */
 353                 + nla_total_size(4)   /* OVS_KEY_ATTR_SKB_MARK */
 354                 + nla_total_size(12)  /* OVS_KEY_ATTR_ETHERNET */
 355                 + nla_total_size(2)   /* OVS_KEY_ATTR_ETHERTYPE */
 356                 + nla_total_size(4)   /* OVS_KEY_ATTR_8021Q */
 357                 + nla_total_size(0)   /* OVS_KEY_ATTR_ENCAP */
 358                 + nla_total_size(2)   /* OVS_KEY_ATTR_ETHERTYPE */
 359                 + nla_total_size(40)  /* OVS_KEY_ATTR_IPV6 */
 360                 + nla_total_size(2)   /* OVS_KEY_ATTR_ICMPV6 */
 361                 + nla_total_size(28); /* OVS_KEY_ATTR_ND */
 362 }
 363
 364 static size_t upcall_msg_size(const struct sk_buff *skb,
 365                               const struct nlattr *userdata)
 366 {
 367         size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
 368                 + nla_total_size(skb->len) /* OVS_PACKET_ATTR_PACKET */
 369                 + nla_total_size(key_attr_size()); /* OVS_PACKET_ATTR_KEY */
 370
 371         /* OVS_PACKET_ATTR_USERDATA */
 372         if (userdata)
 373                 size += NLA_ALIGN(userdata->nla_len);
 374
 375         return size;
 376 }
 377
 378 static int queue_userspace_packet(struct net *net, int dp_ifindex,
 379                                   struct sk_buff *skb,
 380                                   const struct dp_upcall_info *upcall_info)
 381 {
 382         struct ovs_header *upcall;
 383         struct sk_buff *nskb = NULL;
 384         struct sk_buff *user_skb; /* to be queued to userspace */
 385         struct nlattr *nla;
 386         int err;
 387
 388         if (vlan_tx_tag_present(skb)) {
 389                 nskb = skb_clone(skb, GFP_ATOMIC);
 390                 if (!nskb)
 391                         return -ENOMEM;
 392
 393                 err = vlan_deaccel_tag(nskb);
 394                 if (err)
 395                         return err;
 396
 397                 skb = nskb;
 398         }
 399
 400         if (nla_attr_size(skb->len) > USHRT_MAX) {
 401                 err = -EFBIG;
 402                 goto out;
 403         }
 404
 405         user_skb = genlmsg_new(upcall_msg_size(skb, upcall_info->userdata), GFP_ATOMIC);
 406         if (!user_skb) {
 407                 err = -ENOMEM;
 408                 goto out;
 409         }
 410
 411         upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family,
 412                              0, upcall_info->cmd);
 413         upcall->dp_ifindex = dp_ifindex;
 414
 415         nla = nla_nest_start(user_skb, OVS_PACKET_ATTR_KEY);
 416         ovs_flow_to_nlattrs(upcall_info->key, user_skb);
 417         nla_nest_end(user_skb, nla);
 418
 419         if (upcall_info->userdata)
 420                 __nla_put(user_skb, OVS_PACKET_ATTR_USERDATA,
 421                           nla_len(upcall_info->userdata),
 422                           nla_data(upcall_info->userdata));
 423
 424         nla = __nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, skb->len);
 425
 426         skb_copy_and_csum_dev(skb, nla_data(nla));
 427
 428         genlmsg_end(user_skb, upcall);
 429         err = genlmsg_unicast(net, user_skb, upcall_info->portid);
 430
 431 out:
 432         kfree_skb(nskb);
 433         return err;
 434 }
 435
 436 /* Called with genl_mutex. */
 437 static int flush_flows(struct datapath *dp)
 438 {
 439         struct flow_table *old_table;
 440         struct flow_table *new_table;
 441
 442         old_table = genl_dereference(dp->table);
 443         new_table = ovs_flow_tbl_alloc(TBL_MIN_BUCKETS);
 444         if (!new_table)
 445                 return -ENOMEM;
 446
 447         rcu_assign_pointer(dp->table, new_table);
 448
 449         ovs_flow_tbl_deferred_destroy(old_table);
 450         return 0;
 451 }
 452
 453 static struct nlattr *reserve_sfa_size(struct sw_flow_actions **sfa, int attr_len)
 454 {
 455
 456         struct sw_flow_actions *acts;
 457         int new_acts_size;
 458         int req_size = NLA_ALIGN(attr_len);
 459         int next_offset = offsetof(struct sw_flow_actions, actions) +
 460                                         (*sfa)->actions_len;
 461
 462         if (req_size <= (ksize(*sfa) - next_offset))
 463                 goto out;
 464
 465         new_acts_size = ksize(*sfa) * 2;
 466
 467         if (new_acts_size > MAX_ACTIONS_BUFSIZE) {
 468                 if ((MAX_ACTIONS_BUFSIZE - next_offset) < req_size)
 469                         return ERR_PTR(-EMSGSIZE);
 470                 new_acts_size = MAX_ACTIONS_BUFSIZE;
 471         }
 472
 473         acts = ovs_flow_actions_alloc(new_acts_size);
 474         if (IS_ERR(acts))
 475                 return (void *)acts;
 476
 477         memcpy(acts->actions, (*sfa)->actions, (*sfa)->actions_len);
 478         acts->actions_len = (*sfa)->actions_len;
 479         kfree(*sfa);
 480         *sfa = acts;
 481
 482 out:
 483         (*sfa)->actions_len += req_size;
 484         return  (struct nlattr *) ((unsigned char *)(*sfa) + next_offset);
 485 }
 486
 487 static int add_action(struct sw_flow_actions **sfa, int attrtype, void *data, int len)
 488 {
 489         struct nlattr *a;
 490
 491         a = reserve_sfa_size(sfa, nla_attr_size(len));
 492         if (IS_ERR(a))
 493                 return PTR_ERR(a);
 494
 495         a->nla_type = attrtype;
 496         a->nla_len = nla_attr_size(len);
 497
 498         if (data)
 499                 memcpy(nla_data(a), data, len);
 500         memset((unsigned char *) a + a->nla_len, 0, nla_padlen(len));
 501
 502         return 0;
 503 }
 504
 505 static inline int add_nested_action_start(struct sw_flow_actions **sfa, int attrtype)
 506 {
 507         int used = (*sfa)->actions_len;
 508         int err;
 509
 510         err = add_action(sfa, attrtype, NULL, 0);
 511         if (err)
 512                 return err;
 513
 514         return used;
 515 }
 516
 517 static inline void add_nested_action_end(struct sw_flow_actions *sfa, int st_offset)
 518 {
 519         struct nlattr *a = (struct nlattr *) ((unsigned char *)sfa->actions + st_offset);
 520
 521         a->nla_len = sfa->actions_len - st_offset;
 522 }
 523
 524 static int validate_and_copy_actions(const struct nlattr *attr,
 525                                 const struct sw_flow_key *key, int depth,
 526                                 struct sw_flow_actions **sfa);
 527
 528 static int validate_and_copy_sample(const struct nlattr *attr,
 529                            const struct sw_flow_key *key, int depth,
 530                            struct sw_flow_actions **sfa)
 531 {
 532         const struct nlattr *attrs[OVS_SAMPLE_ATTR_MAX + 1];
 533         const struct nlattr *probability, *actions;
 534         const struct nlattr *a;
 535         int rem, start, err, st_acts;
 536
 537         memset(attrs, 0, sizeof(attrs));
 538         nla_for_each_nested(a, attr, rem) {
 539                 int type = nla_type(a);
 540                 if (!type || type > OVS_SAMPLE_ATTR_MAX || attrs[type])
 541                         return -EINVAL;
 542                 attrs[type] = a;
 543         }
 544         if (rem)
 545                 return -EINVAL;
 546
 547         probability = attrs[OVS_SAMPLE_ATTR_PROBABILITY];
 548         if (!probability || nla_len(probability) != sizeof(u32))
 549                 return -EINVAL;
 550
 551         actions = attrs[OVS_SAMPLE_ATTR_ACTIONS];
 552         if (!actions || (nla_len(actions) && nla_len(actions) < NLA_HDRLEN))
 553                 return -EINVAL;
 554
 555         /* validation done, copy sample action. */
 556         start = add_nested_action_start(sfa, OVS_ACTION_ATTR_SAMPLE);
 557         if (start < 0)
 558                 return start;
 559         err = add_action(sfa, OVS_SAMPLE_ATTR_PROBABILITY, nla_data(probability), sizeof(u32));
 560         if (err)
 561                 return err;
 562         st_acts = add_nested_action_start(sfa, OVS_SAMPLE_ATTR_ACTIONS);
 563         if (st_acts < 0)
 564                 return st_acts;
 565
 566         err = validate_and_copy_actions(actions, key, depth + 1, sfa);
 567         if (err)
 568                 return err;
 569
 570         add_nested_action_end(*sfa, st_acts);
 571         add_nested_action_end(*sfa, start);
 572
 573         return 0;
 574 }
 575
 576 static int validate_tp_port(const struct sw_flow_key *flow_key)
 577 {
 578         if (flow_key->eth.type == htons(ETH_P_IP)) {
 579                 if (flow_key->ipv4.tp.src || flow_key->ipv4.tp.dst)
 580                         return 0;
 581         } else if (flow_key->eth.type == htons(ETH_P_IPV6)) {
 582                 if (flow_key->ipv6.tp.src || flow_key->ipv6.tp.dst)
 583                         return 0;
 584         }
 585
 586         return -EINVAL;
 587 }
 588
 589 static int validate_and_copy_set_tun(const struct nlattr *attr,
 590                                      struct sw_flow_actions **sfa)
 591 {
 592         struct ovs_key_ipv4_tunnel tun_key;
 593         int err, start;
 594
 595         err = ipv4_tun_from_nlattr(nla_data(attr), &tun_key);
 596         if (err)
 597                 return err;
 598
 599         start = add_nested_action_start(sfa, OVS_ACTION_ATTR_SET);
 600         if (start < 0)
 601                 return start;
 602
 603         err = add_action(sfa, OVS_KEY_ATTR_IPV4_TUNNEL, &tun_key, sizeof(tun_key));
 604         add_nested_action_end(*sfa, start);
 605
 606         return err;
 607 }
 608
 609 static int validate_set(const struct nlattr *a,
 610                         const struct sw_flow_key *flow_key,
 611                         struct sw_flow_actions **sfa,
 612                         bool *set_tun)
 613 {
 614         const struct nlattr *ovs_key = nla_data(a);
 615         int key_type = nla_type(ovs_key);
 616
 617         /* There can be only one key in a action */
 618         if (nla_total_size(nla_len(ovs_key)) != nla_len(a))
 619                 return -EINVAL;
 620
 621         if (key_type > OVS_KEY_ATTR_MAX ||
 622             (ovs_key_lens[key_type] != nla_len(ovs_key) &&
 623              ovs_key_lens[key_type] != -1))
 624                 return -EINVAL;
 625
 626         switch (key_type) {
 627         const struct ovs_key_ipv4 *ipv4_key;
 628         const struct ovs_key_ipv6 *ipv6_key;
 629         int err;
 630
 631         case OVS_KEY_ATTR_PRIORITY:
 632         case OVS_KEY_ATTR_ETHERNET:
 633                 break;
 634
 635         case OVS_KEY_ATTR_SKB_MARK:
 636 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) && !defined(CONFIG_NETFILTER)
 637                 if (nla_get_u32(ovs_key) != 0)
 638                         return -EINVAL;
 639 #endif
 640                 break;
 641
 642         case OVS_KEY_ATTR_TUNNEL:
 643                 *set_tun = true;
 644                 err = validate_and_copy_set_tun(a, sfa);
 645                 if (err)
 646                         return err;
 647                 break;
 648
 649         case OVS_KEY_ATTR_IPV4:
 650                 if (flow_key->eth.type != htons(ETH_P_IP))
 651                         return -EINVAL;
 652
 653                 if (!flow_key->ip.proto)
 654                         return -EINVAL;
 655
 656                 ipv4_key = nla_data(ovs_key);
 657                 if (ipv4_key->ipv4_proto != flow_key->ip.proto)
 658                         return -EINVAL;
 659
 660                 if (ipv4_key->ipv4_frag != flow_key->ip.frag)
 661                         return -EINVAL;
 662
 663                 break;
 664
 665         case OVS_KEY_ATTR_IPV6:
 666                 if (flow_key->eth.type != htons(ETH_P_IPV6))
 667                         return -EINVAL;
 668
 669                 if (!flow_key->ip.proto)
 670                         return -EINVAL;
 671
 672                 ipv6_key = nla_data(ovs_key);
 673                 if (ipv6_key->ipv6_proto != flow_key->ip.proto)
 674                         return -EINVAL;
 675
 676                 if (ipv6_key->ipv6_frag != flow_key->ip.frag)
 677                         return -EINVAL;
 678
 679                 if (ntohl(ipv6_key->ipv6_label) & 0xFFF00000)
 680                         return -EINVAL;
 681
 682                 break;
 683
 684         case OVS_KEY_ATTR_TCP:
 685                 if (flow_key->ip.proto != IPPROTO_TCP)
 686                         return -EINVAL;
 687
 688                 return validate_tp_port(flow_key);
 689
 690         case OVS_KEY_ATTR_UDP:
 691                 if (flow_key->ip.proto != IPPROTO_UDP)
 692                         return -EINVAL;
 693
 694                 return validate_tp_port(flow_key);
 695
 696         default:
 697                 return -EINVAL;
 698         }
 699
 700         return 0;
 701 }
 702
 703 static int validate_userspace(const struct nlattr *attr)
 704 {
 705         static const struct nla_policy userspace_policy[OVS_USERSPACE_ATTR_MAX + 1] =   {
 706                 [OVS_USERSPACE_ATTR_PID] = {.type = NLA_U32 },
 707                 [OVS_USERSPACE_ATTR_USERDATA] = {.type = NLA_UNSPEC },
 708         };
 709         struct nlattr *a[OVS_USERSPACE_ATTR_MAX + 1];
 710         int error;
 711
 712         error = nla_parse_nested(a, OVS_USERSPACE_ATTR_MAX,
 713                                  attr, userspace_policy);
 714         if (error)
 715                 return error;
 716
 717         if (!a[OVS_USERSPACE_ATTR_PID] ||
 718             !nla_get_u32(a[OVS_USERSPACE_ATTR_PID]))
 719                 return -EINVAL;
 720
 721         return 0;
 722 }
 723
 724 static int copy_action(const struct nlattr *from,
 725                       struct sw_flow_actions **sfa)
 726 {
 727         int totlen = NLA_ALIGN(from->nla_len);
 728         struct nlattr *to;
 729
 730         to = reserve_sfa_size(sfa, from->nla_len);
 731         if (IS_ERR(to))
 732                 return PTR_ERR(to);
 733
 734         memcpy(to, from, totlen);
 735         return 0;
 736 }
 737
 738 static int validate_and_copy_actions(const struct nlattr *attr,
 739                                 const struct sw_flow_key *key,
 740                                 int depth,
 741                                 struct sw_flow_actions **sfa)
 742 {
 743         const struct nlattr *a;
 744         int rem, err;
 745
 746         if (depth >= SAMPLE_ACTION_DEPTH)
 747                 return -EOVERFLOW;
 748
 749         nla_for_each_nested(a, attr, rem) {
 750                 /* Expected argument lengths, (u32)-1 for variable length. */
 751                 static const u32 action_lens[OVS_ACTION_ATTR_MAX + 1] = {
 752                         [OVS_ACTION_ATTR_OUTPUT] = sizeof(u32),
 753                         [OVS_ACTION_ATTR_USERSPACE] = (u32)-1,
 754                         [OVS_ACTION_ATTR_PUSH_VLAN] = sizeof(struct ovs_action_push_vlan),
 755                         [OVS_ACTION_ATTR_POP_VLAN] = 0,
 756                         [OVS_ACTION_ATTR_SET] = (u32)-1,
 757                         [OVS_ACTION_ATTR_SAMPLE] = (u32)-1
 758                 };
 759                 const struct ovs_action_push_vlan *vlan;
 760                 int type = nla_type(a);
 761                 bool skip_copy;
 762
 763                 if (type > OVS_ACTION_ATTR_MAX ||
 764                     (action_lens[type] != nla_len(a) &&
 765                      action_lens[type] != (u32)-1))
 766                         return -EINVAL;
 767
 768                 skip_copy = false;
 769                 switch (type) {
 770                 case OVS_ACTION_ATTR_UNSPEC:
 771                         return -EINVAL;
 772
 773                 case OVS_ACTION_ATTR_USERSPACE:
 774                         err = validate_userspace(a);
 775                         if (err)
 776                                 return err;
 777                         break;
 778
 779                 case OVS_ACTION_ATTR_OUTPUT:
 780                         if (nla_get_u32(a) >= DP_MAX_PORTS)
 781                                 return -EINVAL;
 782                         break;
 783
 784
 785                 case OVS_ACTION_ATTR_POP_VLAN:
 786                         break;
 787
 788                 case OVS_ACTION_ATTR_PUSH_VLAN:
 789                         vlan = nla_data(a);
 790                         if (vlan->vlan_tpid != htons(ETH_P_8021Q))
 791                                 return -EINVAL;
 792                         if (!(vlan->vlan_tci & htons(VLAN_TAG_PRESENT)))
 793                                 return -EINVAL;
 794                         break;
 795
 796                 case OVS_ACTION_ATTR_SET:
 797                         err = validate_set(a, key, sfa, &skip_copy);
 798                         if (err)
 799                                 return err;
 800                         break;
 801
 802                 case OVS_ACTION_ATTR_SAMPLE:
 803                         err = validate_and_copy_sample(a, key, depth, sfa);
 804                         if (err)
 805                                 return err;
 806                         skip_copy = true;
 807                         break;
 808
 809                 default:
 810                         return -EINVAL;
 811                 }
 812                 if (!skip_copy) {
 813                         err = copy_action(a, sfa);
 814                         if (err)
 815                                 return err;
 816                 }
 817         }
 818
 819         if (rem > 0)
 820                 return -EINVAL;
 821
 822         return 0;
 823 }
 824
 825 static void clear_stats(struct sw_flow *flow)
 826 {
 827         flow->used = 0;
 828         flow->tcp_flags = 0;
 829         flow->packet_count = 0;
 830         flow->byte_count = 0;
 831 }
 832
 833 static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
 834 {
 835         struct ovs_header *ovs_header = info->userhdr;
 836         struct nlattr **a = info->attrs;
 837         struct sw_flow_actions *acts;
 838         struct sk_buff *packet;
 839         struct sw_flow *flow;
 840         struct datapath *dp;
 841         struct ethhdr *eth;
 842         int len;
 843         int err;
 844         int key_len;
 845
 846         err = -EINVAL;
 847         if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] ||
 848             !a[OVS_PACKET_ATTR_ACTIONS])
 849                 goto err;
 850
 851         len = nla_len(a[OVS_PACKET_ATTR_PACKET]);
 852         packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL);
 853         err = -ENOMEM;
 854         if (!packet)
 855                 goto err;
 856         skb_reserve(packet, NET_IP_ALIGN);
 857
 858         nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len);
 859
 860         skb_reset_mac_header(packet);
 861         eth = eth_hdr(packet);
 862
 863         /* Normally, setting the skb 'protocol' field would be handled by a
 864          * call to eth_type_trans(), but it assumes there's a sending
 865          * device, which we may not have. */
 866         if (ntohs(eth->h_proto) >= ETH_P_802_3_MIN)
 867                 packet->protocol = eth->h_proto;
 868         else
 869                 packet->protocol = htons(ETH_P_802_2);
 870
 871         /* Build an sw_flow for sending this packet. */
 872         flow = ovs_flow_alloc();
 873         err = PTR_ERR(flow);
 874         if (IS_ERR(flow))
 875                 goto err_kfree_skb;
 876
 877         err = ovs_flow_extract(packet, -1, &flow->key, &key_len);
 878         if (err)
 879                 goto err_flow_free;
 880
 881         err = ovs_flow_metadata_from_nlattrs(flow, key_len, a[OVS_PACKET_ATTR_KEY]);
 882         if (err)
 883                 goto err_flow_free;
 884         acts = ovs_flow_actions_alloc(nla_len(a[OVS_PACKET_ATTR_ACTIONS]));
 885         err = PTR_ERR(acts);
 886         if (IS_ERR(acts))
 887                 goto err_flow_free;
 888
 889         err = validate_and_copy_actions(a[OVS_PACKET_ATTR_ACTIONS], &flow->key, 0, &acts);
 890         rcu_assign_pointer(flow->sf_acts, acts);
 891         if (err)
 892                 goto err_flow_free;
 893
 894         OVS_CB(packet)->flow = flow;
 895         packet->priority = flow->key.phy.priority;
 896         skb_set_mark(packet, flow->key.phy.skb_mark);
 897
 898         rcu_read_lock();
 899         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
 900         err = -ENODEV;
 901         if (!dp)
 902                 goto err_unlock;
 903
 904         local_bh_disable();
 905         err = ovs_execute_actions(dp, packet);
 906         local_bh_enable();
 907         rcu_read_unlock();
 908
 909         ovs_flow_free(flow);
 910         return err;
 911
 912 err_unlock:
 913         rcu_read_unlock();
 914 err_flow_free:
 915         ovs_flow_free(flow);
 916 err_kfree_skb:
 917         kfree_skb(packet);
 918 err:
 919         return err;
 920 }
 921
 922 static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = {
 923 #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,18)
 924         [OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN },
 925 #else
 926         [OVS_PACKET_ATTR_PACKET] = { .minlen = ETH_HLEN },
 927 #endif
 928         [OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED },
 929         [OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED },
 930 };
 931
 932 static struct genl_ops dp_packet_genl_ops[] = {
 933         { .cmd = OVS_PACKET_CMD_EXECUTE,
 934           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
 935           .policy = packet_policy,
 936           .doit = ovs_packet_cmd_execute
 937         }
 938 };
 939
 940 static void get_dp_stats(struct datapath *dp, struct ovs_dp_stats *stats)
 941 {
 942         int i;
 943         struct flow_table *table = genl_dereference(dp->table);
 944
 945         stats->n_flows = ovs_flow_tbl_count(table);
 946
 947         stats->n_hit = stats->n_missed = stats->n_lost = 0;
 948         for_each_possible_cpu(i) {
 949                 const struct dp_stats_percpu *percpu_stats;
 950                 struct dp_stats_percpu local_stats;
 951                 unsigned int start;
 952
 953                 percpu_stats = per_cpu_ptr(dp->stats_percpu, i);
 954
 955                 do {
 956                         start = u64_stats_fetch_begin_bh(&percpu_stats->sync);
 957                         local_stats = *percpu_stats;
 958                 } while (u64_stats_fetch_retry_bh(&percpu_stats->sync, start));
 959
 960                 stats->n_hit += local_stats.n_hit;
 961                 stats->n_missed += local_stats.n_missed;
 962                 stats->n_lost += local_stats.n_lost;
 963         }
 964 }
 965
 966 static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = {
 967         [OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
 968         [OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
 969         [OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
 970 };
 971
 972 static struct genl_family dp_flow_genl_family = {
 973         .id = GENL_ID_GENERATE,
 974         .hdrsize = sizeof(struct ovs_header),
 975         .name = OVS_FLOW_FAMILY,
 976         .version = OVS_FLOW_VERSION,
 977         .maxattr = OVS_FLOW_ATTR_MAX,
 978          SET_NETNSOK
 979 };
 980
 981 static struct genl_multicast_group ovs_dp_flow_multicast_group = {
 982         .name = OVS_FLOW_MCGROUP
 983 };
 984
 985 static int actions_to_attr(const struct nlattr *attr, int len, struct sk_buff *skb);
 986 static int sample_action_to_attr(const struct nlattr *attr, struct sk_buff *skb)
 987 {
 988         const struct nlattr *a;
 989         struct nlattr *start;
 990         int err = 0, rem;
 991
 992         start = nla_nest_start(skb, OVS_ACTION_ATTR_SAMPLE);
 993         if (!start)
 994                 return -EMSGSIZE;
 995
 996         nla_for_each_nested(a, attr, rem) {
 997                 int type = nla_type(a);
 998                 struct nlattr *st_sample;
 999
1000                 switch (type) {
1001                 case OVS_SAMPLE_ATTR_PROBABILITY:
1002                         if (nla_put(skb, OVS_SAMPLE_ATTR_PROBABILITY, sizeof(u32), nla_data(a)))
1003                                 return -EMSGSIZE;
1004                         break;
1005                 case OVS_SAMPLE_ATTR_ACTIONS:
1006                         st_sample = nla_nest_start(skb, OVS_SAMPLE_ATTR_ACTIONS);
1007                         if (!st_sample)
1008                                 return -EMSGSIZE;
1009                         err = actions_to_attr(nla_data(a), nla_len(a), skb);
1010                         if (err)
1011                                 return err;
1012                         nla_nest_end(skb, st_sample);
1013                         break;
1014                 }
1015         }
1016
1017         nla_nest_end(skb, start);
1018         return err;
1019 }
1020
1021 static int set_action_to_attr(const struct nlattr *a, struct sk_buff *skb)
1022 {
1023         const struct nlattr *ovs_key = nla_data(a);
1024         int key_type = nla_type(ovs_key);
1025         struct nlattr *start;
1026         int err;
1027
1028         switch (key_type) {
1029         case OVS_KEY_ATTR_IPV4_TUNNEL:
1030                 start = nla_nest_start(skb, OVS_ACTION_ATTR_SET);
1031                 if (!start)
1032                         return -EMSGSIZE;
1033
1034                 err = ipv4_tun_to_nlattr(skb, nla_data(ovs_key));
1035                 if (err)
1036                         return err;
1037                 nla_nest_end(skb, start);
1038                 break;
1039         default:
1040                 if (nla_put(skb, OVS_ACTION_ATTR_SET, nla_len(a), ovs_key))
1041                         return -EMSGSIZE;
1042                 break;
1043         }
1044
1045         return 0;
1046 }
1047
1048 static int actions_to_attr(const struct nlattr *attr, int len, struct sk_buff *skb)
1049 {
1050         const struct nlattr *a;
1051         int rem, err;
1052
1053         nla_for_each_attr(a, attr, len, rem) {
1054                 int type = nla_type(a);
1055
1056                 switch (type) {
1057                 case OVS_ACTION_ATTR_SET:
1058                         err = set_action_to_attr(a, skb);
1059                         if (err)
1060                                 return err;
1061                         break;
1062
1063                 case OVS_ACTION_ATTR_SAMPLE:
1064                         err = sample_action_to_attr(a, skb);
1065                         if (err)
1066                                 return err;
1067                         break;
1068                 default:
1069                         if (nla_put(skb, type, nla_len(a), nla_data(a)))
1070                                 return -EMSGSIZE;
1071                         break;
1072                 }
1073         }
1074
1075         return 0;
1076 }
1077
1078 static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts)
1079 {
1080         return NLMSG_ALIGN(sizeof(struct ovs_header))
1081                 + nla_total_size(key_attr_size()) /* OVS_FLOW_ATTR_KEY */
1082                 + nla_total_size(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */
1083                 + nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */
1084                 + nla_total_size(8) /* OVS_FLOW_ATTR_USED */
1085                 + nla_total_size(acts->actions_len); /* OVS_FLOW_ATTR_ACTIONS */
1086 }
1087
1088 /* Called with genl_lock. */
1089 static int ovs_flow_cmd_fill_info(struct sw_flow *flow, struct datapath *dp,
1090                                   struct sk_buff *skb, u32 portid,
1091                                   u32 seq, u32 flags, u8 cmd)
1092 {
1093         const int skb_orig_len = skb->len;
1094         const struct sw_flow_actions *sf_acts;
1095         struct nlattr *start;
1096         struct ovs_flow_stats stats;
1097         struct ovs_header *ovs_header;
1098         struct nlattr *nla;
1099         unsigned long used;
1100         u8 tcp_flags;
1101         int err;
1102
1103         sf_acts = rcu_dereference_protected(flow->sf_acts,
1104                                             lockdep_genl_is_held());
1105
1106         ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family, flags, cmd);
1107         if (!ovs_header)
1108                 return -EMSGSIZE;
1109
1110         ovs_header->dp_ifindex = get_dpifindex(dp);
1111
1112         nla = nla_nest_start(skb, OVS_FLOW_ATTR_KEY);
1113         if (!nla)
1114                 goto nla_put_failure;
1115         err = ovs_flow_to_nlattrs(&flow->key, skb);
1116         if (err)
1117                 goto error;
1118         nla_nest_end(skb, nla);
1119
1120         spin_lock_bh(&flow->lock);
1121         used = flow->used;
1122         stats.n_packets = flow->packet_count;
1123         stats.n_bytes = flow->byte_count;
1124         tcp_flags = flow->tcp_flags;
1125         spin_unlock_bh(&flow->lock);
1126
1127         if (used &&
1128             nla_put_u64(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used)))
1129                 goto nla_put_failure;
1130
1131         if (stats.n_packets &&
1132             nla_put(skb, OVS_FLOW_ATTR_STATS,
1133                     sizeof(struct ovs_flow_stats), &stats))
1134                 goto nla_put_failure;
1135
1136         if (tcp_flags &&
1137             nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, tcp_flags))
1138                 goto nla_put_failure;
1139
1140         /* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
1141          * this is the first flow to be dumped into 'skb'.  This is unusual for
1142          * Netlink but individual action lists can be longer than
1143          * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this.
1144          * The userspace caller can always fetch the actions separately if it
1145          * really wants them.  (Most userspace callers in fact don't care.)
1146          *
1147          * This can only fail for dump operations because the skb is always
1148          * properly sized for single flows.
1149          */
1150         start = nla_nest_start(skb, OVS_FLOW_ATTR_ACTIONS);
1151         if (start) {
1152                 err = actions_to_attr(sf_acts->actions, sf_acts->actions_len, skb);
1153                 if (!err)
1154                         nla_nest_end(skb, start);
1155                 else {
1156                         if (skb_orig_len)
1157                                 goto error;
1158
1159                         nla_nest_cancel(skb, start);
1160                 }
1161         } else if (skb_orig_len)
1162                 goto nla_put_failure;
1163
1164         return genlmsg_end(skb, ovs_header);
1165
1166 nla_put_failure:
1167         err = -EMSGSIZE;
1168 error:
1169         genlmsg_cancel(skb, ovs_header);
1170         return err;
1171 }
1172
1173 static struct sk_buff *ovs_flow_cmd_alloc_info(struct sw_flow *flow)
1174 {
1175         const struct sw_flow_actions *sf_acts;
1176
1177         sf_acts = rcu_dereference_protected(flow->sf_acts,
1178                                             lockdep_genl_is_held());
1179
1180         return genlmsg_new(ovs_flow_cmd_msg_size(sf_acts), GFP_KERNEL);
1181 }
1182
1183 static struct sk_buff *ovs_flow_cmd_build_info(struct sw_flow *flow,
1184                                                struct datapath *dp,
1185                                                u32 portid, u32 seq, u8 cmd)
1186 {
1187         struct sk_buff *skb;
1188         int retval;
1189
1190         skb = ovs_flow_cmd_alloc_info(flow);
1191         if (!skb)
1192                 return ERR_PTR(-ENOMEM);
1193
1194         retval = ovs_flow_cmd_fill_info(flow, dp, skb, portid, seq, 0, cmd);
1195         BUG_ON(retval < 0);
1196         return skb;
1197 }
1198
1199 static int ovs_flow_cmd_new_or_set(struct sk_buff *skb, struct genl_info *info)
1200 {
1201         struct nlattr **a = info->attrs;
1202         struct ovs_header *ovs_header = info->userhdr;
1203         struct sw_flow_key key;
1204         struct sw_flow *flow;
1205         struct sk_buff *reply;
1206         struct datapath *dp;
1207         struct flow_table *table;
1208         struct sw_flow_actions *acts = NULL;
1209         int error;
1210         int key_len;
1211
1212         /* Extract key. */
1213         error = -EINVAL;
1214         if (!a[OVS_FLOW_ATTR_KEY])
1215                 goto error;
1216         error = ovs_flow_from_nlattrs(&key, &key_len, a[OVS_FLOW_ATTR_KEY]);
1217         if (error)
1218                 goto error;
1219
1220         /* Validate actions. */
1221         if (a[OVS_FLOW_ATTR_ACTIONS]) {
1222                 acts = ovs_flow_actions_alloc(nla_len(a[OVS_FLOW_ATTR_ACTIONS]));
1223                 error = PTR_ERR(acts);
1224                 if (IS_ERR(acts))
1225                         goto error;
1226
1227                 error = validate_and_copy_actions(a[OVS_FLOW_ATTR_ACTIONS], &key,  0, &acts);
1228                 if (error)
1229                         goto err_kfree;
1230         } else if (info->genlhdr->cmd == OVS_FLOW_CMD_NEW) {
1231                 error = -EINVAL;
1232                 goto error;
1233         }
1234
1235         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1236         error = -ENODEV;
1237         if (!dp)
1238                 goto err_kfree;
1239
1240         table = genl_dereference(dp->table);
1241         flow = ovs_flow_tbl_lookup(table, &key, key_len);
1242         if (!flow) {
1243                 /* Bail out if we're not allowed to create a new flow. */
1244                 error = -ENOENT;
1245                 if (info->genlhdr->cmd == OVS_FLOW_CMD_SET)
1246                         goto err_kfree;
1247
1248                 /* Expand table, if necessary, to make room. */
1249                 if (ovs_flow_tbl_need_to_expand(table)) {
1250                         struct flow_table *new_table;
1251
1252                         new_table = ovs_flow_tbl_expand(table);
1253                         if (!IS_ERR(new_table)) {
1254                                 rcu_assign_pointer(dp->table, new_table);
1255                                 ovs_flow_tbl_deferred_destroy(table);
1256                                 table = genl_dereference(dp->table);
1257                         }
1258                 }
1259
1260                 /* Allocate flow. */
1261                 flow = ovs_flow_alloc();
1262                 if (IS_ERR(flow)) {
1263                         error = PTR_ERR(flow);
1264                         goto err_kfree;
1265                 }
1266                 clear_stats(flow);
1267
1268                 rcu_assign_pointer(flow->sf_acts, acts);
1269
1270                 /* Put flow in bucket. */
1271                 ovs_flow_tbl_insert(table, flow, &key, key_len);
1272
1273                 reply = ovs_flow_cmd_build_info(flow, dp, info->snd_portid,
1274                                                 info->snd_seq,
1275                                                 OVS_FLOW_CMD_NEW);
1276         } else {
1277                 /* We found a matching flow. */
1278                 struct sw_flow_actions *old_acts;
1279
1280                 /* Bail out if we're not allowed to modify an existing flow.
1281                  * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
1282                  * because Generic Netlink treats the latter as a dump
1283                  * request.  We also accept NLM_F_EXCL in case that bug ever
1284                  * gets fixed.
1285                  */
1286                 error = -EEXIST;
1287                 if (info->genlhdr->cmd == OVS_FLOW_CMD_NEW &&
1288                     info->nlhdr->nlmsg_flags & (NLM_F_CREATE | NLM_F_EXCL))
1289                         goto err_kfree;
1290
1291                 /* Update actions. */
1292                 old_acts = rcu_dereference_protected(flow->sf_acts,
1293                                                      lockdep_genl_is_held());
1294                 rcu_assign_pointer(flow->sf_acts, acts);
1295                 ovs_flow_deferred_free_acts(old_acts);
1296
1297                 reply = ovs_flow_cmd_build_info(flow, dp, info->snd_portid,
1298                                                info->snd_seq, OVS_FLOW_CMD_NEW);
1299
1300                 /* Clear stats. */
1301                 if (a[OVS_FLOW_ATTR_CLEAR]) {
1302                         spin_lock_bh(&flow->lock);
1303                         clear_stats(flow);
1304                         spin_unlock_bh(&flow->lock);
1305                 }
1306         }
1307
1308         if (!IS_ERR(reply))
1309                 genl_notify(reply, genl_info_net(info), info->snd_portid,
1310                            ovs_dp_flow_multicast_group.id, info->nlhdr,
1311                            GFP_KERNEL);
1312         else
1313                 netlink_set_err(GENL_SOCK(sock_net(skb->sk)), 0,
1314                                 ovs_dp_flow_multicast_group.id, PTR_ERR(reply));
1315         return 0;
1316
1317 err_kfree:
1318         kfree(acts);
1319 error:
1320         return error;
1321 }
1322
1323 static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
1324 {
1325         struct nlattr **a = info->attrs;
1326         struct ovs_header *ovs_header = info->userhdr;
1327         struct sw_flow_key key;
1328         struct sk_buff *reply;
1329         struct sw_flow *flow;
1330         struct datapath *dp;
1331         struct flow_table *table;
1332         int err;
1333         int key_len;
1334
1335         if (!a[OVS_FLOW_ATTR_KEY])
1336                 return -EINVAL;
1337         err = ovs_flow_from_nlattrs(&key, &key_len, a[OVS_FLOW_ATTR_KEY]);
1338         if (err)
1339                 return err;
1340
1341         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1342         if (!dp)
1343                 return -ENODEV;
1344
1345         table = genl_dereference(dp->table);
1346         flow = ovs_flow_tbl_lookup(table, &key, key_len);
1347         if (!flow)
1348                 return -ENOENT;
1349
1350         reply = ovs_flow_cmd_build_info(flow, dp, info->snd_portid,
1351                                         info->snd_seq, OVS_FLOW_CMD_NEW);
1352         if (IS_ERR(reply))
1353                 return PTR_ERR(reply);
1354
1355         return genlmsg_reply(reply, info);
1356 }
1357
1358 static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
1359 {
1360         struct nlattr **a = info->attrs;
1361         struct ovs_header *ovs_header = info->userhdr;
1362         struct sw_flow_key key;
1363         struct sk_buff *reply;
1364         struct sw_flow *flow;
1365         struct datapath *dp;
1366         struct flow_table *table;
1367         int err;
1368         int key_len;
1369
1370         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1371         if (!dp)
1372                 return -ENODEV;
1373
1374         if (!a[OVS_FLOW_ATTR_KEY])
1375                 return flush_flows(dp);
1376
1377         err = ovs_flow_from_nlattrs(&key, &key_len, a[OVS_FLOW_ATTR_KEY]);
1378         if (err)
1379                 return err;
1380
1381         table = genl_dereference(dp->table);
1382         flow = ovs_flow_tbl_lookup(table, &key, key_len);
1383         if (!flow)
1384                 return -ENOENT;
1385
1386         reply = ovs_flow_cmd_alloc_info(flow);
1387         if (!reply)
1388                 return -ENOMEM;
1389
1390         ovs_flow_tbl_remove(table, flow);
1391
1392         err = ovs_flow_cmd_fill_info(flow, dp, reply, info->snd_portid,
1393                                      info->snd_seq, 0, OVS_FLOW_CMD_DEL);
1394         BUG_ON(err < 0);
1395
1396         ovs_flow_deferred_free(flow);
1397
1398         genl_notify(reply, genl_info_net(info), info->snd_portid,
1399                     ovs_dp_flow_multicast_group.id, info->nlhdr, GFP_KERNEL);
1400         return 0;
1401 }
1402
1403 static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1404 {
1405         struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
1406         struct datapath *dp;
1407         struct flow_table *table;
1408
1409         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1410         if (!dp)
1411                 return -ENODEV;
1412
1413         table = genl_dereference(dp->table);
1414
1415         for (;;) {
1416                 struct sw_flow *flow;
1417                 u32 bucket, obj;
1418
1419                 bucket = cb->args[0];
1420                 obj = cb->args[1];
1421                 flow = ovs_flow_tbl_next(table, &bucket, &obj);
1422                 if (!flow)
1423                         break;
1424
1425                 if (ovs_flow_cmd_fill_info(flow, dp, skb,
1426                                            NETLINK_CB(cb->skb).portid,
1427                                            cb->nlh->nlmsg_seq, NLM_F_MULTI,
1428                                            OVS_FLOW_CMD_NEW) < 0)
1429                         break;
1430
1431                 cb->args[0] = bucket;
1432                 cb->args[1] = obj;
1433         }
1434         return skb->len;
1435 }
1436
1437 static struct genl_ops dp_flow_genl_ops[] = {
1438         { .cmd = OVS_FLOW_CMD_NEW,
1439           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1440           .policy = flow_policy,
1441           .doit = ovs_flow_cmd_new_or_set
1442         },
1443         { .cmd = OVS_FLOW_CMD_DEL,
1444           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1445           .policy = flow_policy,
1446           .doit = ovs_flow_cmd_del
1447         },
1448         { .cmd = OVS_FLOW_CMD_GET,
1449           .flags = 0,               /* OK for unprivileged users. */
1450           .policy = flow_policy,
1451           .doit = ovs_flow_cmd_get,
1452           .dumpit = ovs_flow_cmd_dump
1453         },
1454         { .cmd = OVS_FLOW_CMD_SET,
1455           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1456           .policy = flow_policy,
1457           .doit = ovs_flow_cmd_new_or_set,
1458         },
1459 };
1460
1461 static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
1462 #ifdef HAVE_NLA_NUL_STRING
1463         [OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
1464 #endif
1465         [OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
1466 };
1467
1468 static struct genl_family dp_datapath_genl_family = {
1469         .id = GENL_ID_GENERATE,
1470         .hdrsize = sizeof(struct ovs_header),
1471         .name = OVS_DATAPATH_FAMILY,
1472         .version = OVS_DATAPATH_VERSION,
1473         .maxattr = OVS_DP_ATTR_MAX,
1474          SET_NETNSOK
1475 };
1476
1477 static struct genl_multicast_group ovs_dp_datapath_multicast_group = {
1478         .name = OVS_DATAPATH_MCGROUP
1479 };
1480
1481 static size_t ovs_dp_cmd_msg_size(void)
1482 {
1483         size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));
1484
1485         msgsize += nla_total_size(IFNAMSIZ);
1486         msgsize += nla_total_size(sizeof(struct ovs_dp_stats));
1487
1488         return msgsize;
1489 }
1490
1491 static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
1492                                 u32 portid, u32 seq, u32 flags, u8 cmd)
1493 {
1494         struct ovs_header *ovs_header;
1495         struct ovs_dp_stats dp_stats;
1496         int err;
1497
1498         ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
1499                                    flags, cmd);
1500         if (!ovs_header)
1501                 goto error;
1502
1503         ovs_header->dp_ifindex = get_dpifindex(dp);
1504
1505         rcu_read_lock();
1506         err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp));
1507         rcu_read_unlock();
1508         if (err)
1509                 goto nla_put_failure;
1510
1511         get_dp_stats(dp, &dp_stats);
1512         if (nla_put(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats), &dp_stats))
1513                 goto nla_put_failure;
1514
1515         return genlmsg_end(skb, ovs_header);
1516
1517 nla_put_failure:
1518         genlmsg_cancel(skb, ovs_header);
1519 error:
1520         return -EMSGSIZE;
1521 }
1522
1523 static struct sk_buff *ovs_dp_cmd_build_info(struct datapath *dp, u32 portid,
1524                                              u32 seq, u8 cmd)
1525 {
1526         struct sk_buff *skb;
1527         int retval;
1528
1529         skb = genlmsg_new(ovs_dp_cmd_msg_size(), GFP_KERNEL);
1530         if (!skb)
1531                 return ERR_PTR(-ENOMEM);
1532
1533         retval = ovs_dp_cmd_fill_info(dp, skb, portid, seq, 0, cmd);
1534         if (retval < 0) {
1535                 kfree_skb(skb);
1536                 return ERR_PTR(retval);
1537         }
1538         return skb;
1539 }
1540
1541 static int ovs_dp_cmd_validate(struct nlattr *a[OVS_DP_ATTR_MAX + 1])
1542 {
1543         return CHECK_NUL_STRING(a[OVS_DP_ATTR_NAME], IFNAMSIZ - 1);
1544 }
1545
1546 /* Called with genl_mutex and optionally with RTNL lock also. */
1547 static struct datapath *lookup_datapath(struct net *net,
1548                                         struct ovs_header *ovs_header,
1549                                         struct nlattr *a[OVS_DP_ATTR_MAX + 1])
1550 {
1551         struct datapath *dp;
1552
1553         if (!a[OVS_DP_ATTR_NAME])
1554                 dp = get_dp(net, ovs_header->dp_ifindex);
1555         else {
1556                 struct vport *vport;
1557
1558                 rcu_read_lock();
1559                 vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME]));
1560                 dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL;
1561                 rcu_read_unlock();
1562         }
1563         return dp ? dp : ERR_PTR(-ENODEV);
1564 }
1565
1566 static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
1567 {
1568         struct nlattr **a = info->attrs;
1569         struct vport_parms parms;
1570         struct sk_buff *reply;
1571         struct datapath *dp;
1572         struct vport *vport;
1573         struct ovs_net *ovs_net;
1574         int err, i;
1575
1576         err = -EINVAL;
1577         if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID])
1578                 goto err;
1579
1580         err = ovs_dp_cmd_validate(a);
1581         if (err)
1582                 goto err;
1583
1584         rtnl_lock();
1585
1586         err = -ENOMEM;
1587         dp = kzalloc(sizeof(*dp), GFP_KERNEL);
1588         if (dp == NULL)
1589                 goto err_unlock_rtnl;
1590
1591         ovs_dp_set_net(dp, hold_net(sock_net(skb->sk)));
1592
1593         /* Allocate table. */
1594         err = -ENOMEM;
1595         rcu_assign_pointer(dp->table, ovs_flow_tbl_alloc(TBL_MIN_BUCKETS));
1596         if (!dp->table)
1597                 goto err_free_dp;
1598
1599         dp->stats_percpu = alloc_percpu(struct dp_stats_percpu);
1600         if (!dp->stats_percpu) {
1601                 err = -ENOMEM;
1602                 goto err_destroy_table;
1603         }
1604
1605         dp->ports = kmalloc(DP_VPORT_HASH_BUCKETS * sizeof(struct hlist_head),
1606                             GFP_KERNEL);
1607         if (!dp->ports) {
1608                 err = -ENOMEM;
1609                 goto err_destroy_percpu;
1610         }
1611
1612         for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
1613                 INIT_HLIST_HEAD(&dp->ports[i]);
1614
1615         /* Set up our datapath device. */
1616         parms.name = nla_data(a[OVS_DP_ATTR_NAME]);
1617         parms.type = OVS_VPORT_TYPE_INTERNAL;
1618         parms.options = NULL;
1619         parms.dp = dp;
1620         parms.port_no = OVSP_LOCAL;
1621         parms.upcall_portid = nla_get_u32(a[OVS_DP_ATTR_UPCALL_PID]);
1622
1623         vport = new_vport(&parms);
1624         if (IS_ERR(vport)) {
1625                 err = PTR_ERR(vport);
1626                 if (err == -EBUSY)
1627                         err = -EEXIST;
1628
1629                 goto err_destroy_ports_array;
1630         }
1631
1632         reply = ovs_dp_cmd_build_info(dp, info->snd_portid,
1633                                       info->snd_seq, OVS_DP_CMD_NEW);
1634         err = PTR_ERR(reply);
1635         if (IS_ERR(reply))
1636                 goto err_destroy_local_port;
1637
1638         ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);
1639         list_add_tail(&dp->list_node, &ovs_net->dps);
1640
1641         rtnl_unlock();
1642
1643         genl_notify(reply, genl_info_net(info), info->snd_portid,
1644                     ovs_dp_datapath_multicast_group.id, info->nlhdr,
1645                     GFP_KERNEL);
1646         return 0;
1647
1648 err_destroy_local_port:
1649         ovs_dp_detach_port(ovs_vport_rtnl(dp, OVSP_LOCAL));
1650 err_destroy_ports_array:
1651         kfree(dp->ports);
1652 err_destroy_percpu:
1653         free_percpu(dp->stats_percpu);
1654 err_destroy_table:
1655         ovs_flow_tbl_destroy(genl_dereference(dp->table));
1656 err_free_dp:
1657         release_net(ovs_dp_get_net(dp));
1658         kfree(dp);
1659 err_unlock_rtnl:
1660         rtnl_unlock();
1661 err:
1662         return err;
1663 }
1664
1665 /* Called with genl_mutex. */
1666 static void __dp_destroy(struct datapath *dp)
1667 {
1668         int i;
1669
1670         rtnl_lock();
1671
1672         for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
1673                 struct vport *vport;
1674                 struct hlist_node *n;
1675
1676                 hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node)
1677                         if (vport->port_no != OVSP_LOCAL)
1678                                 ovs_dp_detach_port(vport);
1679         }
1680
1681         list_del(&dp->list_node);
1682         ovs_dp_detach_port(ovs_vport_rtnl(dp, OVSP_LOCAL));
1683
1684         /* rtnl_unlock() will wait until all the references to devices that
1685          * are pending unregistration have been dropped.  We do it here to
1686          * ensure that any internal devices (which contain DP pointers) are
1687          * fully destroyed before freeing the datapath.
1688          */
1689         rtnl_unlock();
1690
1691         call_rcu(&dp->rcu, destroy_dp_rcu);
1692 }
1693
1694 static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info)
1695 {
1696         struct sk_buff *reply;
1697         struct datapath *dp;
1698         int err;
1699
1700         err = ovs_dp_cmd_validate(info->attrs);
1701         if (err)
1702                 return err;
1703
1704         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1705         err = PTR_ERR(dp);
1706         if (IS_ERR(dp))
1707                 return err;
1708
1709         reply = ovs_dp_cmd_build_info(dp, info->snd_portid,
1710                                       info->snd_seq, OVS_DP_CMD_DEL);
1711         err = PTR_ERR(reply);
1712         if (IS_ERR(reply))
1713                 return err;
1714
1715         __dp_destroy(dp);
1716
1717         genl_notify(reply, genl_info_net(info), info->snd_portid,
1718                     ovs_dp_datapath_multicast_group.id, info->nlhdr,
1719                     GFP_KERNEL);
1720
1721         return 0;
1722 }
1723
1724 static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info)
1725 {
1726         struct sk_buff *reply;
1727         struct datapath *dp;
1728         int err;
1729
1730         err = ovs_dp_cmd_validate(info->attrs);
1731         if (err)
1732                 return err;
1733
1734         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1735         if (IS_ERR(dp))
1736                 return PTR_ERR(dp);
1737
1738         reply = ovs_dp_cmd_build_info(dp, info->snd_portid,
1739                                       info->snd_seq, OVS_DP_CMD_NEW);
1740         if (IS_ERR(reply)) {
1741                 err = PTR_ERR(reply);
1742                 netlink_set_err(GENL_SOCK(sock_net(skb->sk)), 0,
1743                                 ovs_dp_datapath_multicast_group.id, err);
1744                 return 0;
1745         }
1746
1747         genl_notify(reply, genl_info_net(info), info->snd_portid,
1748                     ovs_dp_datapath_multicast_group.id, info->nlhdr,
1749                     GFP_KERNEL);
1750
1751         return 0;
1752 }
1753
1754 static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info)
1755 {
1756         struct sk_buff *reply;
1757         struct datapath *dp;
1758         int err;
1759
1760         err = ovs_dp_cmd_validate(info->attrs);
1761         if (err)
1762                 return err;
1763
1764         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1765         if (IS_ERR(dp))
1766                 return PTR_ERR(dp);
1767
1768         reply = ovs_dp_cmd_build_info(dp, info->snd_portid,
1769                                       info->snd_seq, OVS_DP_CMD_NEW);
1770         if (IS_ERR(reply))
1771                 return PTR_ERR(reply);
1772
1773         return genlmsg_reply(reply, info);
1774 }
1775
1776 static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1777 {
1778         struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
1779         struct datapath *dp;
1780         int skip = cb->args[0];
1781         int i = 0;
1782
1783         list_for_each_entry(dp, &ovs_net->dps, list_node) {
1784                 if (i >= skip &&
1785                     ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid,
1786                                          cb->nlh->nlmsg_seq, NLM_F_MULTI,
1787                                          OVS_DP_CMD_NEW) < 0)
1788                         break;
1789                 i++;
1790         }
1791
1792         cb->args[0] = i;
1793
1794         return skb->len;
1795 }
1796
1797 static struct genl_ops dp_datapath_genl_ops[] = {
1798         { .cmd = OVS_DP_CMD_NEW,
1799           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1800           .policy = datapath_policy,
1801           .doit = ovs_dp_cmd_new
1802         },
1803         { .cmd = OVS_DP_CMD_DEL,
1804           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1805           .policy = datapath_policy,
1806           .doit = ovs_dp_cmd_del
1807         },
1808         { .cmd = OVS_DP_CMD_GET,
1809           .flags = 0,               /* OK for unprivileged users. */
1810           .policy = datapath_policy,
1811           .doit = ovs_dp_cmd_get,
1812           .dumpit = ovs_dp_cmd_dump
1813         },
1814         { .cmd = OVS_DP_CMD_SET,
1815           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1816           .policy = datapath_policy,
1817           .doit = ovs_dp_cmd_set,
1818         },
1819 };
1820
1821 static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = {
1822 #ifdef HAVE_NLA_NUL_STRING
1823         [OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
1824         [OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) },
1825 #else
1826         [OVS_VPORT_ATTR_STATS] = { .minlen = sizeof(struct ovs_vport_stats) },
1827 #endif
1828         [OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 },
1829         [OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 },
1830         [OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_U32 },
1831         [OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED },
1832 };
1833
1834 static struct genl_family dp_vport_genl_family = {
1835         .id = GENL_ID_GENERATE,
1836         .hdrsize = sizeof(struct ovs_header),
1837         .name = OVS_VPORT_FAMILY,
1838         .version = OVS_VPORT_VERSION,
1839         .maxattr = OVS_VPORT_ATTR_MAX,
1840          SET_NETNSOK
1841 };
1842
1843 struct genl_multicast_group ovs_dp_vport_multicast_group = {
1844         .name = OVS_VPORT_MCGROUP
1845 };
1846
1847 /* Called with RTNL lock or RCU read lock. */
1848 static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb,
1849                                    u32 portid, u32 seq, u32 flags, u8 cmd)
1850 {
1851         struct ovs_header *ovs_header;
1852         struct ovs_vport_stats vport_stats;
1853         int err;
1854
1855         ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family,
1856                                  flags, cmd);
1857         if (!ovs_header)
1858                 return -EMSGSIZE;
1859
1860         ovs_header->dp_ifindex = get_dpifindex(vport->dp);
1861
1862         if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) ||
1863             nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) ||
1864             nla_put_string(skb, OVS_VPORT_ATTR_NAME, vport->ops->get_name(vport)) ||
1865             nla_put_u32(skb, OVS_VPORT_ATTR_UPCALL_PID, vport->upcall_portid))
1866                 goto nla_put_failure;
1867
1868         ovs_vport_get_stats(vport, &vport_stats);
1869         if (nla_put(skb, OVS_VPORT_ATTR_STATS, sizeof(struct ovs_vport_stats),
1870                     &vport_stats))
1871                 goto nla_put_failure;
1872
1873         err = ovs_vport_get_options(vport, skb);
1874         if (err == -EMSGSIZE)
1875                 goto error;
1876
1877         return genlmsg_end(skb, ovs_header);
1878
1879 nla_put_failure:
1880         err = -EMSGSIZE;
1881 error:
1882         genlmsg_cancel(skb, ovs_header);
1883         return err;
1884 }
1885
1886 /* Called with RTNL lock or RCU read lock. */
1887 struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, u32 portid,
1888                                          u32 seq, u8 cmd)
1889 {
1890         struct sk_buff *skb;
1891         int retval;
1892
1893         skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
1894         if (!skb)
1895                 return ERR_PTR(-ENOMEM);
1896
1897         retval = ovs_vport_cmd_fill_info(vport, skb, portid, seq, 0, cmd);
1898         BUG_ON(retval < 0);
1899
1900         return skb;
1901 }
1902
1903 static int ovs_vport_cmd_validate(struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
1904 {
1905         return CHECK_NUL_STRING(a[OVS_VPORT_ATTR_NAME], IFNAMSIZ - 1);
1906 }
1907
1908 /* Called with RTNL lock or RCU read lock. */
1909 static struct vport *lookup_vport(struct net *net,
1910                                   struct ovs_header *ovs_header,
1911                                   struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
1912 {
1913         struct datapath *dp;
1914         struct vport *vport;
1915
1916         if (a[OVS_VPORT_ATTR_NAME]) {
1917                 vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME]));
1918                 if (!vport)
1919                         return ERR_PTR(-ENODEV);
1920                 if (ovs_header->dp_ifindex &&
1921                     ovs_header->dp_ifindex != get_dpifindex(vport->dp))
1922                         return ERR_PTR(-ENODEV);
1923                 return vport;
1924         } else if (a[OVS_VPORT_ATTR_PORT_NO]) {
1925                 u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
1926
1927                 if (port_no >= DP_MAX_PORTS)
1928                         return ERR_PTR(-EFBIG);
1929
1930                 dp = get_dp(net, ovs_header->dp_ifindex);
1931                 if (!dp)
1932                         return ERR_PTR(-ENODEV);
1933
1934                 vport = ovs_vport_rtnl_rcu(dp, port_no);
1935                 if (!vport)
1936                         return ERR_PTR(-ENODEV);
1937                 return vport;
1938         } else
1939                 return ERR_PTR(-EINVAL);
1940 }
1941
1942 static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
1943 {
1944         struct nlattr **a = info->attrs;
1945         struct ovs_header *ovs_header = info->userhdr;
1946         struct vport_parms parms;
1947         struct sk_buff *reply;
1948         struct vport *vport;
1949         struct datapath *dp;
1950         u32 port_no;
1951         int err;
1952
1953         err = -EINVAL;
1954         if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] ||
1955             !a[OVS_VPORT_ATTR_UPCALL_PID])
1956                 goto exit;
1957
1958         err = ovs_vport_cmd_validate(a);
1959         if (err)
1960                 goto exit;
1961
1962         rtnl_lock();
1963         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1964         err = -ENODEV;
1965         if (!dp)
1966                 goto exit_unlock;
1967
1968         if (a[OVS_VPORT_ATTR_PORT_NO]) {
1969                 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
1970
1971                 err = -EFBIG;
1972                 if (port_no >= DP_MAX_PORTS)
1973                         goto exit_unlock;
1974
1975                 vport = ovs_vport_rtnl(dp, port_no);
1976                 err = -EBUSY;
1977                 if (vport)
1978                         goto exit_unlock;
1979         } else {
1980                 for (port_no = 1; ; port_no++) {
1981                         if (port_no >= DP_MAX_PORTS) {
1982                                 err = -EFBIG;
1983                                 goto exit_unlock;
1984                         }
1985                         vport = ovs_vport_rtnl(dp, port_no);
1986                         if (!vport)
1987                                 break;
1988                 }
1989         }
1990
1991         parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
1992         parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
1993         parms.options = a[OVS_VPORT_ATTR_OPTIONS];
1994         parms.dp = dp;
1995         parms.port_no = port_no;
1996         parms.upcall_portid = nla_get_u32(a[OVS_VPORT_ATTR_UPCALL_PID]);
1997
1998         vport = new_vport(&parms);
1999         err = PTR_ERR(vport);
2000         if (IS_ERR(vport))
2001                 goto exit_unlock;
2002
2003         err = 0;
2004         if (a[OVS_VPORT_ATTR_STATS])
2005                 ovs_vport_set_stats(vport, nla_data(a[OVS_VPORT_ATTR_STATS]));
2006
2007         reply = ovs_vport_cmd_build_info(vport, info->snd_portid, info->snd_seq,
2008                                          OVS_VPORT_CMD_NEW);
2009         if (IS_ERR(reply)) {
2010                 err = PTR_ERR(reply);
2011                 ovs_dp_detach_port(vport);
2012                 goto exit_unlock;
2013         }
2014         genl_notify(reply, genl_info_net(info), info->snd_portid,
2015                     ovs_dp_vport_multicast_group.id, info->nlhdr, GFP_KERNEL);
2016
2017 exit_unlock:
2018         rtnl_unlock();
2019 exit:
2020         return err;
2021 }
2022
2023 static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info)
2024 {
2025         struct nlattr **a = info->attrs;
2026         struct sk_buff *reply;
2027         struct vport *vport;
2028         int err;
2029
2030         err = ovs_vport_cmd_validate(a);
2031         if (err)
2032                 goto exit;
2033
2034         rtnl_lock();
2035         vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
2036         err = PTR_ERR(vport);
2037         if (IS_ERR(vport))
2038                 goto exit_unlock;
2039
2040         err = 0;
2041         if (a[OVS_VPORT_ATTR_TYPE] &&
2042             nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type)
2043                 err = -EINVAL;
2044
2045         reply = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2046         if (!reply) {
2047                 err = -ENOMEM;
2048                 goto exit_unlock;
2049         }
2050
2051         if (!err && a[OVS_VPORT_ATTR_OPTIONS])
2052                 err = ovs_vport_set_options(vport, a[OVS_VPORT_ATTR_OPTIONS]);
2053         if (err)
2054                 goto exit_free;
2055
2056         if (a[OVS_VPORT_ATTR_STATS])
2057                 ovs_vport_set_stats(vport, nla_data(a[OVS_VPORT_ATTR_STATS]));
2058
2059         if (a[OVS_VPORT_ATTR_UPCALL_PID])
2060                 vport->upcall_portid = nla_get_u32(a[OVS_VPORT_ATTR_UPCALL_PID]);
2061
2062         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
2063                                       info->snd_seq, 0, OVS_VPORT_CMD_NEW);
2064         BUG_ON(err < 0);
2065
2066         genl_notify(reply, genl_info_net(info), info->snd_portid,
2067                     ovs_dp_vport_multicast_group.id, info->nlhdr, GFP_KERNEL);
2068
2069         rtnl_unlock();
2070         return 0;
2071
2072 exit_free:
2073         kfree_skb(reply);
2074 exit_unlock:
2075         rtnl_unlock();
2076 exit:
2077         return err;
2078 }
2079
2080 static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info)
2081 {
2082         struct nlattr **a = info->attrs;
2083         struct sk_buff *reply;
2084         struct vport *vport;
2085         int err;
2086
2087         err = ovs_vport_cmd_validate(a);
2088         if (err)
2089                 goto exit;
2090
2091         rtnl_lock();
2092         vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
2093         err = PTR_ERR(vport);
2094         if (IS_ERR(vport))
2095                 goto exit_unlock;
2096
2097         if (vport->port_no == OVSP_LOCAL) {
2098                 err = -EINVAL;
2099                 goto exit_unlock;
2100         }
2101
2102         reply = ovs_vport_cmd_build_info(vport, info->snd_portid,
2103                                          info->snd_seq, OVS_VPORT_CMD_DEL);
2104         err = PTR_ERR(reply);
2105         if (IS_ERR(reply))
2106                 goto exit_unlock;
2107
2108         err = 0;
2109         ovs_dp_detach_port(vport);
2110
2111         genl_notify(reply, genl_info_net(info), info->snd_portid,
2112                     ovs_dp_vport_multicast_group.id, info->nlhdr, GFP_KERNEL);
2113
2114 exit_unlock:
2115         rtnl_unlock();
2116 exit:
2117         return err;
2118 }
2119
2120 static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info)
2121 {
2122         struct nlattr **a = info->attrs;
2123         struct ovs_header *ovs_header = info->userhdr;
2124         struct sk_buff *reply;
2125         struct vport *vport;
2126         int err;
2127
2128         err = ovs_vport_cmd_validate(a);
2129         if (err)
2130                 goto exit;
2131
2132         rcu_read_lock();
2133         vport = lookup_vport(sock_net(skb->sk), ovs_header, a);
2134         err = PTR_ERR(vport);
2135         if (IS_ERR(vport))
2136                 goto exit_unlock;
2137
2138         reply = ovs_vport_cmd_build_info(vport, info->snd_portid,
2139                                          info->snd_seq, OVS_VPORT_CMD_NEW);
2140         err = PTR_ERR(reply);
2141         if (IS_ERR(reply))
2142                 goto exit_unlock;
2143
2144         rcu_read_unlock();
2145
2146         return genlmsg_reply(reply, info);
2147
2148 exit_unlock:
2149         rcu_read_unlock();
2150 exit:
2151         return err;
2152 }
2153
2154 static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
2155 {
2156         struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
2157         struct datapath *dp;
2158         int bucket = cb->args[0], skip = cb->args[1];
2159         int i, j = 0;
2160
2161         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
2162         if (!dp)
2163                 return -ENODEV;
2164
2165         rcu_read_lock();
2166         for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) {
2167                 struct vport *vport;
2168
2169                 j = 0;
2170                 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
2171                         if (j >= skip &&
2172                             ovs_vport_cmd_fill_info(vport, skb,
2173                                                     NETLINK_CB(cb->skb).portid,
2174                                                     cb->nlh->nlmsg_seq,
2175                                                     NLM_F_MULTI,
2176                                                     OVS_VPORT_CMD_NEW) < 0)
2177                                 goto out;
2178
2179                         j++;
2180                 }
2181                 skip = 0;
2182         }
2183 out:
2184         rcu_read_unlock();
2185
2186         cb->args[0] = i;
2187         cb->args[1] = j;
2188
2189         return skb->len;
2190 }
2191
2192 static struct genl_ops dp_vport_genl_ops[] = {
2193         { .cmd = OVS_VPORT_CMD_NEW,
2194           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2195           .policy = vport_policy,
2196           .doit = ovs_vport_cmd_new
2197         },
2198         { .cmd = OVS_VPORT_CMD_DEL,
2199           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2200           .policy = vport_policy,
2201           .doit = ovs_vport_cmd_del
2202         },
2203         { .cmd = OVS_VPORT_CMD_GET,
2204           .flags = 0,               /* OK for unprivileged users. */
2205           .policy = vport_policy,
2206           .doit = ovs_vport_cmd_get,
2207           .dumpit = ovs_vport_cmd_dump
2208         },
2209         { .cmd = OVS_VPORT_CMD_SET,
2210           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2211           .policy = vport_policy,
2212           .doit = ovs_vport_cmd_set,
2213         },
2214 };
2215
2216 struct genl_family_and_ops {
2217         struct genl_family *family;
2218         struct genl_ops *ops;
2219         int n_ops;
2220         struct genl_multicast_group *group;
2221 };
2222
2223 static const struct genl_family_and_ops dp_genl_families[] = {
2224         { &dp_datapath_genl_family,
2225           dp_datapath_genl_ops, ARRAY_SIZE(dp_datapath_genl_ops),
2226           &ovs_dp_datapath_multicast_group },
2227         { &dp_vport_genl_family,
2228           dp_vport_genl_ops, ARRAY_SIZE(dp_vport_genl_ops),
2229           &ovs_dp_vport_multicast_group },
2230         { &dp_flow_genl_family,
2231           dp_flow_genl_ops, ARRAY_SIZE(dp_flow_genl_ops),
2232           &ovs_dp_flow_multicast_group },
2233         { &dp_packet_genl_family,
2234           dp_packet_genl_ops, ARRAY_SIZE(dp_packet_genl_ops),
2235           NULL },
2236 };
2237
2238 static void dp_unregister_genl(int n_families)
2239 {
2240         int i;
2241
2242         for (i = 0; i < n_families; i++)
2243                 genl_unregister_family(dp_genl_families[i].family);
2244 }
2245
2246 static int dp_register_genl(void)
2247 {
2248         int n_registered;
2249         int err;
2250         int i;
2251
2252         n_registered = 0;
2253         for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
2254                 const struct genl_family_and_ops *f = &dp_genl_families[i];
2255
2256                 err = genl_register_family_with_ops(f->family, f->ops,
2257                                                     f->n_ops);
2258                 if (err)
2259                         goto error;
2260                 n_registered++;
2261
2262                 if (f->group) {
2263                         err = genl_register_mc_group(f->family, f->group);
2264                         if (err)
2265                                 goto error;
2266                 }
2267         }
2268
2269         return 0;
2270
2271 error:
2272         dp_unregister_genl(n_registered);
2273         return err;
2274 }
2275
2276 static int __rehash_flow_table(void *dummy)
2277 {
2278         struct datapath *dp;
2279         struct net *net;
2280
2281         rtnl_lock();
2282         for_each_net(net) {
2283                 struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2284
2285                 list_for_each_entry(dp, &ovs_net->dps, list_node) {
2286                         struct flow_table *old_table = genl_dereference(dp->table);
2287                         struct flow_table *new_table;
2288
2289                         new_table = ovs_flow_tbl_rehash(old_table);
2290                         if (!IS_ERR(new_table)) {
2291                                 rcu_assign_pointer(dp->table, new_table);
2292                                 ovs_flow_tbl_deferred_destroy(old_table);
2293                         }
2294                 }
2295         }
2296         rtnl_unlock();
2297         return 0;
2298 }
2299
2300 static void rehash_flow_table(struct work_struct *work)
2301 {
2302         genl_exec(__rehash_flow_table, NULL);
2303         schedule_delayed_work(&rehash_flow_wq, REHASH_FLOW_INTERVAL);
2304 }
2305
2306 static int dp_destroy_all(void *data)
2307 {
2308         struct datapath *dp, *dp_next;
2309         struct ovs_net *ovs_net = data;
2310
2311         list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node)
2312                 __dp_destroy(dp);
2313
2314         return 0;
2315 }
2316
2317 static int __net_init ovs_init_net(struct net *net)
2318 {
2319         struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2320
2321         INIT_LIST_HEAD(&ovs_net->dps);
2322         return 0;
2323 }
2324
2325 static void __net_exit ovs_exit_net(struct net *net)
2326 {
2327         struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2328
2329         genl_exec(dp_destroy_all, ovs_net);
2330 }
2331
2332 static struct pernet_operations ovs_net_ops = {
2333         .init = ovs_init_net,
2334         .exit = ovs_exit_net,
2335         .id   = &ovs_net_id,
2336         .size = sizeof(struct ovs_net),
2337 };
2338
2339 static int __init dp_init(void)
2340 {
2341         int err;
2342
2343         BUILD_BUG_ON(sizeof(struct ovs_skb_cb) > FIELD_SIZEOF(struct sk_buff, cb));
2344
2345         pr_info("Open vSwitch switching datapath %s, built "__DATE__" "__TIME__"\n",
2346                 VERSION);
2347
2348         err = genl_exec_init();
2349         if (err)
2350                 goto error;
2351
2352         err = ovs_workqueues_init();
2353         if (err)
2354                 goto error_genl_exec;
2355
2356         err = ovs_flow_init();
2357         if (err)
2358                 goto error_wq;
2359
2360         err = ovs_vport_init();
2361         if (err)
2362                 goto error_flow_exit;
2363
2364         err = register_pernet_device(&ovs_net_ops);
2365         if (err)
2366                 goto error_vport_exit;
2367
2368         err = register_netdevice_notifier(&ovs_dp_device_notifier);
2369         if (err)
2370                 goto error_netns_exit;
2371
2372         err = dp_register_genl();
2373         if (err < 0)
2374                 goto error_unreg_notifier;
2375
2376         schedule_delayed_work(&rehash_flow_wq, REHASH_FLOW_INTERVAL);
2377
2378         return 0;
2379
2380 error_unreg_notifier:
2381         unregister_netdevice_notifier(&ovs_dp_device_notifier);
2382 error_netns_exit:
2383         unregister_pernet_device(&ovs_net_ops);
2384 error_vport_exit:
2385         ovs_vport_exit();
2386 error_flow_exit:
2387         ovs_flow_exit();
2388 error_wq:
2389         ovs_workqueues_exit();
2390 error_genl_exec:
2391         genl_exec_exit();
2392 error:
2393         return err;
2394 }
2395
2396 static void dp_cleanup(void)
2397 {
2398         cancel_delayed_work_sync(&rehash_flow_wq);
2399         dp_unregister_genl(ARRAY_SIZE(dp_genl_families));
2400         unregister_netdevice_notifier(&ovs_dp_device_notifier);
2401         unregister_pernet_device(&ovs_net_ops);
2402         rcu_barrier();
2403         ovs_vport_exit();
2404         ovs_flow_exit();
2405         ovs_workqueues_exit();
2406         genl_exec_exit();
2407 }
2408
2409 module_init(dp_init);
2410 module_exit(dp_cleanup);
2411
2412 MODULE_DESCRIPTION("Open vSwitch switching datapath");
2413 MODULE_LICENSE("GPL");
2414 MODULE_VERSION(VERSION);