datapath/datapath.c

   1 /*
   2  * Copyright (c) 2007-2012 Nicira, Inc.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of version 2 of the GNU General Public
   6  * License as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  16  * 02110-1301, USA
  17  */
  18
  19 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  20
  21 #include <linux/init.h>
  22 #include <linux/module.h>
  23 #include <linux/if_arp.h>
  24 #include <linux/if_vlan.h>
  25 #include <linux/in.h>
  26 #include <linux/ip.h>
  27 #include <linux/jhash.h>
  28 #include <linux/delay.h>
  29 #include <linux/time.h>
  30 #include <linux/etherdevice.h>
  31 #include <linux/genetlink.h>
  32 #include <linux/kernel.h>
  33 #include <linux/kthread.h>
  34 #include <linux/mutex.h>
  35 #include <linux/percpu.h>
  36 #include <linux/rcupdate.h>
  37 #include <linux/tcp.h>
  38 #include <linux/udp.h>
  39 #include <linux/version.h>
  40 #include <linux/ethtool.h>
  41 #include <linux/wait.h>
  42 #include <asm/div64.h>
  43 #include <linux/highmem.h>
  44 #include <linux/netfilter_bridge.h>
  45 #include <linux/netfilter_ipv4.h>
  46 #include <linux/inetdevice.h>
  47 #include <linux/list.h>
  48 #include <linux/openvswitch.h>
  49 #include <linux/rculist.h>
  50 #include <linux/dmi.h>
  51 #include <net/genetlink.h>
  52 #include <net/net_namespace.h>
  53 #include <net/netns/generic.h>
  54
  55 #include "checksum.h"
  56 #include "datapath.h"
  57 #include "flow.h"
  58 #include "genl_exec.h"
  59 #include "vlan.h"
  60 #include "tunnel.h"
  61 #include "vport-internal_dev.h"
  62
  63 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) || \
  64     LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0)
  65 #error Kernels before 2.6.18 or after 3.8 are not supported by this version of Open vSwitch.
  66 #endif
  67
  68 #define REHASH_FLOW_INTERVAL (10 * 60 * HZ)
  69 static void rehash_flow_table(struct work_struct *work);
  70 static DECLARE_DELAYED_WORK(rehash_flow_wq, rehash_flow_table);
  71
  72 int ovs_net_id __read_mostly;
  73
  74 static void ovs_notify(struct sk_buff *skb, struct genl_info *info,
  75                        struct genl_multicast_group *grp)
  76 {
  77         genl_notify(skb, genl_info_net(info), info->snd_portid,
  78                     grp->id, info->nlhdr, GFP_KERNEL);
  79 }
  80
  81 /**
  82  * DOC: Locking:
  83  *
  84  * Writes to device state (add/remove datapath, port, set operations on vports,
  85  * etc.) are protected by RTNL.
  86  *
  87  * Writes to other state (flow table modifications, set miscellaneous datapath
  88  * parameters, etc.) are protected by genl_mutex.  The RTNL lock nests inside
  89  * genl_mutex.
  90  *
  91  * Reads are protected by RCU.
  92  *
  93  * There are a few special cases (mostly stats) that have their own
  94  * synchronization but they nest under all of above and don't interact with
  95  * each other.
  96  */
  97
  98 static struct vport *new_vport(const struct vport_parms *);
  99 static int queue_gso_packets(struct net *, int dp_ifindex, struct sk_buff *,
 100                              const struct dp_upcall_info *);
 101 static int queue_userspace_packet(struct net *, int dp_ifindex,
 102                                   struct sk_buff *,
 103                                   const struct dp_upcall_info *);
 104
 105 /* Must be called with rcu_read_lock, genl_mutex, or RTNL lock. */
 106 static struct datapath *get_dp(struct net *net, int dp_ifindex)
 107 {
 108         struct datapath *dp = NULL;
 109         struct net_device *dev;
 110
 111         rcu_read_lock();
 112         dev = dev_get_by_index_rcu(net, dp_ifindex);
 113         if (dev) {
 114                 struct vport *vport = ovs_internal_dev_get_vport(dev);
 115                 if (vport)
 116                         dp = vport->dp;
 117         }
 118         rcu_read_unlock();
 119
 120         return dp;
 121 }
 122
 123 /* Must be called with rcu_read_lock or RTNL lock. */
 124 const char *ovs_dp_name(const struct datapath *dp)
 125 {
 126         struct vport *vport = ovs_vport_rtnl_rcu(dp, OVSP_LOCAL);
 127         return vport->ops->get_name(vport);
 128 }
 129
 130 static int get_dpifindex(struct datapath *dp)
 131 {
 132         struct vport *local;
 133         int ifindex;
 134
 135         rcu_read_lock();
 136
 137         local = ovs_vport_rcu(dp, OVSP_LOCAL);
 138         if (local)
 139                 ifindex = local->ops->get_ifindex(local);
 140         else
 141                 ifindex = 0;
 142
 143         rcu_read_unlock();
 144
 145         return ifindex;
 146 }
 147
 148 static void destroy_dp_rcu(struct rcu_head *rcu)
 149 {
 150         struct datapath *dp = container_of(rcu, struct datapath, rcu);
 151
 152         ovs_flow_tbl_destroy((__force struct flow_table *)dp->table);
 153         free_percpu(dp->stats_percpu);
 154         release_net(ovs_dp_get_net(dp));
 155         kfree(dp->ports);
 156         kfree(dp);
 157 }
 158
 159 static struct hlist_head *vport_hash_bucket(const struct datapath *dp,
 160                                             u16 port_no)
 161 {
 162         return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)];
 163 }
 164
 165 struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no)
 166 {
 167         struct vport *vport;
 168         struct hlist_head *head;
 169
 170         head = vport_hash_bucket(dp, port_no);
 171         hlist_for_each_entry_rcu(vport, head, dp_hash_node) {
 172                 if (vport->port_no == port_no)
 173                         return vport;
 174         }
 175         return NULL;
 176 }
 177
 178 /* Called with RTNL lock and genl_lock. */
 179 static struct vport *new_vport(const struct vport_parms *parms)
 180 {
 181         struct vport *vport;
 182
 183         vport = ovs_vport_add(parms);
 184         if (!IS_ERR(vport)) {
 185                 struct datapath *dp = parms->dp;
 186                 struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);
 187
 188                 hlist_add_head_rcu(&vport->dp_hash_node, head);
 189         }
 190         return vport;
 191 }
 192
 193 /* Called with RTNL lock. */
 194 void ovs_dp_detach_port(struct vport *p)
 195 {
 196         ASSERT_RTNL();
 197
 198         /* First drop references to device. */
 199         hlist_del_rcu(&p->dp_hash_node);
 200
 201         /* Then destroy it. */
 202         ovs_vport_del(p);
 203 }
 204
 205 /* Must be called with rcu_read_lock. */
 206 void ovs_dp_process_received_packet(struct vport *p, struct sk_buff *skb)
 207 {
 208         struct datapath *dp = p->dp;
 209         struct sw_flow *flow;
 210         struct dp_stats_percpu *stats;
 211         struct sw_flow_key key;
 212         u64 *stats_counter;
 213         int error;
 214         int key_len;
 215
 216         stats = this_cpu_ptr(dp->stats_percpu);
 217
 218         /* Extract flow from 'skb' into 'key'. */
 219         error = ovs_flow_extract(skb, p->port_no, &key, &key_len);
 220         if (unlikely(error)) {
 221                 kfree_skb(skb);
 222                 return;
 223         }
 224
 225         /* Look up flow. */
 226         flow = ovs_flow_tbl_lookup(rcu_dereference(dp->table), &key, key_len);
 227         if (unlikely(!flow)) {
 228                 struct dp_upcall_info upcall;
 229
 230                 upcall.cmd = OVS_PACKET_CMD_MISS;
 231                 upcall.key = &key;
 232                 upcall.userdata = NULL;
 233                 upcall.portid = p->upcall_portid;
 234                 ovs_dp_upcall(dp, skb, &upcall);
 235                 consume_skb(skb);
 236                 stats_counter = &stats->n_missed;
 237                 goto out;
 238         }
 239
 240         OVS_CB(skb)->flow = flow;
 241
 242         stats_counter = &stats->n_hit;
 243         ovs_flow_used(OVS_CB(skb)->flow, skb);
 244         ovs_execute_actions(dp, skb);
 245
 246 out:
 247         /* Update datapath statistics. */
 248         u64_stats_update_begin(&stats->sync);
 249         (*stats_counter)++;
 250         u64_stats_update_end(&stats->sync);
 251 }
 252
 253 static struct genl_family dp_packet_genl_family = {
 254         .id = GENL_ID_GENERATE,
 255         .hdrsize = sizeof(struct ovs_header),
 256         .name = OVS_PACKET_FAMILY,
 257         .version = OVS_PACKET_VERSION,
 258         .maxattr = OVS_PACKET_ATTR_MAX,
 259          SET_NETNSOK
 260 };
 261
 262 int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
 263                   const struct dp_upcall_info *upcall_info)
 264 {
 265         struct dp_stats_percpu *stats;
 266         int dp_ifindex;
 267         int err;
 268
 269         if (upcall_info->portid == 0) {
 270                 err = -ENOTCONN;
 271                 goto err;
 272         }
 273
 274         dp_ifindex = get_dpifindex(dp);
 275         if (!dp_ifindex) {
 276                 err = -ENODEV;
 277                 goto err;
 278         }
 279
 280         forward_ip_summed(skb, true);
 281
 282         if (!skb_is_gso(skb))
 283                 err = queue_userspace_packet(ovs_dp_get_net(dp), dp_ifindex, skb, upcall_info);
 284         else
 285                 err = queue_gso_packets(ovs_dp_get_net(dp), dp_ifindex, skb, upcall_info);
 286         if (err)
 287                 goto err;
 288
 289         return 0;
 290
 291 err:
 292         stats = this_cpu_ptr(dp->stats_percpu);
 293
 294         u64_stats_update_begin(&stats->sync);
 295         stats->n_lost++;
 296         u64_stats_update_end(&stats->sync);
 297
 298         return err;
 299 }
 300
 301 static int queue_gso_packets(struct net *net, int dp_ifindex,
 302                              struct sk_buff *skb,
 303                              const struct dp_upcall_info *upcall_info)
 304 {
 305         unsigned short gso_type = skb_shinfo(skb)->gso_type;
 306         struct dp_upcall_info later_info;
 307         struct sw_flow_key later_key;
 308         struct sk_buff *segs, *nskb;
 309         int err;
 310
 311         segs = __skb_gso_segment(skb, NETIF_F_SG | NETIF_F_HW_CSUM, false);
 312         if (IS_ERR(segs))
 313                 return PTR_ERR(segs);
 314
 315         /* Queue all of the segments. */
 316         skb = segs;
 317         do {
 318                 err = queue_userspace_packet(net, dp_ifindex, skb, upcall_info);
 319                 if (err)
 320                         break;
 321
 322                 if (skb == segs && gso_type & SKB_GSO_UDP) {
 323                         /* The initial flow key extracted by ovs_flow_extract()
 324                          * in this case is for a first fragment, so we need to
 325                          * properly mark later fragments.
 326                          */
 327                         later_key = *upcall_info->key;
 328                         later_key.ip.frag = OVS_FRAG_TYPE_LATER;
 329
 330                         later_info = *upcall_info;
 331                         later_info.key = &later_key;
 332                         upcall_info = &later_info;
 333                 }
 334         } while ((skb = skb->next));
 335
 336         /* Free all of the segments. */
 337         skb = segs;
 338         do {
 339                 nskb = skb->next;
 340                 if (err)
 341                         kfree_skb(skb);
 342                 else
 343                         consume_skb(skb);
 344         } while ((skb = nskb));
 345         return err;
 346 }
 347
 348 static size_t key_attr_size(void)
 349 {
 350         return    nla_total_size(4)   /* OVS_KEY_ATTR_PRIORITY */
 351                 + nla_total_size(0)   /* OVS_KEY_ATTR_TUNNEL */
 352                   + nla_total_size(8)   /* OVS_TUNNEL_KEY_ATTR_ID */
 353                   + nla_total_size(4)   /* OVS_TUNNEL_KEY_ATTR_IPV4_SRC */
 354                   + nla_total_size(4)   /* OVS_TUNNEL_KEY_ATTR_IPV4_DST */
 355                   + nla_total_size(1)   /* OVS_TUNNEL_KEY_ATTR_TOS */
 356                   + nla_total_size(1)   /* OVS_TUNNEL_KEY_ATTR_TTL */
 357                   + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT */
 358                   + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_CSUM */
 359                 + nla_total_size(4)   /* OVS_KEY_ATTR_IN_PORT */
 360                 + nla_total_size(4)   /* OVS_KEY_ATTR_SKB_MARK */
 361                 + nla_total_size(12)  /* OVS_KEY_ATTR_ETHERNET */
 362                 + nla_total_size(2)   /* OVS_KEY_ATTR_ETHERTYPE */
 363                 + nla_total_size(4)   /* OVS_KEY_ATTR_8021Q */
 364                 + nla_total_size(0)   /* OVS_KEY_ATTR_ENCAP */
 365                 + nla_total_size(2)   /* OVS_KEY_ATTR_ETHERTYPE */
 366                 + nla_total_size(40)  /* OVS_KEY_ATTR_IPV6 */
 367                 + nla_total_size(2)   /* OVS_KEY_ATTR_ICMPV6 */
 368                 + nla_total_size(28); /* OVS_KEY_ATTR_ND */
 369 }
 370
 371 static size_t upcall_msg_size(const struct sk_buff *skb,
 372                               const struct nlattr *userdata)
 373 {
 374         size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
 375                 + nla_total_size(skb->len) /* OVS_PACKET_ATTR_PACKET */
 376                 + nla_total_size(key_attr_size()); /* OVS_PACKET_ATTR_KEY */
 377
 378         /* OVS_PACKET_ATTR_USERDATA */
 379         if (userdata)
 380                 size += NLA_ALIGN(userdata->nla_len);
 381
 382         return size;
 383 }
 384
 385 static int queue_userspace_packet(struct net *net, int dp_ifindex,
 386                                   struct sk_buff *skb,
 387                                   const struct dp_upcall_info *upcall_info)
 388 {
 389         struct ovs_header *upcall;
 390         struct sk_buff *nskb = NULL;
 391         struct sk_buff *user_skb; /* to be queued to userspace */
 392         struct nlattr *nla;
 393         int err;
 394
 395         if (vlan_tx_tag_present(skb)) {
 396                 nskb = skb_clone(skb, GFP_ATOMIC);
 397                 if (!nskb)
 398                         return -ENOMEM;
 399
 400                 err = vlan_deaccel_tag(nskb);
 401                 if (err)
 402                         return err;
 403
 404                 skb = nskb;
 405         }
 406
 407         if (nla_attr_size(skb->len) > USHRT_MAX) {
 408                 err = -EFBIG;
 409                 goto out;
 410         }
 411
 412         user_skb = genlmsg_new(upcall_msg_size(skb, upcall_info->userdata), GFP_ATOMIC);
 413         if (!user_skb) {
 414                 err = -ENOMEM;
 415                 goto out;
 416         }
 417
 418         upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family,
 419                              0, upcall_info->cmd);
 420         upcall->dp_ifindex = dp_ifindex;
 421
 422         nla = nla_nest_start(user_skb, OVS_PACKET_ATTR_KEY);
 423         ovs_flow_to_nlattrs(upcall_info->key, user_skb);
 424         nla_nest_end(user_skb, nla);
 425
 426         if (upcall_info->userdata)
 427                 __nla_put(user_skb, OVS_PACKET_ATTR_USERDATA,
 428                           nla_len(upcall_info->userdata),
 429                           nla_data(upcall_info->userdata));
 430
 431         nla = __nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, skb->len);
 432
 433         skb_copy_and_csum_dev(skb, nla_data(nla));
 434
 435         genlmsg_end(user_skb, upcall);
 436         err = genlmsg_unicast(net, user_skb, upcall_info->portid);
 437
 438 out:
 439         kfree_skb(nskb);
 440         return err;
 441 }
 442
 443 /* Called with genl_mutex. */
 444 static int flush_flows(struct datapath *dp)
 445 {
 446         struct flow_table *old_table;
 447         struct flow_table *new_table;
 448
 449         old_table = genl_dereference(dp->table);
 450         new_table = ovs_flow_tbl_alloc(TBL_MIN_BUCKETS);
 451         if (!new_table)
 452                 return -ENOMEM;
 453
 454         rcu_assign_pointer(dp->table, new_table);
 455
 456         ovs_flow_tbl_deferred_destroy(old_table);
 457         return 0;
 458 }
 459
 460 static struct nlattr *reserve_sfa_size(struct sw_flow_actions **sfa, int attr_len)
 461 {
 462
 463         struct sw_flow_actions *acts;
 464         int new_acts_size;
 465         int req_size = NLA_ALIGN(attr_len);
 466         int next_offset = offsetof(struct sw_flow_actions, actions) +
 467                                         (*sfa)->actions_len;
 468
 469         if (req_size <= (ksize(*sfa) - next_offset))
 470                 goto out;
 471
 472         new_acts_size = ksize(*sfa) * 2;
 473
 474         if (new_acts_size > MAX_ACTIONS_BUFSIZE) {
 475                 if ((MAX_ACTIONS_BUFSIZE - next_offset) < req_size)
 476                         return ERR_PTR(-EMSGSIZE);
 477                 new_acts_size = MAX_ACTIONS_BUFSIZE;
 478         }
 479
 480         acts = ovs_flow_actions_alloc(new_acts_size);
 481         if (IS_ERR(acts))
 482                 return (void *)acts;
 483
 484         memcpy(acts->actions, (*sfa)->actions, (*sfa)->actions_len);
 485         acts->actions_len = (*sfa)->actions_len;
 486         kfree(*sfa);
 487         *sfa = acts;
 488
 489 out:
 490         (*sfa)->actions_len += req_size;
 491         return  (struct nlattr *) ((unsigned char *)(*sfa) + next_offset);
 492 }
 493
 494 static int add_action(struct sw_flow_actions **sfa, int attrtype, void *data, int len)
 495 {
 496         struct nlattr *a;
 497
 498         a = reserve_sfa_size(sfa, nla_attr_size(len));
 499         if (IS_ERR(a))
 500                 return PTR_ERR(a);
 501
 502         a->nla_type = attrtype;
 503         a->nla_len = nla_attr_size(len);
 504
 505         if (data)
 506                 memcpy(nla_data(a), data, len);
 507         memset((unsigned char *) a + a->nla_len, 0, nla_padlen(len));
 508
 509         return 0;
 510 }
 511
 512 static inline int add_nested_action_start(struct sw_flow_actions **sfa, int attrtype)
 513 {
 514         int used = (*sfa)->actions_len;
 515         int err;
 516
 517         err = add_action(sfa, attrtype, NULL, 0);
 518         if (err)
 519                 return err;
 520
 521         return used;
 522 }
 523
 524 static inline void add_nested_action_end(struct sw_flow_actions *sfa, int st_offset)
 525 {
 526         struct nlattr *a = (struct nlattr *) ((unsigned char *)sfa->actions + st_offset);
 527
 528         a->nla_len = sfa->actions_len - st_offset;
 529 }
 530
 531 static int validate_and_copy_actions(const struct nlattr *attr,
 532                                 const struct sw_flow_key *key, int depth,
 533                                 struct sw_flow_actions **sfa);
 534
 535 static int validate_and_copy_sample(const struct nlattr *attr,
 536                            const struct sw_flow_key *key, int depth,
 537                            struct sw_flow_actions **sfa)
 538 {
 539         const struct nlattr *attrs[OVS_SAMPLE_ATTR_MAX + 1];
 540         const struct nlattr *probability, *actions;
 541         const struct nlattr *a;
 542         int rem, start, err, st_acts;
 543
 544         memset(attrs, 0, sizeof(attrs));
 545         nla_for_each_nested(a, attr, rem) {
 546                 int type = nla_type(a);
 547                 if (!type || type > OVS_SAMPLE_ATTR_MAX || attrs[type])
 548                         return -EINVAL;
 549                 attrs[type] = a;
 550         }
 551         if (rem)
 552                 return -EINVAL;
 553
 554         probability = attrs[OVS_SAMPLE_ATTR_PROBABILITY];
 555         if (!probability || nla_len(probability) != sizeof(u32))
 556                 return -EINVAL;
 557
 558         actions = attrs[OVS_SAMPLE_ATTR_ACTIONS];
 559         if (!actions || (nla_len(actions) && nla_len(actions) < NLA_HDRLEN))
 560                 return -EINVAL;
 561
 562         /* validation done, copy sample action. */
 563         start = add_nested_action_start(sfa, OVS_ACTION_ATTR_SAMPLE);
 564         if (start < 0)
 565                 return start;
 566         err = add_action(sfa, OVS_SAMPLE_ATTR_PROBABILITY, nla_data(probability), sizeof(u32));
 567         if (err)
 568                 return err;
 569         st_acts = add_nested_action_start(sfa, OVS_SAMPLE_ATTR_ACTIONS);
 570         if (st_acts < 0)
 571                 return st_acts;
 572
 573         err = validate_and_copy_actions(actions, key, depth + 1, sfa);
 574         if (err)
 575                 return err;
 576
 577         add_nested_action_end(*sfa, st_acts);
 578         add_nested_action_end(*sfa, start);
 579
 580         return 0;
 581 }
 582
 583 static int validate_tp_port(const struct sw_flow_key *flow_key)
 584 {
 585         if (flow_key->eth.type == htons(ETH_P_IP)) {
 586                 if (flow_key->ipv4.tp.src || flow_key->ipv4.tp.dst)
 587                         return 0;
 588         } else if (flow_key->eth.type == htons(ETH_P_IPV6)) {
 589                 if (flow_key->ipv6.tp.src || flow_key->ipv6.tp.dst)
 590                         return 0;
 591         }
 592
 593         return -EINVAL;
 594 }
 595
 596 static int validate_and_copy_set_tun(const struct nlattr *attr,
 597                                      struct sw_flow_actions **sfa)
 598 {
 599         struct ovs_key_ipv4_tunnel tun_key;
 600         int err, start;
 601
 602         err = ipv4_tun_from_nlattr(nla_data(attr), &tun_key);
 603         if (err)
 604                 return err;
 605
 606         start = add_nested_action_start(sfa, OVS_ACTION_ATTR_SET);
 607         if (start < 0)
 608                 return start;
 609
 610         err = add_action(sfa, OVS_KEY_ATTR_IPV4_TUNNEL, &tun_key, sizeof(tun_key));
 611         add_nested_action_end(*sfa, start);
 612
 613         return err;
 614 }
 615
 616 static int validate_set(const struct nlattr *a,
 617                         const struct sw_flow_key *flow_key,
 618                         struct sw_flow_actions **sfa,
 619                         bool *set_tun)
 620 {
 621         const struct nlattr *ovs_key = nla_data(a);
 622         int key_type = nla_type(ovs_key);
 623
 624         /* There can be only one key in a action */
 625         if (nla_total_size(nla_len(ovs_key)) != nla_len(a))
 626                 return -EINVAL;
 627
 628         if (key_type > OVS_KEY_ATTR_MAX ||
 629             (ovs_key_lens[key_type] != nla_len(ovs_key) &&
 630              ovs_key_lens[key_type] != -1))
 631                 return -EINVAL;
 632
 633         switch (key_type) {
 634         const struct ovs_key_ipv4 *ipv4_key;
 635         const struct ovs_key_ipv6 *ipv6_key;
 636         int err;
 637
 638         case OVS_KEY_ATTR_PRIORITY:
 639         case OVS_KEY_ATTR_ETHERNET:
 640                 break;
 641
 642         case OVS_KEY_ATTR_SKB_MARK:
 643 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) && !defined(CONFIG_NETFILTER)
 644                 if (nla_get_u32(ovs_key) != 0)
 645                         return -EINVAL;
 646 #endif
 647                 break;
 648
 649         case OVS_KEY_ATTR_TUNNEL:
 650                 *set_tun = true;
 651                 err = validate_and_copy_set_tun(a, sfa);
 652                 if (err)
 653                         return err;
 654                 break;
 655
 656         case OVS_KEY_ATTR_IPV4:
 657                 if (flow_key->eth.type != htons(ETH_P_IP))
 658                         return -EINVAL;
 659
 660                 if (!flow_key->ip.proto)
 661                         return -EINVAL;
 662
 663                 ipv4_key = nla_data(ovs_key);
 664                 if (ipv4_key->ipv4_proto != flow_key->ip.proto)
 665                         return -EINVAL;
 666
 667                 if (ipv4_key->ipv4_frag != flow_key->ip.frag)
 668                         return -EINVAL;
 669
 670                 break;
 671
 672         case OVS_KEY_ATTR_IPV6:
 673                 if (flow_key->eth.type != htons(ETH_P_IPV6))
 674                         return -EINVAL;
 675
 676                 if (!flow_key->ip.proto)
 677                         return -EINVAL;
 678
 679                 ipv6_key = nla_data(ovs_key);
 680                 if (ipv6_key->ipv6_proto != flow_key->ip.proto)
 681                         return -EINVAL;
 682
 683                 if (ipv6_key->ipv6_frag != flow_key->ip.frag)
 684                         return -EINVAL;
 685
 686                 if (ntohl(ipv6_key->ipv6_label) & 0xFFF00000)
 687                         return -EINVAL;
 688
 689                 break;
 690
 691         case OVS_KEY_ATTR_TCP:
 692                 if (flow_key->ip.proto != IPPROTO_TCP)
 693                         return -EINVAL;
 694
 695                 return validate_tp_port(flow_key);
 696
 697         case OVS_KEY_ATTR_UDP:
 698                 if (flow_key->ip.proto != IPPROTO_UDP)
 699                         return -EINVAL;
 700
 701                 return validate_tp_port(flow_key);
 702
 703         default:
 704                 return -EINVAL;
 705         }
 706
 707         return 0;
 708 }
 709
 710 static int validate_userspace(const struct nlattr *attr)
 711 {
 712         static const struct nla_policy userspace_policy[OVS_USERSPACE_ATTR_MAX + 1] =   {
 713                 [OVS_USERSPACE_ATTR_PID] = {.type = NLA_U32 },
 714                 [OVS_USERSPACE_ATTR_USERDATA] = {.type = NLA_UNSPEC },
 715         };
 716         struct nlattr *a[OVS_USERSPACE_ATTR_MAX + 1];
 717         int error;
 718
 719         error = nla_parse_nested(a, OVS_USERSPACE_ATTR_MAX,
 720                                  attr, userspace_policy);
 721         if (error)
 722                 return error;
 723
 724         if (!a[OVS_USERSPACE_ATTR_PID] ||
 725             !nla_get_u32(a[OVS_USERSPACE_ATTR_PID]))
 726                 return -EINVAL;
 727
 728         return 0;
 729 }
 730
 731 static int copy_action(const struct nlattr *from,
 732                       struct sw_flow_actions **sfa)
 733 {
 734         int totlen = NLA_ALIGN(from->nla_len);
 735         struct nlattr *to;
 736
 737         to = reserve_sfa_size(sfa, from->nla_len);
 738         if (IS_ERR(to))
 739                 return PTR_ERR(to);
 740
 741         memcpy(to, from, totlen);
 742         return 0;
 743 }
 744
 745 static int validate_and_copy_actions(const struct nlattr *attr,
 746                                 const struct sw_flow_key *key,
 747                                 int depth,
 748                                 struct sw_flow_actions **sfa)
 749 {
 750         const struct nlattr *a;
 751         int rem, err;
 752
 753         if (depth >= SAMPLE_ACTION_DEPTH)
 754                 return -EOVERFLOW;
 755
 756         nla_for_each_nested(a, attr, rem) {
 757                 /* Expected argument lengths, (u32)-1 for variable length. */
 758                 static const u32 action_lens[OVS_ACTION_ATTR_MAX + 1] = {
 759                         [OVS_ACTION_ATTR_OUTPUT] = sizeof(u32),
 760                         [OVS_ACTION_ATTR_USERSPACE] = (u32)-1,
 761                         [OVS_ACTION_ATTR_PUSH_VLAN] = sizeof(struct ovs_action_push_vlan),
 762                         [OVS_ACTION_ATTR_POP_VLAN] = 0,
 763                         [OVS_ACTION_ATTR_SET] = (u32)-1,
 764                         [OVS_ACTION_ATTR_SAMPLE] = (u32)-1
 765                 };
 766                 const struct ovs_action_push_vlan *vlan;
 767                 int type = nla_type(a);
 768                 bool skip_copy;
 769
 770                 if (type > OVS_ACTION_ATTR_MAX ||
 771                     (action_lens[type] != nla_len(a) &&
 772                      action_lens[type] != (u32)-1))
 773                         return -EINVAL;
 774
 775                 skip_copy = false;
 776                 switch (type) {
 777                 case OVS_ACTION_ATTR_UNSPEC:
 778                         return -EINVAL;
 779
 780                 case OVS_ACTION_ATTR_USERSPACE:
 781                         err = validate_userspace(a);
 782                         if (err)
 783                                 return err;
 784                         break;
 785
 786                 case OVS_ACTION_ATTR_OUTPUT:
 787                         if (nla_get_u32(a) >= DP_MAX_PORTS)
 788                                 return -EINVAL;
 789                         break;
 790
 791
 792                 case OVS_ACTION_ATTR_POP_VLAN:
 793                         break;
 794
 795                 case OVS_ACTION_ATTR_PUSH_VLAN:
 796                         vlan = nla_data(a);
 797                         if (vlan->vlan_tpid != htons(ETH_P_8021Q))
 798                                 return -EINVAL;
 799                         if (!(vlan->vlan_tci & htons(VLAN_TAG_PRESENT)))
 800                                 return -EINVAL;
 801                         break;
 802
 803                 case OVS_ACTION_ATTR_SET:
 804                         err = validate_set(a, key, sfa, &skip_copy);
 805                         if (err)
 806                                 return err;
 807                         break;
 808
 809                 case OVS_ACTION_ATTR_SAMPLE:
 810                         err = validate_and_copy_sample(a, key, depth, sfa);
 811                         if (err)
 812                                 return err;
 813                         skip_copy = true;
 814                         break;
 815
 816                 default:
 817                         return -EINVAL;
 818                 }
 819                 if (!skip_copy) {
 820                         err = copy_action(a, sfa);
 821                         if (err)
 822                                 return err;
 823                 }
 824         }
 825
 826         if (rem > 0)
 827                 return -EINVAL;
 828
 829         return 0;
 830 }
 831
 832 static void clear_stats(struct sw_flow *flow)
 833 {
 834         flow->used = 0;
 835         flow->tcp_flags = 0;
 836         flow->packet_count = 0;
 837         flow->byte_count = 0;
 838 }
 839
 840 static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
 841 {
 842         struct ovs_header *ovs_header = info->userhdr;
 843         struct nlattr **a = info->attrs;
 844         struct sw_flow_actions *acts;
 845         struct sk_buff *packet;
 846         struct sw_flow *flow;
 847         struct datapath *dp;
 848         struct ethhdr *eth;
 849         int len;
 850         int err;
 851         int key_len;
 852
 853         err = -EINVAL;
 854         if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] ||
 855             !a[OVS_PACKET_ATTR_ACTIONS])
 856                 goto err;
 857
 858         len = nla_len(a[OVS_PACKET_ATTR_PACKET]);
 859         packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL);
 860         err = -ENOMEM;
 861         if (!packet)
 862                 goto err;
 863         skb_reserve(packet, NET_IP_ALIGN);
 864
 865         nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len);
 866
 867         skb_reset_mac_header(packet);
 868         eth = eth_hdr(packet);
 869
 870         /* Normally, setting the skb 'protocol' field would be handled by a
 871          * call to eth_type_trans(), but it assumes there's a sending
 872          * device, which we may not have. */
 873         if (ntohs(eth->h_proto) >= ETH_P_802_3_MIN)
 874                 packet->protocol = eth->h_proto;
 875         else
 876                 packet->protocol = htons(ETH_P_802_2);
 877
 878         /* Build an sw_flow for sending this packet. */
 879         flow = ovs_flow_alloc();
 880         err = PTR_ERR(flow);
 881         if (IS_ERR(flow))
 882                 goto err_kfree_skb;
 883
 884         err = ovs_flow_extract(packet, -1, &flow->key, &key_len);
 885         if (err)
 886                 goto err_flow_free;
 887
 888         err = ovs_flow_metadata_from_nlattrs(flow, key_len, a[OVS_PACKET_ATTR_KEY]);
 889         if (err)
 890                 goto err_flow_free;
 891         acts = ovs_flow_actions_alloc(nla_len(a[OVS_PACKET_ATTR_ACTIONS]));
 892         err = PTR_ERR(acts);
 893         if (IS_ERR(acts))
 894                 goto err_flow_free;
 895
 896         err = validate_and_copy_actions(a[OVS_PACKET_ATTR_ACTIONS], &flow->key, 0, &acts);
 897         rcu_assign_pointer(flow->sf_acts, acts);
 898         if (err)
 899                 goto err_flow_free;
 900
 901         OVS_CB(packet)->flow = flow;
 902         packet->priority = flow->key.phy.priority;
 903         skb_set_mark(packet, flow->key.phy.skb_mark);
 904
 905         rcu_read_lock();
 906         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
 907         err = -ENODEV;
 908         if (!dp)
 909                 goto err_unlock;
 910
 911         local_bh_disable();
 912         err = ovs_execute_actions(dp, packet);
 913         local_bh_enable();
 914         rcu_read_unlock();
 915
 916         ovs_flow_free(flow);
 917         return err;
 918
 919 err_unlock:
 920         rcu_read_unlock();
 921 err_flow_free:
 922         ovs_flow_free(flow);
 923 err_kfree_skb:
 924         kfree_skb(packet);
 925 err:
 926         return err;
 927 }
 928
 929 static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = {
 930 #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,18)
 931         [OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN },
 932 #else
 933         [OVS_PACKET_ATTR_PACKET] = { .minlen = ETH_HLEN },
 934 #endif
 935         [OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED },
 936         [OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED },
 937 };
 938
 939 static struct genl_ops dp_packet_genl_ops[] = {
 940         { .cmd = OVS_PACKET_CMD_EXECUTE,
 941           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
 942           .policy = packet_policy,
 943           .doit = ovs_packet_cmd_execute
 944         }
 945 };
 946
 947 static void get_dp_stats(struct datapath *dp, struct ovs_dp_stats *stats)
 948 {
 949         int i;
 950         struct flow_table *table = genl_dereference(dp->table);
 951
 952         stats->n_flows = ovs_flow_tbl_count(table);
 953
 954         stats->n_hit = stats->n_missed = stats->n_lost = 0;
 955         for_each_possible_cpu(i) {
 956                 const struct dp_stats_percpu *percpu_stats;
 957                 struct dp_stats_percpu local_stats;
 958                 unsigned int start;
 959
 960                 percpu_stats = per_cpu_ptr(dp->stats_percpu, i);
 961
 962                 do {
 963                         start = u64_stats_fetch_begin_bh(&percpu_stats->sync);
 964                         local_stats = *percpu_stats;
 965                 } while (u64_stats_fetch_retry_bh(&percpu_stats->sync, start));
 966
 967                 stats->n_hit += local_stats.n_hit;
 968                 stats->n_missed += local_stats.n_missed;
 969                 stats->n_lost += local_stats.n_lost;
 970         }
 971 }
 972
 973 static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = {
 974         [OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
 975         [OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
 976         [OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
 977 };
 978
 979 static struct genl_family dp_flow_genl_family = {
 980         .id = GENL_ID_GENERATE,
 981         .hdrsize = sizeof(struct ovs_header),
 982         .name = OVS_FLOW_FAMILY,
 983         .version = OVS_FLOW_VERSION,
 984         .maxattr = OVS_FLOW_ATTR_MAX,
 985          SET_NETNSOK
 986 };
 987
 988 static struct genl_multicast_group ovs_dp_flow_multicast_group = {
 989         .name = OVS_FLOW_MCGROUP
 990 };
 991
 992 static int actions_to_attr(const struct nlattr *attr, int len, struct sk_buff *skb);
 993 static int sample_action_to_attr(const struct nlattr *attr, struct sk_buff *skb)
 994 {
 995         const struct nlattr *a;
 996         struct nlattr *start;
 997         int err = 0, rem;
 998
 999         start = nla_nest_start(skb, OVS_ACTION_ATTR_SAMPLE);
1000         if (!start)
1001                 return -EMSGSIZE;
1002
1003         nla_for_each_nested(a, attr, rem) {
1004                 int type = nla_type(a);
1005                 struct nlattr *st_sample;
1006
1007                 switch (type) {
1008                 case OVS_SAMPLE_ATTR_PROBABILITY:
1009                         if (nla_put(skb, OVS_SAMPLE_ATTR_PROBABILITY, sizeof(u32), nla_data(a)))
1010                                 return -EMSGSIZE;
1011                         break;
1012                 case OVS_SAMPLE_ATTR_ACTIONS:
1013                         st_sample = nla_nest_start(skb, OVS_SAMPLE_ATTR_ACTIONS);
1014                         if (!st_sample)
1015                                 return -EMSGSIZE;
1016                         err = actions_to_attr(nla_data(a), nla_len(a), skb);
1017                         if (err)
1018                                 return err;
1019                         nla_nest_end(skb, st_sample);
1020                         break;
1021                 }
1022         }
1023
1024         nla_nest_end(skb, start);
1025         return err;
1026 }
1027
1028 static int set_action_to_attr(const struct nlattr *a, struct sk_buff *skb)
1029 {
1030         const struct nlattr *ovs_key = nla_data(a);
1031         int key_type = nla_type(ovs_key);
1032         struct nlattr *start;
1033         int err;
1034
1035         switch (key_type) {
1036         case OVS_KEY_ATTR_IPV4_TUNNEL:
1037                 start = nla_nest_start(skb, OVS_ACTION_ATTR_SET);
1038                 if (!start)
1039                         return -EMSGSIZE;
1040
1041                 err = ipv4_tun_to_nlattr(skb, nla_data(ovs_key));
1042                 if (err)
1043                         return err;
1044                 nla_nest_end(skb, start);
1045                 break;
1046         default:
1047                 if (nla_put(skb, OVS_ACTION_ATTR_SET, nla_len(a), ovs_key))
1048                         return -EMSGSIZE;
1049                 break;
1050         }
1051
1052         return 0;
1053 }
1054
1055 static int actions_to_attr(const struct nlattr *attr, int len, struct sk_buff *skb)
1056 {
1057         const struct nlattr *a;
1058         int rem, err;
1059
1060         nla_for_each_attr(a, attr, len, rem) {
1061                 int type = nla_type(a);
1062
1063                 switch (type) {
1064                 case OVS_ACTION_ATTR_SET:
1065                         err = set_action_to_attr(a, skb);
1066                         if (err)
1067                                 return err;
1068                         break;
1069
1070                 case OVS_ACTION_ATTR_SAMPLE:
1071                         err = sample_action_to_attr(a, skb);
1072                         if (err)
1073                                 return err;
1074                         break;
1075                 default:
1076                         if (nla_put(skb, type, nla_len(a), nla_data(a)))
1077                                 return -EMSGSIZE;
1078                         break;
1079                 }
1080         }
1081
1082         return 0;
1083 }
1084
1085 static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts)
1086 {
1087         return NLMSG_ALIGN(sizeof(struct ovs_header))
1088                 + nla_total_size(key_attr_size()) /* OVS_FLOW_ATTR_KEY */
1089                 + nla_total_size(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */
1090                 + nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */
1091                 + nla_total_size(8) /* OVS_FLOW_ATTR_USED */
1092                 + nla_total_size(acts->actions_len); /* OVS_FLOW_ATTR_ACTIONS */
1093 }
1094
1095 /* Called with genl_lock. */
1096 static int ovs_flow_cmd_fill_info(struct sw_flow *flow, struct datapath *dp,
1097                                   struct sk_buff *skb, u32 portid,
1098                                   u32 seq, u32 flags, u8 cmd)
1099 {
1100         const int skb_orig_len = skb->len;
1101         const struct sw_flow_actions *sf_acts;
1102         struct nlattr *start;
1103         struct ovs_flow_stats stats;
1104         struct ovs_header *ovs_header;
1105         struct nlattr *nla;
1106         unsigned long used;
1107         u8 tcp_flags;
1108         int err;
1109
1110         sf_acts = rcu_dereference_protected(flow->sf_acts,
1111                                             lockdep_genl_is_held());
1112
1113         ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family, flags, cmd);
1114         if (!ovs_header)
1115                 return -EMSGSIZE;
1116
1117         ovs_header->dp_ifindex = get_dpifindex(dp);
1118
1119         nla = nla_nest_start(skb, OVS_FLOW_ATTR_KEY);
1120         if (!nla)
1121                 goto nla_put_failure;
1122         err = ovs_flow_to_nlattrs(&flow->key, skb);
1123         if (err)
1124                 goto error;
1125         nla_nest_end(skb, nla);
1126
1127         spin_lock_bh(&flow->lock);
1128         used = flow->used;
1129         stats.n_packets = flow->packet_count;
1130         stats.n_bytes = flow->byte_count;
1131         tcp_flags = flow->tcp_flags;
1132         spin_unlock_bh(&flow->lock);
1133
1134         if (used &&
1135             nla_put_u64(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used)))
1136                 goto nla_put_failure;
1137
1138         if (stats.n_packets &&
1139             nla_put(skb, OVS_FLOW_ATTR_STATS,
1140                     sizeof(struct ovs_flow_stats), &stats))
1141                 goto nla_put_failure;
1142
1143         if (tcp_flags &&
1144             nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, tcp_flags))
1145                 goto nla_put_failure;
1146
1147         /* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
1148          * this is the first flow to be dumped into 'skb'.  This is unusual for
1149          * Netlink but individual action lists can be longer than
1150          * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this.
1151          * The userspace caller can always fetch the actions separately if it
1152          * really wants them.  (Most userspace callers in fact don't care.)
1153          *
1154          * This can only fail for dump operations because the skb is always
1155          * properly sized for single flows.
1156          */
1157         start = nla_nest_start(skb, OVS_FLOW_ATTR_ACTIONS);
1158         if (start) {
1159                 err = actions_to_attr(sf_acts->actions, sf_acts->actions_len, skb);
1160                 if (!err)
1161                         nla_nest_end(skb, start);
1162                 else {
1163                         if (skb_orig_len)
1164                                 goto error;
1165
1166                         nla_nest_cancel(skb, start);
1167                 }
1168         } else if (skb_orig_len)
1169                 goto nla_put_failure;
1170
1171         return genlmsg_end(skb, ovs_header);
1172
1173 nla_put_failure:
1174         err = -EMSGSIZE;
1175 error:
1176         genlmsg_cancel(skb, ovs_header);
1177         return err;
1178 }
1179
1180 static struct sk_buff *ovs_flow_cmd_alloc_info(struct sw_flow *flow)
1181 {
1182         const struct sw_flow_actions *sf_acts;
1183
1184         sf_acts = rcu_dereference_protected(flow->sf_acts,
1185                                             lockdep_genl_is_held());
1186
1187         return genlmsg_new(ovs_flow_cmd_msg_size(sf_acts), GFP_KERNEL);
1188 }
1189
1190 static struct sk_buff *ovs_flow_cmd_build_info(struct sw_flow *flow,
1191                                                struct datapath *dp,
1192                                                u32 portid, u32 seq, u8 cmd)
1193 {
1194         struct sk_buff *skb;
1195         int retval;
1196
1197         skb = ovs_flow_cmd_alloc_info(flow);
1198         if (!skb)
1199                 return ERR_PTR(-ENOMEM);
1200
1201         retval = ovs_flow_cmd_fill_info(flow, dp, skb, portid, seq, 0, cmd);
1202         BUG_ON(retval < 0);
1203         return skb;
1204 }
1205
1206 static int ovs_flow_cmd_new_or_set(struct sk_buff *skb, struct genl_info *info)
1207 {
1208         struct nlattr **a = info->attrs;
1209         struct ovs_header *ovs_header = info->userhdr;
1210         struct sw_flow_key key;
1211         struct sw_flow *flow;
1212         struct sk_buff *reply;
1213         struct datapath *dp;
1214         struct flow_table *table;
1215         struct sw_flow_actions *acts = NULL;
1216         int error;
1217         int key_len;
1218
1219         /* Extract key. */
1220         error = -EINVAL;
1221         if (!a[OVS_FLOW_ATTR_KEY])
1222                 goto error;
1223         error = ovs_flow_from_nlattrs(&key, &key_len, a[OVS_FLOW_ATTR_KEY]);
1224         if (error)
1225                 goto error;
1226
1227         /* Validate actions. */
1228         if (a[OVS_FLOW_ATTR_ACTIONS]) {
1229                 acts = ovs_flow_actions_alloc(nla_len(a[OVS_FLOW_ATTR_ACTIONS]));
1230                 error = PTR_ERR(acts);
1231                 if (IS_ERR(acts))
1232                         goto error;
1233
1234                 error = validate_and_copy_actions(a[OVS_FLOW_ATTR_ACTIONS], &key,  0, &acts);
1235                 if (error)
1236                         goto err_kfree;
1237         } else if (info->genlhdr->cmd == OVS_FLOW_CMD_NEW) {
1238                 error = -EINVAL;
1239                 goto error;
1240         }
1241
1242         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1243         error = -ENODEV;
1244         if (!dp)
1245                 goto err_kfree;
1246
1247         table = genl_dereference(dp->table);
1248         flow = ovs_flow_tbl_lookup(table, &key, key_len);
1249         if (!flow) {
1250                 /* Bail out if we're not allowed to create a new flow. */
1251                 error = -ENOENT;
1252                 if (info->genlhdr->cmd == OVS_FLOW_CMD_SET)
1253                         goto err_kfree;
1254
1255                 /* Expand table, if necessary, to make room. */
1256                 if (ovs_flow_tbl_need_to_expand(table)) {
1257                         struct flow_table *new_table;
1258
1259                         new_table = ovs_flow_tbl_expand(table);
1260                         if (!IS_ERR(new_table)) {
1261                                 rcu_assign_pointer(dp->table, new_table);
1262                                 ovs_flow_tbl_deferred_destroy(table);
1263                                 table = genl_dereference(dp->table);
1264                         }
1265                 }
1266
1267                 /* Allocate flow. */
1268                 flow = ovs_flow_alloc();
1269                 if (IS_ERR(flow)) {
1270                         error = PTR_ERR(flow);
1271                         goto err_kfree;
1272                 }
1273                 clear_stats(flow);
1274
1275                 rcu_assign_pointer(flow->sf_acts, acts);
1276
1277                 /* Put flow in bucket. */
1278                 ovs_flow_tbl_insert(table, flow, &key, key_len);
1279
1280                 reply = ovs_flow_cmd_build_info(flow, dp, info->snd_portid,
1281                                                 info->snd_seq,
1282                                                 OVS_FLOW_CMD_NEW);
1283         } else {
1284                 /* We found a matching flow. */
1285                 struct sw_flow_actions *old_acts;
1286
1287                 /* Bail out if we're not allowed to modify an existing flow.
1288                  * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
1289                  * because Generic Netlink treats the latter as a dump
1290                  * request.  We also accept NLM_F_EXCL in case that bug ever
1291                  * gets fixed.
1292                  */
1293                 error = -EEXIST;
1294                 if (info->genlhdr->cmd == OVS_FLOW_CMD_NEW &&
1295                     info->nlhdr->nlmsg_flags & (NLM_F_CREATE | NLM_F_EXCL))
1296                         goto err_kfree;
1297
1298                 /* Update actions. */
1299                 old_acts = rcu_dereference_protected(flow->sf_acts,
1300                                                      lockdep_genl_is_held());
1301                 rcu_assign_pointer(flow->sf_acts, acts);
1302                 ovs_flow_deferred_free_acts(old_acts);
1303
1304                 reply = ovs_flow_cmd_build_info(flow, dp, info->snd_portid,
1305                                                info->snd_seq, OVS_FLOW_CMD_NEW);
1306
1307                 /* Clear stats. */
1308                 if (a[OVS_FLOW_ATTR_CLEAR]) {
1309                         spin_lock_bh(&flow->lock);
1310                         clear_stats(flow);
1311                         spin_unlock_bh(&flow->lock);
1312                 }
1313         }
1314
1315         if (!IS_ERR(reply))
1316                 ovs_notify(reply, info, &ovs_dp_flow_multicast_group);
1317         else
1318                 netlink_set_err(GENL_SOCK(sock_net(skb->sk)), 0,
1319                                 ovs_dp_flow_multicast_group.id, PTR_ERR(reply));
1320         return 0;
1321
1322 err_kfree:
1323         kfree(acts);
1324 error:
1325         return error;
1326 }
1327
1328 static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
1329 {
1330         struct nlattr **a = info->attrs;
1331         struct ovs_header *ovs_header = info->userhdr;
1332         struct sw_flow_key key;
1333         struct sk_buff *reply;
1334         struct sw_flow *flow;
1335         struct datapath *dp;
1336         struct flow_table *table;
1337         int err;
1338         int key_len;
1339
1340         if (!a[OVS_FLOW_ATTR_KEY])
1341                 return -EINVAL;
1342         err = ovs_flow_from_nlattrs(&key, &key_len, a[OVS_FLOW_ATTR_KEY]);
1343         if (err)
1344                 return err;
1345
1346         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1347         if (!dp)
1348                 return -ENODEV;
1349
1350         table = genl_dereference(dp->table);
1351         flow = ovs_flow_tbl_lookup(table, &key, key_len);
1352         if (!flow)
1353                 return -ENOENT;
1354
1355         reply = ovs_flow_cmd_build_info(flow, dp, info->snd_portid,
1356                                         info->snd_seq, OVS_FLOW_CMD_NEW);
1357         if (IS_ERR(reply))
1358                 return PTR_ERR(reply);
1359
1360         return genlmsg_reply(reply, info);
1361 }
1362
1363 static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
1364 {
1365         struct nlattr **a = info->attrs;
1366         struct ovs_header *ovs_header = info->userhdr;
1367         struct sw_flow_key key;
1368         struct sk_buff *reply;
1369         struct sw_flow *flow;
1370         struct datapath *dp;
1371         struct flow_table *table;
1372         int err;
1373         int key_len;
1374
1375         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1376         if (!dp)
1377                 return -ENODEV;
1378
1379         if (!a[OVS_FLOW_ATTR_KEY])
1380                 return flush_flows(dp);
1381
1382         err = ovs_flow_from_nlattrs(&key, &key_len, a[OVS_FLOW_ATTR_KEY]);
1383         if (err)
1384                 return err;
1385
1386         table = genl_dereference(dp->table);
1387         flow = ovs_flow_tbl_lookup(table, &key, key_len);
1388         if (!flow)
1389                 return -ENOENT;
1390
1391         reply = ovs_flow_cmd_alloc_info(flow);
1392         if (!reply)
1393                 return -ENOMEM;
1394
1395         ovs_flow_tbl_remove(table, flow);
1396
1397         err = ovs_flow_cmd_fill_info(flow, dp, reply, info->snd_portid,
1398                                      info->snd_seq, 0, OVS_FLOW_CMD_DEL);
1399         BUG_ON(err < 0);
1400
1401         ovs_flow_deferred_free(flow);
1402
1403         ovs_notify(reply, info, &ovs_dp_flow_multicast_group);
1404         return 0;
1405 }
1406
1407 static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1408 {
1409         struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
1410         struct datapath *dp;
1411         struct flow_table *table;
1412
1413         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1414         if (!dp)
1415                 return -ENODEV;
1416
1417         table = genl_dereference(dp->table);
1418
1419         for (;;) {
1420                 struct sw_flow *flow;
1421                 u32 bucket, obj;
1422
1423                 bucket = cb->args[0];
1424                 obj = cb->args[1];
1425                 flow = ovs_flow_tbl_next(table, &bucket, &obj);
1426                 if (!flow)
1427                         break;
1428
1429                 if (ovs_flow_cmd_fill_info(flow, dp, skb,
1430                                            NETLINK_CB(cb->skb).portid,
1431                                            cb->nlh->nlmsg_seq, NLM_F_MULTI,
1432                                            OVS_FLOW_CMD_NEW) < 0)
1433                         break;
1434
1435                 cb->args[0] = bucket;
1436                 cb->args[1] = obj;
1437         }
1438         return skb->len;
1439 }
1440
1441 static struct genl_ops dp_flow_genl_ops[] = {
1442         { .cmd = OVS_FLOW_CMD_NEW,
1443           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1444           .policy = flow_policy,
1445           .doit = ovs_flow_cmd_new_or_set
1446         },
1447         { .cmd = OVS_FLOW_CMD_DEL,
1448           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1449           .policy = flow_policy,
1450           .doit = ovs_flow_cmd_del
1451         },
1452         { .cmd = OVS_FLOW_CMD_GET,
1453           .flags = 0,               /* OK for unprivileged users. */
1454           .policy = flow_policy,
1455           .doit = ovs_flow_cmd_get,
1456           .dumpit = ovs_flow_cmd_dump
1457         },
1458         { .cmd = OVS_FLOW_CMD_SET,
1459           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1460           .policy = flow_policy,
1461           .doit = ovs_flow_cmd_new_or_set,
1462         },
1463 };
1464
1465 static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
1466 #ifdef HAVE_NLA_NUL_STRING
1467         [OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
1468 #endif
1469         [OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
1470 };
1471
1472 static struct genl_family dp_datapath_genl_family = {
1473         .id = GENL_ID_GENERATE,
1474         .hdrsize = sizeof(struct ovs_header),
1475         .name = OVS_DATAPATH_FAMILY,
1476         .version = OVS_DATAPATH_VERSION,
1477         .maxattr = OVS_DP_ATTR_MAX,
1478          SET_NETNSOK
1479 };
1480
1481 static struct genl_multicast_group ovs_dp_datapath_multicast_group = {
1482         .name = OVS_DATAPATH_MCGROUP
1483 };
1484
1485 static size_t ovs_dp_cmd_msg_size(void)
1486 {
1487         size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));
1488
1489         msgsize += nla_total_size(IFNAMSIZ);
1490         msgsize += nla_total_size(sizeof(struct ovs_dp_stats));
1491
1492         return msgsize;
1493 }
1494
1495 static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
1496                                 u32 portid, u32 seq, u32 flags, u8 cmd)
1497 {
1498         struct ovs_header *ovs_header;
1499         struct ovs_dp_stats dp_stats;
1500         int err;
1501
1502         ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
1503                                    flags, cmd);
1504         if (!ovs_header)
1505                 goto error;
1506
1507         ovs_header->dp_ifindex = get_dpifindex(dp);
1508
1509         rcu_read_lock();
1510         err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp));
1511         rcu_read_unlock();
1512         if (err)
1513                 goto nla_put_failure;
1514
1515         get_dp_stats(dp, &dp_stats);
1516         if (nla_put(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats), &dp_stats))
1517                 goto nla_put_failure;
1518
1519         return genlmsg_end(skb, ovs_header);
1520
1521 nla_put_failure:
1522         genlmsg_cancel(skb, ovs_header);
1523 error:
1524         return -EMSGSIZE;
1525 }
1526
1527 static struct sk_buff *ovs_dp_cmd_build_info(struct datapath *dp, u32 portid,
1528                                              u32 seq, u8 cmd)
1529 {
1530         struct sk_buff *skb;
1531         int retval;
1532
1533         skb = genlmsg_new(ovs_dp_cmd_msg_size(), GFP_KERNEL);
1534         if (!skb)
1535                 return ERR_PTR(-ENOMEM);
1536
1537         retval = ovs_dp_cmd_fill_info(dp, skb, portid, seq, 0, cmd);
1538         if (retval < 0) {
1539                 kfree_skb(skb);
1540                 return ERR_PTR(retval);
1541         }
1542         return skb;
1543 }
1544
1545 static int ovs_dp_cmd_validate(struct nlattr *a[OVS_DP_ATTR_MAX + 1])
1546 {
1547         return CHECK_NUL_STRING(a[OVS_DP_ATTR_NAME], IFNAMSIZ - 1);
1548 }
1549
1550 /* Called with genl_mutex and optionally with RTNL lock also. */
1551 static struct datapath *lookup_datapath(struct net *net,
1552                                         struct ovs_header *ovs_header,
1553                                         struct nlattr *a[OVS_DP_ATTR_MAX + 1])
1554 {
1555         struct datapath *dp;
1556
1557         if (!a[OVS_DP_ATTR_NAME])
1558                 dp = get_dp(net, ovs_header->dp_ifindex);
1559         else {
1560                 struct vport *vport;
1561
1562                 rcu_read_lock();
1563                 vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME]));
1564                 dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL;
1565                 rcu_read_unlock();
1566         }
1567         return dp ? dp : ERR_PTR(-ENODEV);
1568 }
1569
1570 static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
1571 {
1572         struct nlattr **a = info->attrs;
1573         struct vport_parms parms;
1574         struct sk_buff *reply;
1575         struct datapath *dp;
1576         struct vport *vport;
1577         struct ovs_net *ovs_net;
1578         int err, i;
1579
1580         err = -EINVAL;
1581         if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID])
1582                 goto err;
1583
1584         err = ovs_dp_cmd_validate(a);
1585         if (err)
1586                 goto err;
1587
1588         rtnl_lock();
1589
1590         err = -ENOMEM;
1591         dp = kzalloc(sizeof(*dp), GFP_KERNEL);
1592         if (dp == NULL)
1593                 goto err_unlock_rtnl;
1594
1595         ovs_dp_set_net(dp, hold_net(sock_net(skb->sk)));
1596
1597         /* Allocate table. */
1598         err = -ENOMEM;
1599         rcu_assign_pointer(dp->table, ovs_flow_tbl_alloc(TBL_MIN_BUCKETS));
1600         if (!dp->table)
1601                 goto err_free_dp;
1602
1603         dp->stats_percpu = alloc_percpu(struct dp_stats_percpu);
1604         if (!dp->stats_percpu) {
1605                 err = -ENOMEM;
1606                 goto err_destroy_table;
1607         }
1608
1609         dp->ports = kmalloc(DP_VPORT_HASH_BUCKETS * sizeof(struct hlist_head),
1610                             GFP_KERNEL);
1611         if (!dp->ports) {
1612                 err = -ENOMEM;
1613                 goto err_destroy_percpu;
1614         }
1615
1616         for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
1617                 INIT_HLIST_HEAD(&dp->ports[i]);
1618
1619         /* Set up our datapath device. */
1620         parms.name = nla_data(a[OVS_DP_ATTR_NAME]);
1621         parms.type = OVS_VPORT_TYPE_INTERNAL;
1622         parms.options = NULL;
1623         parms.dp = dp;
1624         parms.port_no = OVSP_LOCAL;
1625         parms.upcall_portid = nla_get_u32(a[OVS_DP_ATTR_UPCALL_PID]);
1626
1627         vport = new_vport(&parms);
1628         if (IS_ERR(vport)) {
1629                 err = PTR_ERR(vport);
1630                 if (err == -EBUSY)
1631                         err = -EEXIST;
1632
1633                 goto err_destroy_ports_array;
1634         }
1635
1636         reply = ovs_dp_cmd_build_info(dp, info->snd_portid,
1637                                       info->snd_seq, OVS_DP_CMD_NEW);
1638         err = PTR_ERR(reply);
1639         if (IS_ERR(reply))
1640                 goto err_destroy_local_port;
1641
1642         ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);
1643         list_add_tail(&dp->list_node, &ovs_net->dps);
1644
1645         rtnl_unlock();
1646
1647         ovs_notify(reply, info, &ovs_dp_datapath_multicast_group);
1648         return 0;
1649
1650 err_destroy_local_port:
1651         ovs_dp_detach_port(ovs_vport_rtnl(dp, OVSP_LOCAL));
1652 err_destroy_ports_array:
1653         kfree(dp->ports);
1654 err_destroy_percpu:
1655         free_percpu(dp->stats_percpu);
1656 err_destroy_table:
1657         ovs_flow_tbl_destroy(genl_dereference(dp->table));
1658 err_free_dp:
1659         release_net(ovs_dp_get_net(dp));
1660         kfree(dp);
1661 err_unlock_rtnl:
1662         rtnl_unlock();
1663 err:
1664         return err;
1665 }
1666
1667 /* Called with genl_mutex. */
1668 static void __dp_destroy(struct datapath *dp)
1669 {
1670         int i;
1671
1672         rtnl_lock();
1673
1674         for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
1675                 struct vport *vport;
1676                 struct hlist_node *n;
1677
1678                 hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node)
1679                         if (vport->port_no != OVSP_LOCAL)
1680                                 ovs_dp_detach_port(vport);
1681         }
1682
1683         list_del(&dp->list_node);
1684         ovs_dp_detach_port(ovs_vport_rtnl(dp, OVSP_LOCAL));
1685
1686         /* rtnl_unlock() will wait until all the references to devices that
1687          * are pending unregistration have been dropped.  We do it here to
1688          * ensure that any internal devices (which contain DP pointers) are
1689          * fully destroyed before freeing the datapath.
1690          */
1691         rtnl_unlock();
1692
1693         call_rcu(&dp->rcu, destroy_dp_rcu);
1694 }
1695
1696 static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info)
1697 {
1698         struct sk_buff *reply;
1699         struct datapath *dp;
1700         int err;
1701
1702         err = ovs_dp_cmd_validate(info->attrs);
1703         if (err)
1704                 return err;
1705
1706         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1707         err = PTR_ERR(dp);
1708         if (IS_ERR(dp))
1709                 return err;
1710
1711         reply = ovs_dp_cmd_build_info(dp, info->snd_portid,
1712                                       info->snd_seq, OVS_DP_CMD_DEL);
1713         err = PTR_ERR(reply);
1714         if (IS_ERR(reply))
1715                 return err;
1716
1717         __dp_destroy(dp);
1718
1719         ovs_notify(reply, info, &ovs_dp_datapath_multicast_group);
1720
1721         return 0;
1722 }
1723
1724 static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info)
1725 {
1726         struct sk_buff *reply;
1727         struct datapath *dp;
1728         int err;
1729
1730         err = ovs_dp_cmd_validate(info->attrs);
1731         if (err)
1732                 return err;
1733
1734         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1735         if (IS_ERR(dp))
1736                 return PTR_ERR(dp);
1737
1738         reply = ovs_dp_cmd_build_info(dp, info->snd_portid,
1739                                       info->snd_seq, OVS_DP_CMD_NEW);
1740         if (IS_ERR(reply)) {
1741                 err = PTR_ERR(reply);
1742                 netlink_set_err(GENL_SOCK(sock_net(skb->sk)), 0,
1743                                 ovs_dp_datapath_multicast_group.id, err);
1744                 return 0;
1745         }
1746
1747         ovs_notify(reply, info, &ovs_dp_datapath_multicast_group);
1748
1749         return 0;
1750 }
1751
1752 static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info)
1753 {
1754         struct sk_buff *reply;
1755         struct datapath *dp;
1756         int err;
1757
1758         err = ovs_dp_cmd_validate(info->attrs);
1759         if (err)
1760                 return err;
1761
1762         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1763         if (IS_ERR(dp))
1764                 return PTR_ERR(dp);
1765
1766         reply = ovs_dp_cmd_build_info(dp, info->snd_portid,
1767                                       info->snd_seq, OVS_DP_CMD_NEW);
1768         if (IS_ERR(reply))
1769                 return PTR_ERR(reply);
1770
1771         return genlmsg_reply(reply, info);
1772 }
1773
1774 static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1775 {
1776         struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
1777         struct datapath *dp;
1778         int skip = cb->args[0];
1779         int i = 0;
1780
1781         list_for_each_entry(dp, &ovs_net->dps, list_node) {
1782                 if (i >= skip &&
1783                     ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid,
1784                                          cb->nlh->nlmsg_seq, NLM_F_MULTI,
1785                                          OVS_DP_CMD_NEW) < 0)
1786                         break;
1787                 i++;
1788         }
1789
1790         cb->args[0] = i;
1791
1792         return skb->len;
1793 }
1794
1795 static struct genl_ops dp_datapath_genl_ops[] = {
1796         { .cmd = OVS_DP_CMD_NEW,
1797           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1798           .policy = datapath_policy,
1799           .doit = ovs_dp_cmd_new
1800         },
1801         { .cmd = OVS_DP_CMD_DEL,
1802           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1803           .policy = datapath_policy,
1804           .doit = ovs_dp_cmd_del
1805         },
1806         { .cmd = OVS_DP_CMD_GET,
1807           .flags = 0,               /* OK for unprivileged users. */
1808           .policy = datapath_policy,
1809           .doit = ovs_dp_cmd_get,
1810           .dumpit = ovs_dp_cmd_dump
1811         },
1812         { .cmd = OVS_DP_CMD_SET,
1813           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1814           .policy = datapath_policy,
1815           .doit = ovs_dp_cmd_set,
1816         },
1817 };
1818
1819 static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = {
1820 #ifdef HAVE_NLA_NUL_STRING
1821         [OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
1822         [OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) },
1823 #else
1824         [OVS_VPORT_ATTR_STATS] = { .minlen = sizeof(struct ovs_vport_stats) },
1825 #endif
1826         [OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 },
1827         [OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 },
1828         [OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_U32 },
1829         [OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED },
1830 };
1831
1832 static struct genl_family dp_vport_genl_family = {
1833         .id = GENL_ID_GENERATE,
1834         .hdrsize = sizeof(struct ovs_header),
1835         .name = OVS_VPORT_FAMILY,
1836         .version = OVS_VPORT_VERSION,
1837         .maxattr = OVS_VPORT_ATTR_MAX,
1838          SET_NETNSOK
1839 };
1840
1841 struct genl_multicast_group ovs_dp_vport_multicast_group = {
1842         .name = OVS_VPORT_MCGROUP
1843 };
1844
1845 /* Called with RTNL lock or RCU read lock. */
1846 static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb,
1847                                    u32 portid, u32 seq, u32 flags, u8 cmd)
1848 {
1849         struct ovs_header *ovs_header;
1850         struct ovs_vport_stats vport_stats;
1851         int err;
1852
1853         ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family,
1854                                  flags, cmd);
1855         if (!ovs_header)
1856                 return -EMSGSIZE;
1857
1858         ovs_header->dp_ifindex = get_dpifindex(vport->dp);
1859
1860         if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) ||
1861             nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) ||
1862             nla_put_string(skb, OVS_VPORT_ATTR_NAME, vport->ops->get_name(vport)) ||
1863             nla_put_u32(skb, OVS_VPORT_ATTR_UPCALL_PID, vport->upcall_portid))
1864                 goto nla_put_failure;
1865
1866         ovs_vport_get_stats(vport, &vport_stats);
1867         if (nla_put(skb, OVS_VPORT_ATTR_STATS, sizeof(struct ovs_vport_stats),
1868                     &vport_stats))
1869                 goto nla_put_failure;
1870
1871         err = ovs_vport_get_options(vport, skb);
1872         if (err == -EMSGSIZE)
1873                 goto error;
1874
1875         return genlmsg_end(skb, ovs_header);
1876
1877 nla_put_failure:
1878         err = -EMSGSIZE;
1879 error:
1880         genlmsg_cancel(skb, ovs_header);
1881         return err;
1882 }
1883
1884 /* Called with RTNL lock or RCU read lock. */
1885 struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, u32 portid,
1886                                          u32 seq, u8 cmd)
1887 {
1888         struct sk_buff *skb;
1889         int retval;
1890
1891         skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
1892         if (!skb)
1893                 return ERR_PTR(-ENOMEM);
1894
1895         retval = ovs_vport_cmd_fill_info(vport, skb, portid, seq, 0, cmd);
1896         BUG_ON(retval < 0);
1897
1898         return skb;
1899 }
1900
1901 static int ovs_vport_cmd_validate(struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
1902 {
1903         return CHECK_NUL_STRING(a[OVS_VPORT_ATTR_NAME], IFNAMSIZ - 1);
1904 }
1905
1906 /* Called with RTNL lock or RCU read lock. */
1907 static struct vport *lookup_vport(struct net *net,
1908                                   struct ovs_header *ovs_header,
1909                                   struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
1910 {
1911         struct datapath *dp;
1912         struct vport *vport;
1913
1914         if (a[OVS_VPORT_ATTR_NAME]) {
1915                 vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME]));
1916                 if (!vport)
1917                         return ERR_PTR(-ENODEV);
1918                 if (ovs_header->dp_ifindex &&
1919                     ovs_header->dp_ifindex != get_dpifindex(vport->dp))
1920                         return ERR_PTR(-ENODEV);
1921                 return vport;
1922         } else if (a[OVS_VPORT_ATTR_PORT_NO]) {
1923                 u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
1924
1925                 if (port_no >= DP_MAX_PORTS)
1926                         return ERR_PTR(-EFBIG);
1927
1928                 dp = get_dp(net, ovs_header->dp_ifindex);
1929                 if (!dp)
1930                         return ERR_PTR(-ENODEV);
1931
1932                 vport = ovs_vport_rtnl_rcu(dp, port_no);
1933                 if (!vport)
1934                         return ERR_PTR(-ENODEV);
1935                 return vport;
1936         } else
1937                 return ERR_PTR(-EINVAL);
1938 }
1939
1940 static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
1941 {
1942         struct nlattr **a = info->attrs;
1943         struct ovs_header *ovs_header = info->userhdr;
1944         struct vport_parms parms;
1945         struct sk_buff *reply;
1946         struct vport *vport;
1947         struct datapath *dp;
1948         u32 port_no;
1949         int err;
1950
1951         err = -EINVAL;
1952         if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] ||
1953             !a[OVS_VPORT_ATTR_UPCALL_PID])
1954                 goto exit;
1955
1956         err = ovs_vport_cmd_validate(a);
1957         if (err)
1958                 goto exit;
1959
1960         rtnl_lock();
1961         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1962         err = -ENODEV;
1963         if (!dp)
1964                 goto exit_unlock;
1965
1966         if (a[OVS_VPORT_ATTR_PORT_NO]) {
1967                 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
1968
1969                 err = -EFBIG;
1970                 if (port_no >= DP_MAX_PORTS)
1971                         goto exit_unlock;
1972
1973                 vport = ovs_vport_rtnl(dp, port_no);
1974                 err = -EBUSY;
1975                 if (vport)
1976                         goto exit_unlock;
1977         } else {
1978                 for (port_no = 1; ; port_no++) {
1979                         if (port_no >= DP_MAX_PORTS) {
1980                                 err = -EFBIG;
1981                                 goto exit_unlock;
1982                         }
1983                         vport = ovs_vport_rtnl(dp, port_no);
1984                         if (!vport)
1985                                 break;
1986                 }
1987         }
1988
1989         parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
1990         parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
1991         parms.options = a[OVS_VPORT_ATTR_OPTIONS];
1992         parms.dp = dp;
1993         parms.port_no = port_no;
1994         parms.upcall_portid = nla_get_u32(a[OVS_VPORT_ATTR_UPCALL_PID]);
1995
1996         vport = new_vport(&parms);
1997         err = PTR_ERR(vport);
1998         if (IS_ERR(vport))
1999                 goto exit_unlock;
2000
2001         err = 0;
2002         if (a[OVS_VPORT_ATTR_STATS])
2003                 ovs_vport_set_stats(vport, nla_data(a[OVS_VPORT_ATTR_STATS]));
2004
2005         reply = ovs_vport_cmd_build_info(vport, info->snd_portid, info->snd_seq,
2006                                          OVS_VPORT_CMD_NEW);
2007         if (IS_ERR(reply)) {
2008                 err = PTR_ERR(reply);
2009                 ovs_dp_detach_port(vport);
2010                 goto exit_unlock;
2011         }
2012
2013         ovs_notify(reply, info, &ovs_dp_vport_multicast_group);
2014
2015 exit_unlock:
2016         rtnl_unlock();
2017 exit:
2018         return err;
2019 }
2020
2021 static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info)
2022 {
2023         struct nlattr **a = info->attrs;
2024         struct sk_buff *reply;
2025         struct vport *vport;
2026         int err;
2027
2028         err = ovs_vport_cmd_validate(a);
2029         if (err)
2030                 goto exit;
2031
2032         rtnl_lock();
2033         vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
2034         err = PTR_ERR(vport);
2035         if (IS_ERR(vport))
2036                 goto exit_unlock;
2037
2038         err = 0;
2039         if (a[OVS_VPORT_ATTR_TYPE] &&
2040             nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type)
2041                 err = -EINVAL;
2042
2043         reply = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2044         if (!reply) {
2045                 err = -ENOMEM;
2046                 goto exit_unlock;
2047         }
2048
2049         if (!err && a[OVS_VPORT_ATTR_OPTIONS])
2050                 err = ovs_vport_set_options(vport, a[OVS_VPORT_ATTR_OPTIONS]);
2051         if (err)
2052                 goto exit_free;
2053
2054         if (a[OVS_VPORT_ATTR_STATS])
2055                 ovs_vport_set_stats(vport, nla_data(a[OVS_VPORT_ATTR_STATS]));
2056
2057         if (a[OVS_VPORT_ATTR_UPCALL_PID])
2058                 vport->upcall_portid = nla_get_u32(a[OVS_VPORT_ATTR_UPCALL_PID]);
2059
2060         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
2061                                       info->snd_seq, 0, OVS_VPORT_CMD_NEW);
2062         BUG_ON(err < 0);
2063
2064         ovs_notify(reply, info, &ovs_dp_vport_multicast_group);
2065
2066         rtnl_unlock();
2067         return 0;
2068
2069 exit_free:
2070         kfree_skb(reply);
2071 exit_unlock:
2072         rtnl_unlock();
2073 exit:
2074         return err;
2075 }
2076
2077 static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info)
2078 {
2079         struct nlattr **a = info->attrs;
2080         struct sk_buff *reply;
2081         struct vport *vport;
2082         int err;
2083
2084         err = ovs_vport_cmd_validate(a);
2085         if (err)
2086                 goto exit;
2087
2088         rtnl_lock();
2089         vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
2090         err = PTR_ERR(vport);
2091         if (IS_ERR(vport))
2092                 goto exit_unlock;
2093
2094         if (vport->port_no == OVSP_LOCAL) {
2095                 err = -EINVAL;
2096                 goto exit_unlock;
2097         }
2098
2099         reply = ovs_vport_cmd_build_info(vport, info->snd_portid,
2100                                          info->snd_seq, OVS_VPORT_CMD_DEL);
2101         err = PTR_ERR(reply);
2102         if (IS_ERR(reply))
2103                 goto exit_unlock;
2104
2105         err = 0;
2106         ovs_dp_detach_port(vport);
2107
2108         ovs_notify(reply, info, &ovs_dp_vport_multicast_group);
2109
2110 exit_unlock:
2111         rtnl_unlock();
2112 exit:
2113         return err;
2114 }
2115
2116 static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info)
2117 {
2118         struct nlattr **a = info->attrs;
2119         struct ovs_header *ovs_header = info->userhdr;
2120         struct sk_buff *reply;
2121         struct vport *vport;
2122         int err;
2123
2124         err = ovs_vport_cmd_validate(a);
2125         if (err)
2126                 goto exit;
2127
2128         rcu_read_lock();
2129         vport = lookup_vport(sock_net(skb->sk), ovs_header, a);
2130         err = PTR_ERR(vport);
2131         if (IS_ERR(vport))
2132                 goto exit_unlock;
2133
2134         reply = ovs_vport_cmd_build_info(vport, info->snd_portid,
2135                                          info->snd_seq, OVS_VPORT_CMD_NEW);
2136         err = PTR_ERR(reply);
2137         if (IS_ERR(reply))
2138                 goto exit_unlock;
2139
2140         rcu_read_unlock();
2141
2142         return genlmsg_reply(reply, info);
2143
2144 exit_unlock:
2145         rcu_read_unlock();
2146 exit:
2147         return err;
2148 }
2149
2150 static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
2151 {
2152         struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
2153         struct datapath *dp;
2154         int bucket = cb->args[0], skip = cb->args[1];
2155         int i, j = 0;
2156
2157         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
2158         if (!dp)
2159                 return -ENODEV;
2160
2161         rcu_read_lock();
2162         for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) {
2163                 struct vport *vport;
2164
2165                 j = 0;
2166                 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
2167                         if (j >= skip &&
2168                             ovs_vport_cmd_fill_info(vport, skb,
2169                                                     NETLINK_CB(cb->skb).portid,
2170                                                     cb->nlh->nlmsg_seq,
2171                                                     NLM_F_MULTI,
2172                                                     OVS_VPORT_CMD_NEW) < 0)
2173                                 goto out;
2174
2175                         j++;
2176                 }
2177                 skip = 0;
2178         }
2179 out:
2180         rcu_read_unlock();
2181
2182         cb->args[0] = i;
2183         cb->args[1] = j;
2184
2185         return skb->len;
2186 }
2187
2188 static struct genl_ops dp_vport_genl_ops[] = {
2189         { .cmd = OVS_VPORT_CMD_NEW,
2190           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2191           .policy = vport_policy,
2192           .doit = ovs_vport_cmd_new
2193         },
2194         { .cmd = OVS_VPORT_CMD_DEL,
2195           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2196           .policy = vport_policy,
2197           .doit = ovs_vport_cmd_del
2198         },
2199         { .cmd = OVS_VPORT_CMD_GET,
2200           .flags = 0,               /* OK for unprivileged users. */
2201           .policy = vport_policy,
2202           .doit = ovs_vport_cmd_get,
2203           .dumpit = ovs_vport_cmd_dump
2204         },
2205         { .cmd = OVS_VPORT_CMD_SET,
2206           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2207           .policy = vport_policy,
2208           .doit = ovs_vport_cmd_set,
2209         },
2210 };
2211
2212 struct genl_family_and_ops {
2213         struct genl_family *family;
2214         struct genl_ops *ops;
2215         int n_ops;
2216         struct genl_multicast_group *group;
2217 };
2218
2219 static const struct genl_family_and_ops dp_genl_families[] = {
2220         { &dp_datapath_genl_family,
2221           dp_datapath_genl_ops, ARRAY_SIZE(dp_datapath_genl_ops),
2222           &ovs_dp_datapath_multicast_group },
2223         { &dp_vport_genl_family,
2224           dp_vport_genl_ops, ARRAY_SIZE(dp_vport_genl_ops),
2225           &ovs_dp_vport_multicast_group },
2226         { &dp_flow_genl_family,
2227           dp_flow_genl_ops, ARRAY_SIZE(dp_flow_genl_ops),
2228           &ovs_dp_flow_multicast_group },
2229         { &dp_packet_genl_family,
2230           dp_packet_genl_ops, ARRAY_SIZE(dp_packet_genl_ops),
2231           NULL },
2232 };
2233
2234 static void dp_unregister_genl(int n_families)
2235 {
2236         int i;
2237
2238         for (i = 0; i < n_families; i++)
2239                 genl_unregister_family(dp_genl_families[i].family);
2240 }
2241
2242 static int dp_register_genl(void)
2243 {
2244         int n_registered;
2245         int err;
2246         int i;
2247
2248         n_registered = 0;
2249         for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
2250                 const struct genl_family_and_ops *f = &dp_genl_families[i];
2251
2252                 err = genl_register_family_with_ops(f->family, f->ops,
2253                                                     f->n_ops);
2254                 if (err)
2255                         goto error;
2256                 n_registered++;
2257
2258                 if (f->group) {
2259                         err = genl_register_mc_group(f->family, f->group);
2260                         if (err)
2261                                 goto error;
2262                 }
2263         }
2264
2265         return 0;
2266
2267 error:
2268         dp_unregister_genl(n_registered);
2269         return err;
2270 }
2271
2272 static int __rehash_flow_table(void *dummy)
2273 {
2274         struct datapath *dp;
2275         struct net *net;
2276
2277         rtnl_lock();
2278         for_each_net(net) {
2279                 struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2280
2281                 list_for_each_entry(dp, &ovs_net->dps, list_node) {
2282                         struct flow_table *old_table = genl_dereference(dp->table);
2283                         struct flow_table *new_table;
2284
2285                         new_table = ovs_flow_tbl_rehash(old_table);
2286                         if (!IS_ERR(new_table)) {
2287                                 rcu_assign_pointer(dp->table, new_table);
2288                                 ovs_flow_tbl_deferred_destroy(old_table);
2289                         }
2290                 }
2291         }
2292         rtnl_unlock();
2293         return 0;
2294 }
2295
2296 static void rehash_flow_table(struct work_struct *work)
2297 {
2298         genl_exec(__rehash_flow_table, NULL);
2299         schedule_delayed_work(&rehash_flow_wq, REHASH_FLOW_INTERVAL);
2300 }
2301
2302 static int dp_destroy_all(void *data)
2303 {
2304         struct datapath *dp, *dp_next;
2305         struct ovs_net *ovs_net = data;
2306
2307         list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node)
2308                 __dp_destroy(dp);
2309
2310         return 0;
2311 }
2312
2313 static int __net_init ovs_init_net(struct net *net)
2314 {
2315         struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2316
2317         INIT_LIST_HEAD(&ovs_net->dps);
2318         return 0;
2319 }
2320
2321 static void __net_exit ovs_exit_net(struct net *net)
2322 {
2323         struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2324
2325         genl_exec(dp_destroy_all, ovs_net);
2326 }
2327
2328 static struct pernet_operations ovs_net_ops = {
2329         .init = ovs_init_net,
2330         .exit = ovs_exit_net,
2331         .id   = &ovs_net_id,
2332         .size = sizeof(struct ovs_net),
2333 };
2334
2335 static int __init dp_init(void)
2336 {
2337         int err;
2338
2339         BUILD_BUG_ON(sizeof(struct ovs_skb_cb) > FIELD_SIZEOF(struct sk_buff, cb));
2340
2341         pr_info("Open vSwitch switching datapath %s, built "__DATE__" "__TIME__"\n",
2342                 VERSION);
2343
2344         err = genl_exec_init();
2345         if (err)
2346                 goto error;
2347
2348         err = ovs_workqueues_init();
2349         if (err)
2350                 goto error_genl_exec;
2351
2352         err = ovs_flow_init();
2353         if (err)
2354                 goto error_wq;
2355
2356         err = ovs_vport_init();
2357         if (err)
2358                 goto error_flow_exit;
2359
2360         err = register_pernet_device(&ovs_net_ops);
2361         if (err)
2362                 goto error_vport_exit;
2363
2364         err = register_netdevice_notifier(&ovs_dp_device_notifier);
2365         if (err)
2366                 goto error_netns_exit;
2367
2368         err = dp_register_genl();
2369         if (err < 0)
2370                 goto error_unreg_notifier;
2371
2372         schedule_delayed_work(&rehash_flow_wq, REHASH_FLOW_INTERVAL);
2373
2374         return 0;
2375
2376 error_unreg_notifier:
2377         unregister_netdevice_notifier(&ovs_dp_device_notifier);
2378 error_netns_exit:
2379         unregister_pernet_device(&ovs_net_ops);
2380 error_vport_exit:
2381         ovs_vport_exit();
2382 error_flow_exit:
2383         ovs_flow_exit();
2384 error_wq:
2385         ovs_workqueues_exit();
2386 error_genl_exec:
2387         genl_exec_exit();
2388 error:
2389         return err;
2390 }
2391
2392 static void dp_cleanup(void)
2393 {
2394         cancel_delayed_work_sync(&rehash_flow_wq);
2395         dp_unregister_genl(ARRAY_SIZE(dp_genl_families));
2396         unregister_netdevice_notifier(&ovs_dp_device_notifier);
2397         unregister_pernet_device(&ovs_net_ops);
2398         rcu_barrier();
2399         ovs_vport_exit();
2400         ovs_flow_exit();
2401         ovs_workqueues_exit();
2402         genl_exec_exit();
2403 }
2404
2405 module_init(dp_init);
2406 module_exit(dp_cleanup);
2407
2408 MODULE_DESCRIPTION("Open vSwitch switching datapath");
2409 MODULE_LICENSE("GPL");
2410 MODULE_VERSION(VERSION);