+ skb = copy_flow_from_user(uodp_flow, &flowcmd);
+ error = PTR_ERR(skb);
+ if (IS_ERR(skb))
+ goto exit;
+
+ dp = get_dp_locked(flowcmd.dp_idx);
+ error = -ENODEV;
+ if (!dp)
+ goto error_kfree_skb;
+
+ hash = flow_hash(&flowcmd.key);
+ table = get_table_protected(dp);
+ flow_node = tbl_lookup(table, &flowcmd.key, hash, flow_cmp);
+ if (!flow_node) {
+ struct sw_flow_actions *acts;
+
+ /* Bail out if we're not allowed to create a new flow. */
+ error = -ENOENT;
+ if (cmd == ODP_FLOW_SET)
+ goto error_unlock_dp;
+
+ /* Expand table, if necessary, to make room. */
+ if (tbl_count(table) >= tbl_n_buckets(table)) {
+ error = expand_table(dp);
+ if (error)
+ goto error_unlock_dp;
+ table = get_table_protected(dp);
+ }
+
+ /* Allocate flow. */
+ flow = flow_alloc();
+ if (IS_ERR(flow)) {
+ error = PTR_ERR(flow);
+ goto error_unlock_dp;
+ }
+ flow->key = flowcmd.key;
+ clear_stats(flow);
+
+ /* Obtain actions. */
+ acts = get_actions(&flowcmd);
+ error = PTR_ERR(acts);
+ if (IS_ERR(acts))
+ goto error_free_flow;
+ rcu_assign_pointer(flow->sf_acts, acts);
+
+ error = copy_flow_to_user(uodp_flow, dp, flow, flowcmd.total_len, 0);
+ if (error)
+ goto error_free_flow;
+
+ /* Put flow in bucket. */
+ error = tbl_insert(table, &flow->tbl_node, hash);
+ if (error)
+ goto error_free_flow;
+ } else {
+ /* We found a matching flow. */
+ struct sw_flow_actions *old_acts;
+
+ /* Bail out if we're not allowed to modify an existing flow.
+ * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
+ * because Generic Netlink treats the latter as a dump
+ * request. We also accept NLM_F_EXCL in case that bug ever
+ * gets fixed.
+ */
+ error = -EEXIST;
+ if (flowcmd.nlmsg_flags & (NLM_F_CREATE | NLM_F_EXCL))
+ goto error_kfree_skb;
+
+ /* Update actions. */
+ flow = flow_cast(flow_node);
+ old_acts = rcu_dereference_protected(flow->sf_acts,
+ lockdep_is_held(&dp->mutex));
+ if (flowcmd.actions &&
+ (old_acts->actions_len != flowcmd.actions_len ||
+ memcmp(old_acts->actions, flowcmd.actions,
+ flowcmd.actions_len))) {
+ struct sw_flow_actions *new_acts;
+
+ new_acts = get_actions(&flowcmd);
+ error = PTR_ERR(new_acts);
+ if (IS_ERR(new_acts))
+ goto error_kfree_skb;
+
+ rcu_assign_pointer(flow->sf_acts, new_acts);
+ flow_deferred_free_acts(old_acts);
+ }
+
+ error = copy_flow_to_user(uodp_flow, dp, flow, flowcmd.total_len, 0);
+ if (error)
+ goto error_kfree_skb;
+
+ /* Clear stats. */
+ if (flowcmd.clear) {
+ spin_lock_bh(&flow->lock);
+ clear_stats(flow);
+ spin_unlock_bh(&flow->lock);
+ }
+ }
+ kfree_skb(skb);
+ mutex_unlock(&dp->mutex);
+ return 0;
+
+error_free_flow:
+ flow_put(flow);
+error_unlock_dp:
+ mutex_unlock(&dp->mutex);
+error_kfree_skb:
+ kfree_skb(skb);
+exit:
+ return error;
+}
+
+static int get_or_del_flow(unsigned int cmd, struct odp_flow __user *uodp_flow)
+{
+ struct tbl_node *flow_node;
+ struct dp_flowcmd flowcmd;
+ struct sw_flow *flow;
+ struct sk_buff *skb;
+ struct datapath *dp;
+ struct tbl *table;
+ int err;
+
+ skb = copy_flow_from_user(uodp_flow, &flowcmd);
+ err = PTR_ERR(skb);
+ if (IS_ERR(skb))
+ goto exit;
+
+ dp = get_dp_locked(flowcmd.dp_idx);
+ err = -ENODEV;
+ if (!dp)
+ goto exit_kfree_skb;
+
+ table = get_table_protected(dp);
+ flow_node = tbl_lookup(table, &flowcmd.key, flow_hash(&flowcmd.key), flow_cmp);
+ err = -ENOENT;
+ if (!flow_node)
+ goto exit_unlock_dp;
+
+ if (cmd == ODP_FLOW_DEL) {
+ err = tbl_remove(table, flow_node);
+ if (err)
+ goto exit_unlock_dp;
+ }
+
+ flow = flow_cast(flow_node);
+ err = copy_flow_to_user(uodp_flow, dp, flow, flowcmd.total_len, 0);
+ if (!err && cmd == ODP_FLOW_DEL)
+ flow_deferred_free(flow);
+
+exit_unlock_dp:
+ mutex_unlock(&dp->mutex);
+exit_kfree_skb:
+ kfree_skb(skb);
+exit:
+ return err;
+}
+
+static int dump_flow(struct odp_flow __user *uodp_flow)
+{
+ struct tbl_node *flow_node;
+ struct dp_flowcmd flowcmd;
+ struct sw_flow *flow;
+ struct sk_buff *skb;
+ struct datapath *dp;
+ u32 bucket, obj;
+ int err;
+
+ skb = copy_flow_from_user(uodp_flow, &flowcmd);
+ err = PTR_ERR(skb);
+ if (IS_ERR(skb))
+ goto exit;
+
+ dp = get_dp_locked(flowcmd.dp_idx);
+ err = -ENODEV;
+ if (!dp)
+ goto exit_free;
+
+ bucket = flowcmd.state >> 32;
+ obj = flowcmd.state;
+ flow_node = tbl_next(dp->table, &bucket, &obj);
+ err = -ENODEV;
+ if (!flow_node)
+ goto exit_unlock_dp;
+
+ flow = flow_cast(flow_node);
+ err = copy_flow_to_user(uodp_flow, dp, flow, flowcmd.total_len,
+ ((u64)bucket << 32) | obj);
+
+exit_unlock_dp:
+ mutex_unlock(&dp->mutex);
+exit_free:
+ kfree_skb(skb);
+exit:
+ return err;
+}
+
+static const struct nla_policy datapath_policy[ODP_DP_ATTR_MAX + 1] = {
+ [ODP_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
+ [ODP_DP_ATTR_IPV4_FRAGS] = { .type = NLA_U32 },
+ [ODP_DP_ATTR_SAMPLING] = { .type = NLA_U32 },
+};
+
+static int copy_datapath_to_user(void __user *dst, struct datapath *dp, uint32_t total_len)
+{
+ struct odp_datapath *odp_datapath;
+ struct sk_buff *skb;
+ struct nlattr *nla;
+ int err;
+
+ skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
+ err = -ENOMEM;
+ if (!skb)
+ goto exit;
+
+ odp_datapath = (struct odp_datapath*)__skb_put(skb, sizeof(struct odp_datapath));
+ odp_datapath->dp_idx = dp->dp_idx;
+ odp_datapath->total_len = total_len;
+
+ rcu_read_lock();
+ err = nla_put_string(skb, ODP_DP_ATTR_NAME, dp_name(dp));
+ rcu_read_unlock();
+ if (err)
+ goto nla_put_failure;
+
+ nla = nla_reserve(skb, ODP_DP_ATTR_STATS, sizeof(struct odp_stats));
+ if (!nla)
+ goto nla_put_failure;
+ get_dp_stats(dp, nla_data(nla));
+
+ NLA_PUT_U32(skb, ODP_DP_ATTR_IPV4_FRAGS,
+ dp->drop_frags ? ODP_DP_FRAG_DROP : ODP_DP_FRAG_ZERO);
+
+ if (dp->sflow_probability)
+ NLA_PUT_U32(skb, ODP_DP_ATTR_SAMPLING, dp->sflow_probability);
+
+ if (skb->len > total_len)
+ goto nla_put_failure;
+
+ odp_datapath->len = skb->len;
+ err = copy_to_user(dst, skb->data, skb->len) ? -EFAULT : 0;
+ goto exit_free_skb;
+
+nla_put_failure:
+ err = -EMSGSIZE;
+exit_free_skb:
+ kfree_skb(skb);
+exit:
+ return err;
+}
+
+static struct sk_buff *copy_datapath_from_user(struct odp_datapath __user *uodp_datapath, struct nlattr *a[ODP_DP_ATTR_MAX + 1])
+{
+ struct odp_datapath *odp_datapath;
+ struct sk_buff *skb;
+ u32 len;
+ int err;
+
+ if (get_user(len, &uodp_datapath->len))
+ return ERR_PTR(-EFAULT);
+ if (len < sizeof(struct odp_datapath))
+ return ERR_PTR(-EINVAL);
+
+ skb = alloc_skb(len, GFP_KERNEL);
+ if (!skb)
+ return ERR_PTR(-ENOMEM);
+
+ err = -EFAULT;
+ if (copy_from_user(__skb_put(skb, len), uodp_datapath, len))
+ goto error_free_skb;
+
+ odp_datapath = (struct odp_datapath *)skb->data;
+ err = -EINVAL;
+ if (odp_datapath->len != len)
+ goto error_free_skb;
+
+ err = nla_parse(a, ODP_DP_ATTR_MAX,
+ (struct nlattr *)(skb->data + sizeof(struct odp_datapath)),
+ skb->len - sizeof(struct odp_datapath), datapath_policy);
+ if (err)
+ goto error_free_skb;
+
+ if (a[ODP_DP_ATTR_IPV4_FRAGS]) {
+ u32 frags = nla_get_u32(a[ODP_DP_ATTR_IPV4_FRAGS]);
+
+ err = -EINVAL;
+ if (frags != ODP_DP_FRAG_ZERO && frags != ODP_DP_FRAG_DROP)
+ goto error_free_skb;
+ }
+
+ err = VERIFY_NUL_STRING(a[ODP_DP_ATTR_NAME], IFNAMSIZ - 1);
+ if (err)
+ goto error_free_skb;
+
+ return skb;
+
+error_free_skb:
+ kfree_skb(skb);
+ return ERR_PTR(err);
+}
+
+/* Called with dp_mutex and optionally with RTNL lock also.
+ * Holds the returned datapath's mutex on return.
+ */
+static struct datapath *lookup_datapath(struct odp_datapath *odp_datapath, struct nlattr *a[ODP_DP_ATTR_MAX + 1])
+{
+ WARN_ON_ONCE(!mutex_is_locked(&dp_mutex));
+
+ if (!a[ODP_DP_ATTR_NAME]) {
+ struct datapath *dp;
+
+ dp = get_dp(odp_datapath->dp_idx);
+ if (!dp)
+ return ERR_PTR(-ENODEV);
+ mutex_lock(&dp->mutex);
+ return dp;
+ } else {
+ struct datapath *dp;
+ struct vport *vport;
+ int dp_idx;
+
+ vport_lock();
+ vport = vport_locate(nla_data(a[ODP_DP_ATTR_NAME]));
+ dp_idx = vport && vport->port_no == ODPP_LOCAL ? vport->dp->dp_idx : -1;
+ vport_unlock();
+
+ if (dp_idx < 0)
+ return ERR_PTR(-ENODEV);
+
+ dp = get_dp(dp_idx);
+ mutex_lock(&dp->mutex);
+ return dp;
+ }
+}
+
+static void change_datapath(struct datapath *dp, struct nlattr *a[ODP_DP_ATTR_MAX + 1])
+{
+ if (a[ODP_DP_ATTR_IPV4_FRAGS])
+ dp->drop_frags = nla_get_u32(a[ODP_DP_ATTR_IPV4_FRAGS]) == ODP_DP_FRAG_DROP;
+ if (a[ODP_DP_ATTR_SAMPLING])
+ dp->sflow_probability = nla_get_u32(a[ODP_DP_ATTR_SAMPLING]);
+}
+
+static int new_datapath(struct odp_datapath __user *uodp_datapath)
+{
+ struct nlattr *a[ODP_DP_ATTR_MAX + 1];
+ struct odp_datapath *odp_datapath;
+ struct vport_parms parms;
+ struct sk_buff *skb;
+ struct datapath *dp;
+ struct vport *vport;
+ int dp_idx;
+ int err;
+ int i;
+
+ skb = copy_datapath_from_user(uodp_datapath, a);
+ err = PTR_ERR(skb);
+ if (IS_ERR(skb))
+ goto err;
+ odp_datapath = (struct odp_datapath *)skb->data;
+
+ err = -EINVAL;
+ if (!a[ODP_DP_ATTR_NAME])
+ goto err_free_skb;
+
+ rtnl_lock();
+ mutex_lock(&dp_mutex);
+ err = -ENODEV;
+ if (!try_module_get(THIS_MODULE))
+ goto err_unlock_dp_mutex;
+
+ dp_idx = odp_datapath->dp_idx;
+ if (dp_idx < 0) {
+ err = -EFBIG;
+ for (dp_idx = 0; dp_idx < ARRAY_SIZE(dps); dp_idx++) {
+ if (get_dp(dp_idx))
+ continue;
+ err = 0;
+ break;
+ }
+ } else if (dp_idx < ARRAY_SIZE(dps))
+ err = get_dp(dp_idx) ? -EBUSY : 0;
+ else
+ err = -EINVAL;
+ if (err)
+ goto err_put_module;