X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=lib%2Fdpif-netdev.c;h=0de602720c2d424338d3dc34eb383a2d0a46d765;hb=4b0424809b823101c969a0691fc1db0c880ae64a;hp=461e0dc09c7337471ec9ec5489d55eafaccc5f16;hpb=03f976b173e2e670eb2fa77eae8eacd2abf4bf84;p=sliver-openvswitch.git diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c index 461e0dc09..0de602720 100644 --- a/lib/dpif-netdev.c +++ b/lib/dpif-netdev.c @@ -31,6 +31,7 @@ #include #include +#include "classifier.h" #include "csum.h" #include "dpif.h" #include "dpif-provider.h" @@ -39,6 +40,7 @@ #include "flow.h" #include "hmap.h" #include "list.h" +#include "meta-flow.h" #include "netdev.h" #include "netdev-vport.h" #include "netlink.h" @@ -59,6 +61,9 @@ VLOG_DEFINE_THIS_MODULE(dpif_netdev); +/* By default, choose a priority in the middle. */ +#define NETDEV_RULE_PRIORITY 0x8000 + /* Configuration parameters. */ enum { MAX_PORTS = 256 }; /* Maximum number of ports. */ enum { MAX_FLOWS = 65536 }; /* Maximum number of flows in flow table. */ @@ -92,6 +97,7 @@ struct dp_netdev { int max_mtu; /* Maximum MTU of any port added so far. */ struct dp_netdev_queue queues[N_QUEUES]; + struct classifier cls; /* Classifier. */ struct hmap flow_table; /* Flow table. */ struct seq *queue_seq; /* Incremented whenever a packet is queued. */ @@ -118,14 +124,18 @@ struct dp_netdev_port { /* A flow in dp_netdev's 'flow_table'. */ struct dp_netdev_flow { - struct hmap_node node; /* Element in dp_netdev's 'flow_table'. */ - struct flow key; + /* Packet classification. */ + struct cls_rule cr; /* In owning dp_netdev's 'cls'. */ + + /* Hash table index by unmasked flow.*/ + struct hmap_node node; /* In owning dp_netdev's 'flow_table'. */ + struct flow flow; /* The flow that created this entry. */ /* Statistics. */ long long int used; /* Last used time, in monotonic msecs. */ long long int packet_count; /* Number of packets matched. */ long long int byte_count; /* Number of bytes matched. */ - uint8_t tcp_flags; /* Bitwise-OR of seen tcp_flags values. */ + uint16_t tcp_flags; /* Bitwise-OR of seen tcp_flags values. */ /* Actions. */ struct nlattr *actions; @@ -156,17 +166,15 @@ static int do_add_port(struct dp_netdev *, const char *devname, static int do_del_port(struct dp_netdev *, odp_port_t port_no); static int dpif_netdev_open(const struct dpif_class *, const char *name, bool create, struct dpif **); -static int dp_netdev_output_userspace(struct dp_netdev *, const struct ofpbuf *, +static int dp_netdev_output_userspace(struct dp_netdev *, struct ofpbuf *, int queue_no, const struct flow *, const struct nlattr *userdata); -static void dp_netdev_execute_actions(struct dp_netdev *, - struct ofpbuf *, struct flow *, +static void dp_netdev_execute_actions(struct dp_netdev *, const struct flow *, + struct ofpbuf *, struct pkt_metadata *, const struct nlattr *actions, size_t actions_len); -static void dp_netdev_port_input(struct dp_netdev *dp, - struct dp_netdev_port *port, - struct ofpbuf *packet, uint32_t skb_priority, - uint32_t pkt_mark, const struct flow_tnl *tnl); +static void dp_netdev_port_input(struct dp_netdev *dp, struct ofpbuf *packet, + struct pkt_metadata *md); static struct dpif_netdev * dpif_netdev_cast(const struct dpif *dpif) @@ -291,6 +299,7 @@ create_dp_netdev(const char *name, const struct dpif_class *class, dp->queues[i].head = dp->queues[i].tail = 0; } dp->queue_seq = seq_create(); + classifier_init(&dp->cls, NULL); hmap_init(&dp->flow_table); list_init(&dp->port_list); dp->port_seq = seq_create(); @@ -341,6 +350,7 @@ dp_netdev_purge_queues(struct dp_netdev *dp) while (q->tail != q->head) { struct dp_netdev_upcall *u = &q->upcalls[q->tail++ & QUEUE_MASK]; + ofpbuf_uninit(&u->upcall.packet); ofpbuf_uninit(&u->buf); } } @@ -357,6 +367,7 @@ dp_netdev_free(struct dp_netdev *dp) } dp_netdev_purge_queues(dp); seq_destroy(dp->queue_seq); + classifier_destroy(&dp->cls); hmap_destroy(&dp->flow_table); seq_destroy(dp->port_seq); free(dp->name); @@ -402,6 +413,8 @@ dpif_netdev_get_stats(const struct dpif *dpif, struct dpif_dp_stats *stats) stats->n_hit = dp->n_hit; stats->n_missed = dp->n_missed; stats->n_lost = dp->n_lost; + stats->n_masks = UINT32_MAX; + stats->n_mask_hit = UINT64_MAX; ovs_mutex_unlock(&dp_netdev_mutex); return 0; @@ -630,20 +643,25 @@ dpif_netdev_get_max_ports(const struct dpif *dpif OVS_UNUSED) } static void -dp_netdev_free_flow(struct dp_netdev *dp, struct dp_netdev_flow *flow) +dp_netdev_free_flow(struct dp_netdev *dp, struct dp_netdev_flow *netdev_flow) { - hmap_remove(&dp->flow_table, &flow->node); - free(flow->actions); - free(flow); + ovs_rwlock_wrlock(&dp->cls.rwlock); + classifier_remove(&dp->cls, &netdev_flow->cr); + ovs_rwlock_unlock(&dp->cls.rwlock); + cls_rule_destroy(&netdev_flow->cr); + + hmap_remove(&dp->flow_table, &netdev_flow->node); + free(netdev_flow->actions); + free(netdev_flow); } static void dp_netdev_flow_flush(struct dp_netdev *dp) { - struct dp_netdev_flow *flow, *next; + struct dp_netdev_flow *netdev_flow, *next; - HMAP_FOR_EACH_SAFE (flow, next, node, &dp->flow_table) { - dp_netdev_free_flow(dp, flow); + HMAP_FOR_EACH_SAFE (netdev_flow, next, node, &dp->flow_table) { + dp_netdev_free_flow(dp, netdev_flow); } } @@ -740,25 +758,90 @@ dpif_netdev_port_poll_wait(const struct dpif *dpif_) } static struct dp_netdev_flow * -dp_netdev_lookup_flow(const struct dp_netdev *dp, const struct flow *key) +dp_netdev_lookup_flow(const struct dp_netdev *dp, const struct flow *flow) +{ + struct cls_rule *cr; + + ovs_rwlock_wrlock(&dp->cls.rwlock); + cr = classifier_lookup(&dp->cls, flow, NULL); + ovs_rwlock_unlock(&dp->cls.rwlock); + + return (cr + ? CONTAINER_OF(cr, struct dp_netdev_flow, cr) + : NULL); +} + +static struct dp_netdev_flow * +dp_netdev_find_flow(const struct dp_netdev *dp, const struct flow *flow) { - struct dp_netdev_flow *flow; + struct dp_netdev_flow *netdev_flow; - HMAP_FOR_EACH_WITH_HASH (flow, node, flow_hash(key, 0), &dp->flow_table) { - if (flow_equal(&flow->key, key)) { - return flow; + HMAP_FOR_EACH_WITH_HASH (netdev_flow, node, flow_hash(flow, 0), + &dp->flow_table) { + if (flow_equal(&netdev_flow->flow, flow)) { + return netdev_flow; } } return NULL; } static void -get_dpif_flow_stats(struct dp_netdev_flow *flow, struct dpif_flow_stats *stats) +get_dpif_flow_stats(struct dp_netdev_flow *netdev_flow, + struct dpif_flow_stats *stats) { - stats->n_packets = flow->packet_count; - stats->n_bytes = flow->byte_count; - stats->used = flow->used; - stats->tcp_flags = flow->tcp_flags; + stats->n_packets = netdev_flow->packet_count; + stats->n_bytes = netdev_flow->byte_count; + stats->used = netdev_flow->used; + stats->tcp_flags = netdev_flow->tcp_flags; +} + +static int +dpif_netdev_mask_from_nlattrs(const struct nlattr *key, uint32_t key_len, + const struct nlattr *mask_key, + uint32_t mask_key_len, const struct flow *flow, + struct flow *mask) +{ + if (mask_key_len) { + if (odp_flow_key_to_mask(mask_key, mask_key_len, mask, flow)) { + /* This should not happen: it indicates that + * odp_flow_key_from_mask() and odp_flow_key_to_mask() + * disagree on the acceptable form of a mask. Log the problem + * as an error, with enough details to enable debugging. */ + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5); + + if (!VLOG_DROP_ERR(&rl)) { + struct ds s; + + ds_init(&s); + odp_flow_format(key, key_len, mask_key, mask_key_len, NULL, &s, + true); + VLOG_ERR("internal error parsing flow mask %s", ds_cstr(&s)); + ds_destroy(&s); + } + + return EINVAL; + } + /* Force unwildcard the in_port. */ + mask->in_port.odp_port = u32_to_odp(UINT32_MAX); + } else { + enum mf_field_id id; + /* No mask key, unwildcard everything except fields whose + * prerequisities are not met. */ + memset(mask, 0x0, sizeof *mask); + + for (id = 0; id < MFF_N_IDS; ++id) { + /* Skip registers and metadata. */ + if (!(id >= MFF_REG0 && id < MFF_REG0 + FLOW_N_REGS) + && id != MFF_METADATA) { + const struct mf_field *mf = mf_from_id(id); + if (mf_are_prereqs_ok(mf, flow)) { + mf_mask_field(mf, mask); + } + } + } + } + + return 0; } static int @@ -767,7 +850,7 @@ dpif_netdev_flow_from_nlattrs(const struct nlattr *key, uint32_t key_len, { odp_port_t in_port; - if (odp_flow_key_to_flow(key, key_len, flow) != ODP_FIT_PERFECT) { + if (odp_flow_key_to_flow(key, key_len, flow)) { /* This should not happen: it indicates that odp_flow_key_from_flow() * and odp_flow_key_to_flow() disagree on the acceptable form of a * flow. Log the problem as an error, with enough details to enable @@ -778,7 +861,7 @@ dpif_netdev_flow_from_nlattrs(const struct nlattr *key, uint32_t key_len, struct ds s; ds_init(&s); - odp_flow_key_format(key, key_len, &s); + odp_flow_format(key, key_len, NULL, 0, NULL, &s, true); VLOG_ERR("internal error parsing flow key %s", ds_cstr(&s)); ds_destroy(&s); } @@ -800,7 +883,7 @@ dpif_netdev_flow_get(const struct dpif *dpif, struct ofpbuf **actionsp, struct dpif_flow_stats *stats) { struct dp_netdev *dp = get_dp_netdev(dpif); - struct dp_netdev_flow *flow; + struct dp_netdev_flow *netdev_flow; struct flow key; int error; @@ -810,13 +893,14 @@ dpif_netdev_flow_get(const struct dpif *dpif, } ovs_mutex_lock(&dp_netdev_mutex); - flow = dp_netdev_lookup_flow(dp, &key); - if (flow) { + netdev_flow = dp_netdev_find_flow(dp, &key); + if (netdev_flow) { if (stats) { - get_dpif_flow_stats(flow, stats); + get_dpif_flow_stats(netdev_flow, stats); } if (actionsp) { - *actionsp = ofpbuf_clone_data(flow->actions, flow->actions_len); + *actionsp = ofpbuf_clone_data(netdev_flow->actions, + netdev_flow->actions_len); } } else { error = ENOENT; @@ -827,66 +911,87 @@ dpif_netdev_flow_get(const struct dpif *dpif, } static int -set_flow_actions(struct dp_netdev_flow *flow, +set_flow_actions(struct dp_netdev_flow *netdev_flow, const struct nlattr *actions, size_t actions_len) { - flow->actions = xrealloc(flow->actions, actions_len); - flow->actions_len = actions_len; - memcpy(flow->actions, actions, actions_len); + netdev_flow->actions = xrealloc(netdev_flow->actions, actions_len); + netdev_flow->actions_len = actions_len; + memcpy(netdev_flow->actions, actions, actions_len); return 0; } static int -dp_netdev_flow_add(struct dp_netdev *dp, const struct flow *key, - const struct nlattr *actions, size_t actions_len) +dp_netdev_flow_add(struct dp_netdev *dp, const struct flow *flow, + const struct flow_wildcards *wc, + const struct nlattr *actions, + size_t actions_len) { - struct dp_netdev_flow *flow; + struct dp_netdev_flow *netdev_flow; + struct match match; int error; - flow = xzalloc(sizeof *flow); - flow->key = *key; + netdev_flow = xzalloc(sizeof *netdev_flow); + netdev_flow->flow = *flow; - error = set_flow_actions(flow, actions, actions_len); + match_init(&match, flow, wc); + cls_rule_init(&netdev_flow->cr, &match, NETDEV_RULE_PRIORITY); + ovs_rwlock_wrlock(&dp->cls.rwlock); + classifier_insert(&dp->cls, &netdev_flow->cr); + ovs_rwlock_unlock(&dp->cls.rwlock); + + error = set_flow_actions(netdev_flow, actions, actions_len); if (error) { - free(flow); + ovs_rwlock_wrlock(&dp->cls.rwlock); + classifier_remove(&dp->cls, &netdev_flow->cr); + ovs_rwlock_unlock(&dp->cls.rwlock); + cls_rule_destroy(&netdev_flow->cr); + + free(netdev_flow); return error; } - hmap_insert(&dp->flow_table, &flow->node, flow_hash(&flow->key, 0)); + hmap_insert(&dp->flow_table, &netdev_flow->node, flow_hash(flow, 0)); return 0; } static void -clear_stats(struct dp_netdev_flow *flow) +clear_stats(struct dp_netdev_flow *netdev_flow) { - flow->used = 0; - flow->packet_count = 0; - flow->byte_count = 0; - flow->tcp_flags = 0; + netdev_flow->used = 0; + netdev_flow->packet_count = 0; + netdev_flow->byte_count = 0; + netdev_flow->tcp_flags = 0; } static int dpif_netdev_flow_put(struct dpif *dpif, const struct dpif_flow_put *put) { struct dp_netdev *dp = get_dp_netdev(dpif); - struct dp_netdev_flow *flow; - struct flow key; + struct dp_netdev_flow *netdev_flow; + struct flow flow; + struct flow_wildcards wc; int error; - error = dpif_netdev_flow_from_nlattrs(put->key, put->key_len, &key); + error = dpif_netdev_flow_from_nlattrs(put->key, put->key_len, &flow); + if (error) { + return error; + } + error = dpif_netdev_mask_from_nlattrs(put->key, put->key_len, + put->mask, put->mask_len, + &flow, &wc.masks); if (error) { return error; } ovs_mutex_lock(&dp_netdev_mutex); - flow = dp_netdev_lookup_flow(dp, &key); - if (!flow) { + netdev_flow = dp_netdev_lookup_flow(dp, &flow); + if (!netdev_flow) { if (put->flags & DPIF_FP_CREATE) { if (hmap_count(&dp->flow_table) < MAX_FLOWS) { if (put->stats) { memset(put->stats, 0, sizeof *put->stats); } - error = dp_netdev_flow_add(dp, &key, put->actions, + error = dp_netdev_flow_add(dp, &flow, &wc, put->actions, put->actions_len); } else { error = EFBIG; @@ -895,18 +1000,23 @@ dpif_netdev_flow_put(struct dpif *dpif, const struct dpif_flow_put *put) error = ENOENT; } } else { - if (put->flags & DPIF_FP_MODIFY) { - error = set_flow_actions(flow, put->actions, put->actions_len); + if (put->flags & DPIF_FP_MODIFY + && flow_equal(&flow, &netdev_flow->flow)) { + error = set_flow_actions(netdev_flow, put->actions, + put->actions_len); if (!error) { if (put->stats) { - get_dpif_flow_stats(flow, put->stats); + get_dpif_flow_stats(netdev_flow, put->stats); } if (put->flags & DPIF_FP_ZERO_STATS) { - clear_stats(flow); + clear_stats(netdev_flow); } } - } else { + } else if (put->flags & DPIF_FP_CREATE) { error = EEXIST; + } else { + /* Overlapping flow. */ + error = EINVAL; } } ovs_mutex_unlock(&dp_netdev_mutex); @@ -918,7 +1028,7 @@ static int dpif_netdev_flow_del(struct dpif *dpif, const struct dpif_flow_del *del) { struct dp_netdev *dp = get_dp_netdev(dpif); - struct dp_netdev_flow *flow; + struct dp_netdev_flow *netdev_flow; struct flow key; int error; @@ -928,12 +1038,12 @@ dpif_netdev_flow_del(struct dpif *dpif, const struct dpif_flow_del *del) } ovs_mutex_lock(&dp_netdev_mutex); - flow = dp_netdev_lookup_flow(dp, &key); - if (flow) { + netdev_flow = dp_netdev_find_flow(dp, &key); + if (netdev_flow) { if (del->stats) { - get_dpif_flow_stats(flow, del->stats); + get_dpif_flow_stats(netdev_flow, del->stats); } - dp_netdev_free_flow(dp, flow); + dp_netdev_free_flow(dp, netdev_flow); } else { error = ENOENT; } @@ -947,6 +1057,7 @@ struct dp_netdev_flow_state { uint32_t offset; struct nlattr *actions; struct odputil_keybuf keybuf; + struct odputil_keybuf maskbuf; struct dpif_flow_stats stats; }; @@ -971,7 +1082,7 @@ dpif_netdev_flow_dump_next(const struct dpif *dpif, void *state_, { struct dp_netdev_flow_state *state = state_; struct dp_netdev *dp = get_dp_netdev(dpif); - struct dp_netdev_flow *flow; + struct dp_netdev_flow *netdev_flow; struct hmap_node *node; ovs_mutex_lock(&dp_netdev_mutex); @@ -981,33 +1092,43 @@ dpif_netdev_flow_dump_next(const struct dpif *dpif, void *state_, return EOF; } - flow = CONTAINER_OF(node, struct dp_netdev_flow, node); + netdev_flow = CONTAINER_OF(node, struct dp_netdev_flow, node); if (key) { struct ofpbuf buf; ofpbuf_use_stack(&buf, &state->keybuf, sizeof state->keybuf); - odp_flow_key_from_flow(&buf, &flow->key, flow->key.in_port.odp_port); + odp_flow_key_from_flow(&buf, &netdev_flow->flow, + netdev_flow->flow.in_port.odp_port); *key = buf.data; *key_len = buf.size; } - if (mask) { - *mask = NULL; - *mask_len = 0; + if (key && mask) { + struct ofpbuf buf; + struct flow_wildcards wc; + + ofpbuf_use_stack(&buf, &state->maskbuf, sizeof state->maskbuf); + minimask_expand(&netdev_flow->cr.match.mask, &wc); + odp_flow_key_from_mask(&buf, &wc.masks, &netdev_flow->flow, + odp_to_u32(wc.masks.in_port.odp_port)); + + *mask = buf.data; + *mask_len = buf.size; } if (actions) { free(state->actions); - state->actions = xmemdup(flow->actions, flow->actions_len); + state->actions = xmemdup(netdev_flow->actions, + netdev_flow->actions_len); *actions = state->actions; - *actions_len = flow->actions_len; + *actions_len = netdev_flow->actions_len; } if (stats) { - get_dpif_flow_stats(flow, &state->stats); + get_dpif_flow_stats(netdev_flow, &state->stats); *stats = &state->stats; } @@ -1026,35 +1147,25 @@ dpif_netdev_flow_dump_done(const struct dpif *dpif OVS_UNUSED, void *state_) } static int -dpif_netdev_execute(struct dpif *dpif, const struct dpif_execute *execute) +dpif_netdev_execute(struct dpif *dpif, struct dpif_execute *execute) { struct dp_netdev *dp = get_dp_netdev(dpif); - struct ofpbuf copy; + struct pkt_metadata *md = &execute->md; struct flow key; - int error; if (execute->packet->size < ETH_HEADER_LEN || execute->packet->size > UINT16_MAX) { return EINVAL; } - /* Make a deep copy of 'packet', because we might modify its data. */ - ofpbuf_init(©, DP_NETDEV_HEADROOM + execute->packet->size); - ofpbuf_reserve(©, DP_NETDEV_HEADROOM); - ofpbuf_put(©, execute->packet->data, execute->packet->size); - - flow_extract(©, 0, 0, NULL, NULL, &key); - error = dpif_netdev_flow_from_nlattrs(execute->key, execute->key_len, - &key); - if (!error) { - ovs_mutex_lock(&dp_netdev_mutex); - dp_netdev_execute_actions(dp, ©, &key, - execute->actions, execute->actions_len); - ovs_mutex_unlock(&dp_netdev_mutex); - } - - ofpbuf_uninit(©); - return error; + /* Extract flow key. */ + flow_extract(execute->packet, md->skb_priority, md->pkt_mark, &md->tunnel, + (union flow_in_port *)&md->in_port, &key); + ovs_mutex_lock(&dp_netdev_mutex); + dp_netdev_execute_actions(dp, &key, execute->packet, md, execute->actions, + execute->actions_len); + ovs_mutex_unlock(&dp_netdev_mutex); + return 0; } static int @@ -1099,7 +1210,6 @@ dpif_netdev_recv(struct dpif *dpif, struct dpif_upcall *upcall, struct dp_netdev_upcall *u = &q->upcalls[q->tail++ & QUEUE_MASK]; *upcall = u->upcall; - upcall->packet = buf; ofpbuf_uninit(buf); *buf = u->buf; @@ -1139,33 +1249,33 @@ dpif_netdev_recv_purge(struct dpif *dpif) } static void -dp_netdev_flow_used(struct dp_netdev_flow *flow, const struct ofpbuf *packet) +dp_netdev_flow_used(struct dp_netdev_flow *netdev_flow, + const struct ofpbuf *packet) { - flow->used = time_msec(); - flow->packet_count++; - flow->byte_count += packet->size; - flow->tcp_flags |= packet_get_tcp_flags(packet, &flow->key); + netdev_flow->used = time_msec(); + netdev_flow->packet_count++; + netdev_flow->byte_count += packet->size; + netdev_flow->tcp_flags |= packet_get_tcp_flags(packet, &netdev_flow->flow); } static void -dp_netdev_port_input(struct dp_netdev *dp, struct dp_netdev_port *port, - struct ofpbuf *packet, uint32_t skb_priority, - uint32_t pkt_mark, const struct flow_tnl *tnl) +dp_netdev_port_input(struct dp_netdev *dp, struct ofpbuf *packet, + struct pkt_metadata *md) { - struct dp_netdev_flow *flow; + struct dp_netdev_flow *netdev_flow; struct flow key; - union flow_in_port in_port_; if (packet->size < ETH_HEADER_LEN) { return; } - in_port_.odp_port = port->port_no; - flow_extract(packet, skb_priority, pkt_mark, tnl, &in_port_, &key); - flow = dp_netdev_lookup_flow(dp, &key); - if (flow) { - dp_netdev_flow_used(flow, packet); - dp_netdev_execute_actions(dp, packet, &key, - flow->actions, flow->actions_len); + flow_extract(packet, md->skb_priority, md->pkt_mark, &md->tunnel, + (union flow_in_port *)&md->in_port, &key); + netdev_flow = dp_netdev_lookup_flow(dp, &key); + if (netdev_flow) { + dp_netdev_flow_used(netdev_flow, packet); + dp_netdev_execute_actions(dp, &key, packet, md, + netdev_flow->actions, + netdev_flow->actions_len); dp->n_hit++; } else { dp->n_missed++; @@ -1179,22 +1289,25 @@ dpif_netdev_run(struct dpif *dpif) struct dp_netdev_port *port; struct dp_netdev *dp; struct ofpbuf packet; + size_t buf_size; ovs_mutex_lock(&dp_netdev_mutex); dp = get_dp_netdev(dpif); - ofpbuf_init(&packet, - DP_NETDEV_HEADROOM + VLAN_ETH_HEADER_LEN + dp->max_mtu); + ofpbuf_init(&packet, 0); + + buf_size = DP_NETDEV_HEADROOM + VLAN_ETH_HEADER_LEN + dp->max_mtu; LIST_FOR_EACH (port, node, &dp->port_list) { int error; - /* Reset packet contents. */ + /* Reset packet contents. Packet data may have been stolen. */ ofpbuf_clear(&packet); - ofpbuf_reserve(&packet, DP_NETDEV_HEADROOM); + ofpbuf_reserve_with_tailroom(&packet, DP_NETDEV_HEADROOM, buf_size); error = port->rx ? netdev_rx_recv(port->rx, &packet) : EOPNOTSUPP; if (!error) { - dp_netdev_port_input(dp, port, &packet, 0, 0, NULL); + struct pkt_metadata md = PKT_METADATA_INITIALIZER(port->port_no); + dp_netdev_port_input(dp, &packet, &md); } else if (error != EAGAIN && error != EOPNOTSUPP) { static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5); @@ -1234,17 +1347,17 @@ dpif_netdev_wait(struct dpif *dpif) } static void -dp_netdev_output_port(void *dp_, struct ofpbuf *packet, uint32_t out_port) +dp_netdev_output_port(struct dp_netdev *dp, struct ofpbuf *packet, + odp_port_t out_port) { - struct dp_netdev *dp = dp_; - struct dp_netdev_port *p = dp->ports[out_port]; + struct dp_netdev_port *p = dp->ports[odp_to_u32(out_port)]; if (p) { netdev_send(p->netdev, packet); } } static int -dp_netdev_output_userspace(struct dp_netdev *dp, const struct ofpbuf *packet, +dp_netdev_output_userspace(struct dp_netdev *dp, struct ofpbuf *packet, int queue_no, const struct flow *flow, const struct nlattr *userdata) { @@ -1258,7 +1371,7 @@ dp_netdev_output_userspace(struct dp_netdev *dp, const struct ofpbuf *packet, upcall->type = queue_no; /* Allocate buffer big enough for everything. */ - buf_size = ODPUTIL_FLOW_KEY_BYTES + 2 + packet->size; + buf_size = ODPUTIL_FLOW_KEY_BYTES; if (userdata) { buf_size += NLA_ALIGN(userdata->nla_len); } @@ -1275,15 +1388,10 @@ dp_netdev_output_userspace(struct dp_netdev *dp, const struct ofpbuf *packet, NLA_ALIGN(userdata->nla_len)); } - /* Put packet. - * - * We adjust 'data' and 'size' in 'buf' so that only the packet itself - * is visible in 'upcall->packet'. The ODP flow and (if present) - * userdata become part of the headroom. */ - ofpbuf_put_zeros(buf, 2); - buf->data = ofpbuf_put(buf, packet->data, packet->size); - buf->size = packet->size; - upcall->packet = buf; + /* Steal packet data. */ + ovs_assert(packet->source == OFPBUF_MALLOC); + upcall->packet = *packet; + ofpbuf_use(packet, NULL, 0); seq_change(dp->queue_seq); @@ -1294,22 +1402,60 @@ dp_netdev_output_userspace(struct dp_netdev *dp, const struct ofpbuf *packet, } } +struct dp_netdev_execute_aux { + struct dp_netdev *dp; + const struct flow *key; +}; + static void -dp_netdev_action_userspace(void *dp, struct ofpbuf *packet, - const struct flow *key, - const struct nlattr *userdata) +dp_execute_cb(void *aux_, struct ofpbuf *packet, + const struct pkt_metadata *md OVS_UNUSED, + const struct nlattr *a, bool may_steal) { - dp_netdev_output_userspace(dp, packet, DPIF_UC_ACTION, key, userdata); + struct dp_netdev_execute_aux *aux = aux_; + int type = nl_attr_type(a); + + switch ((enum ovs_action_attr)type) { + case OVS_ACTION_ATTR_OUTPUT: + dp_netdev_output_port(aux->dp, packet, u32_to_odp(nl_attr_get_u32(a))); + break; + + case OVS_ACTION_ATTR_USERSPACE: { + const struct nlattr *userdata; + + userdata = nl_attr_find_nested(a, OVS_USERSPACE_ATTR_USERDATA); + + /* Make a copy if we are not allowed to steal the packet's data. */ + if (!may_steal) { + packet = ofpbuf_clone_with_headroom(packet, DP_NETDEV_HEADROOM); + } + dp_netdev_output_userspace(aux->dp, packet, DPIF_UC_ACTION, aux->key, + userdata); + if (!may_steal) { + ofpbuf_uninit(packet); + } + break; + } + case OVS_ACTION_ATTR_PUSH_VLAN: + case OVS_ACTION_ATTR_POP_VLAN: + case OVS_ACTION_ATTR_PUSH_MPLS: + case OVS_ACTION_ATTR_POP_MPLS: + case OVS_ACTION_ATTR_SET: + case OVS_ACTION_ATTR_SAMPLE: + case OVS_ACTION_ATTR_UNSPEC: + case __OVS_ACTION_ATTR_MAX: + OVS_NOT_REACHED(); + } } static void -dp_netdev_execute_actions(struct dp_netdev *dp, - struct ofpbuf *packet, struct flow *key, - const struct nlattr *actions, - size_t actions_len) +dp_netdev_execute_actions(struct dp_netdev *dp, const struct flow *key, + struct ofpbuf *packet, struct pkt_metadata *md, + const struct nlattr *actions, size_t actions_len) { - odp_execute_actions(dp, packet, key, actions, actions_len, - dp_netdev_output_port, dp_netdev_action_userspace); + struct dp_netdev_execute_aux aux = {dp, key}; + + odp_execute_actions(&aux, packet, md, actions, actions_len, dp_execute_cb); } #define DPIF_NETDEV_CLASS_FUNCTIONS \