+/* U32 indices for segmented flow classification. */
+const uint8_t flow_segment_u32s[4] = {
+ FLOW_SEGMENT_1_ENDS_AT / 4,
+ FLOW_SEGMENT_2_ENDS_AT / 4,
+ FLOW_SEGMENT_3_ENDS_AT / 4,
+ FLOW_U32S
+};
+
+/* miniflow_extract() assumes the following to be true to optimize the
+ * extraction process. */
+BUILD_ASSERT_DECL(offsetof(struct flow, dl_type) + 2
+ == offsetof(struct flow, vlan_tci) &&
+ offsetof(struct flow, dl_type) / 4
+ == offsetof(struct flow, vlan_tci) / 4 );
+
+BUILD_ASSERT_DECL(offsetof(struct flow, nw_frag) + 3
+ == offsetof(struct flow, nw_proto) &&
+ offsetof(struct flow, nw_tos) + 2
+ == offsetof(struct flow, nw_proto) &&
+ offsetof(struct flow, nw_ttl) + 1
+ == offsetof(struct flow, nw_proto) &&
+ offsetof(struct flow, nw_frag) / 4
+ == offsetof(struct flow, nw_tos) / 4 &&
+ offsetof(struct flow, nw_ttl) / 4
+ == offsetof(struct flow, nw_tos) / 4 &&
+ offsetof(struct flow, nw_proto) / 4
+ == offsetof(struct flow, nw_tos) / 4);
+
+/* TCP flags in the first half of a BE32, zeroes in the other half. */
+BUILD_ASSERT_DECL(offsetof(struct flow, tcp_flags) + 2
+ == offsetof(struct flow, pad) &&
+ offsetof(struct flow, tcp_flags) / 4
+ == offsetof(struct flow, pad) / 4);
+#if WORDS_BIGENDIAN
+#define TCP_FLAGS_BE32(tcp_ctl) ((OVS_FORCE ovs_be32)TCP_FLAGS_BE16(tcp_ctl) \
+ << 16)
+#else
+#define TCP_FLAGS_BE32(tcp_ctl) ((OVS_FORCE ovs_be32)TCP_FLAGS_BE16(tcp_ctl))
+#endif
+
+BUILD_ASSERT_DECL(offsetof(struct flow, tp_src) + 2
+ == offsetof(struct flow, tp_dst) &&
+ offsetof(struct flow, tp_src) / 4
+ == offsetof(struct flow, tp_dst) / 4);
+
+/* Removes 'size' bytes from the head end of '*datap', of size '*sizep', which
+ * must contain at least 'size' bytes of data. Returns the first byte of data
+ * removed. */
+static inline const void *
+data_pull(void **datap, size_t *sizep, size_t size)
+{
+ char *data = (char *)*datap;
+ *datap = data + size;
+ *sizep -= size;
+ return data;
+}
+
+/* If '*datap' has at least 'size' bytes of data, removes that many bytes from
+ * the head end of '*datap' and returns the first byte removed. Otherwise,
+ * returns a null pointer without modifying '*datap'. */
+static inline const void *
+data_try_pull(void **datap, size_t *sizep, size_t size)
+{
+ return OVS_LIKELY(*sizep >= size) ? data_pull(datap, sizep, size) : NULL;
+}
+
+/* Context for pushing data to a miniflow. */
+struct mf_ctx {
+ uint64_t map;
+ uint32_t *data;
+ uint32_t * const end;
+};
+
+/* miniflow_push_* macros allow filling in a miniflow data values in order.
+ * Assertions are needed only when the layout of the struct flow is modified.
+ * 'ofs' is a compile-time constant, which allows most of the code be optimized
+ * away. Some GCC versions gave warnigns on ALWAYS_INLINE, so these are
+ * defined as macros. */
+
+#if (FLOW_WC_SEQ != 26)
+#define MINIFLOW_ASSERT(X) ovs_assert(X)
+#else
+#define MINIFLOW_ASSERT(X)
+#endif
+
+#define miniflow_push_uint32_(MF, OFS, VALUE) \
+{ \
+ MINIFLOW_ASSERT(MF.data < MF.end && (OFS) % 4 == 0 \
+ && !(MF.map & (UINT64_MAX << (OFS) / 4))); \
+ *MF.data++ = VALUE; \
+ MF.map |= UINT64_C(1) << (OFS) / 4; \
+}
+
+#define miniflow_push_be32_(MF, OFS, VALUE) \
+ miniflow_push_uint32_(MF, OFS, (OVS_FORCE uint32_t)(VALUE))
+
+#define miniflow_push_uint16_(MF, OFS, VALUE) \
+{ \
+ MINIFLOW_ASSERT(MF.data < MF.end && \
+ (((OFS) % 4 == 0 && !(MF.map & (UINT64_MAX << (OFS) / 4))) \
+ || ((OFS) % 4 == 2 && MF.map & (UINT64_C(1) << (OFS) / 4) \
+ && !(MF.map & (UINT64_MAX << ((OFS) / 4 + 1)))))); \
+ \
+ if ((OFS) % 4 == 0) { \
+ *(uint16_t *)MF.data = VALUE; \
+ MF.map |= UINT64_C(1) << (OFS) / 4; \
+ } else if ((OFS) % 4 == 2) { \
+ *((uint16_t *)MF.data + 1) = VALUE; \
+ MF.data++; \
+ } \
+}
+
+#define miniflow_push_be16_(MF, OFS, VALUE) \
+ miniflow_push_uint16_(MF, OFS, (OVS_FORCE uint16_t)VALUE);
+
+/* Data at 'valuep' may be unaligned. */
+#define miniflow_push_words_(MF, OFS, VALUEP, N_WORDS) \
+{ \
+ int ofs32 = (OFS) / 4; \
+ \
+ MINIFLOW_ASSERT(MF.data + (N_WORDS) <= MF.end && (OFS) % 4 == 0 \
+ && !(MF.map & (UINT64_MAX << ofs32))); \
+ \
+ memcpy(MF.data, (VALUEP), (N_WORDS) * sizeof *MF.data); \
+ MF.data += (N_WORDS); \
+ MF.map |= ((UINT64_MAX >> (64 - (N_WORDS))) << ofs32); \
+}
+
+#define miniflow_push_uint32(MF, FIELD, VALUE) \
+ miniflow_push_uint32_(MF, offsetof(struct flow, FIELD), VALUE)
+
+#define miniflow_push_be32(MF, FIELD, VALUE) \
+ miniflow_push_be32_(MF, offsetof(struct flow, FIELD), VALUE)
+
+#define miniflow_push_uint32_check(MF, FIELD, VALUE) \
+ { if (OVS_LIKELY(VALUE)) { \
+ miniflow_push_uint32_(MF, offsetof(struct flow, FIELD), VALUE); \
+ } \