+ ctx_mae, error);
+}
+
+static const struct sfc_mae_field_locator flocs_vlan[] = {
+ /* Outermost tag */
+ {
+ EFX_MAE_FIELD_VLAN0_TCI_BE,
+ RTE_SIZEOF_FIELD(struct rte_flow_item_vlan, tci),
+ offsetof(struct rte_flow_item_vlan, tci),
+ },
+ {
+ /*
+ * This locator is used only for building supported fields mask.
+ * The field is handled by sfc_mae_rule_process_pattern_data().
+ */
+ SFC_MAE_FIELD_HANDLING_DEFERRED,
+ RTE_SIZEOF_FIELD(struct rte_flow_item_vlan, inner_type),
+ offsetof(struct rte_flow_item_vlan, inner_type),
+ },
+
+ /* Innermost tag */
+ {
+ EFX_MAE_FIELD_VLAN1_TCI_BE,
+ RTE_SIZEOF_FIELD(struct rte_flow_item_vlan, tci),
+ offsetof(struct rte_flow_item_vlan, tci),
+ },
+ {
+ /*
+ * This locator is used only for building supported fields mask.
+ * The field is handled by sfc_mae_rule_process_pattern_data().
+ */
+ SFC_MAE_FIELD_HANDLING_DEFERRED,
+ RTE_SIZEOF_FIELD(struct rte_flow_item_vlan, inner_type),
+ offsetof(struct rte_flow_item_vlan, inner_type),
+ },
+};
+
+static int
+sfc_mae_rule_parse_item_vlan(const struct rte_flow_item *item,
+ struct sfc_flow_parse_ctx *ctx,
+ struct rte_flow_error *error)
+{
+ struct sfc_mae_parse_ctx *ctx_mae = ctx->mae;
+ struct sfc_mae_pattern_data *pdata = &ctx_mae->pattern_data;
+ const struct sfc_mae_field_locator *flocs;
+ struct rte_flow_item_vlan supp_mask;
+ const uint8_t *spec = NULL;
+ const uint8_t *mask = NULL;
+ unsigned int nb_flocs;
+ int rc;
+
+ RTE_BUILD_BUG_ON(SFC_MAE_MATCH_VLAN_MAX_NTAGS != 2);
+
+ if (pdata->nb_vlan_tags == SFC_MAE_MATCH_VLAN_MAX_NTAGS) {
+ return rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ITEM, item,
+ "Can't match that many VLAN tags");
+ }
+
+ nb_flocs = RTE_DIM(flocs_vlan) / SFC_MAE_MATCH_VLAN_MAX_NTAGS;
+ flocs = flocs_vlan + pdata->nb_vlan_tags * nb_flocs;
+
+ /* If parsing fails, this can remain incremented. */
+ ++pdata->nb_vlan_tags;
+
+ sfc_mae_item_build_supp_mask(flocs, nb_flocs,
+ &supp_mask, sizeof(supp_mask));
+
+ rc = sfc_flow_parse_init(item,
+ (const void **)&spec, (const void **)&mask,
+ (const void *)&supp_mask,
+ &rte_flow_item_vlan_mask,
+ sizeof(struct rte_flow_item_vlan), error);
+ if (rc != 0)
+ return rc;
+
+ if (spec != NULL) {
+ struct sfc_mae_ethertype *ethertypes = pdata->ethertypes;
+ const struct rte_flow_item_vlan *item_spec;
+ const struct rte_flow_item_vlan *item_mask;
+
+ item_spec = (const struct rte_flow_item_vlan *)spec;
+ item_mask = (const struct rte_flow_item_vlan *)mask;
+
+ ethertypes[pdata->nb_vlan_tags].value = item_spec->inner_type;
+ ethertypes[pdata->nb_vlan_tags].mask = item_mask->inner_type;
+ } else {
+ /*
+ * The specification is empty. This is wrong in the case
+ * when there are more network patterns in line. Other
+ * than that, any Ethernet can match. All of that is
+ * checked at the end of parsing.
+ */
+ return 0;
+ }
+
+ return sfc_mae_parse_item(flocs, nb_flocs, spec, mask, ctx_mae, error);
+}
+
+static const struct sfc_mae_field_locator flocs_ipv4[] = {
+ {
+ EFX_MAE_FIELD_SRC_IP4_BE,
+ RTE_SIZEOF_FIELD(struct rte_flow_item_ipv4, hdr.src_addr),
+ offsetof(struct rte_flow_item_ipv4, hdr.src_addr),
+ },
+ {
+ EFX_MAE_FIELD_DST_IP4_BE,
+ RTE_SIZEOF_FIELD(struct rte_flow_item_ipv4, hdr.dst_addr),
+ offsetof(struct rte_flow_item_ipv4, hdr.dst_addr),
+ },
+ {
+ /*
+ * This locator is used only for building supported fields mask.
+ * The field is handled by sfc_mae_rule_process_pattern_data().
+ */
+ SFC_MAE_FIELD_HANDLING_DEFERRED,
+ RTE_SIZEOF_FIELD(struct rte_flow_item_ipv4, hdr.next_proto_id),
+ offsetof(struct rte_flow_item_ipv4, hdr.next_proto_id),
+ },
+ {
+ EFX_MAE_FIELD_IP_TOS,
+ RTE_SIZEOF_FIELD(struct rte_flow_item_ipv4,
+ hdr.type_of_service),
+ offsetof(struct rte_flow_item_ipv4, hdr.type_of_service),
+ },
+ {
+ EFX_MAE_FIELD_IP_TTL,
+ RTE_SIZEOF_FIELD(struct rte_flow_item_ipv4, hdr.time_to_live),
+ offsetof(struct rte_flow_item_ipv4, hdr.time_to_live),
+ },
+};
+
+static int
+sfc_mae_rule_parse_item_ipv4(const struct rte_flow_item *item,
+ struct sfc_flow_parse_ctx *ctx,
+ struct rte_flow_error *error)
+{
+ rte_be16_t ethertype_ipv4_be = RTE_BE16(RTE_ETHER_TYPE_IPV4);
+ struct sfc_mae_parse_ctx *ctx_mae = ctx->mae;
+ struct sfc_mae_pattern_data *pdata = &ctx_mae->pattern_data;
+ struct rte_flow_item_ipv4 supp_mask;
+ const uint8_t *spec = NULL;
+ const uint8_t *mask = NULL;
+ int rc;
+
+ sfc_mae_item_build_supp_mask(flocs_ipv4, RTE_DIM(flocs_ipv4),
+ &supp_mask, sizeof(supp_mask));
+
+ rc = sfc_flow_parse_init(item,
+ (const void **)&spec, (const void **)&mask,
+ (const void *)&supp_mask,
+ &rte_flow_item_ipv4_mask,
+ sizeof(struct rte_flow_item_ipv4), error);
+ if (rc != 0)
+ return rc;
+
+ pdata->innermost_ethertype_restriction.value = ethertype_ipv4_be;
+ pdata->innermost_ethertype_restriction.mask = RTE_BE16(0xffff);
+
+ if (spec != NULL) {
+ const struct rte_flow_item_ipv4 *item_spec;
+ const struct rte_flow_item_ipv4 *item_mask;
+
+ item_spec = (const struct rte_flow_item_ipv4 *)spec;
+ item_mask = (const struct rte_flow_item_ipv4 *)mask;
+
+ pdata->l3_next_proto_value = item_spec->hdr.next_proto_id;
+ pdata->l3_next_proto_mask = item_mask->hdr.next_proto_id;
+ } else {
+ return 0;
+ }
+
+ return sfc_mae_parse_item(flocs_ipv4, RTE_DIM(flocs_ipv4), spec, mask,
+ ctx_mae, error);
+}
+
+static const struct sfc_mae_field_locator flocs_ipv6[] = {
+ {
+ EFX_MAE_FIELD_SRC_IP6_BE,
+ RTE_SIZEOF_FIELD(struct rte_flow_item_ipv6, hdr.src_addr),
+ offsetof(struct rte_flow_item_ipv6, hdr.src_addr),
+ },
+ {
+ EFX_MAE_FIELD_DST_IP6_BE,
+ RTE_SIZEOF_FIELD(struct rte_flow_item_ipv6, hdr.dst_addr),
+ offsetof(struct rte_flow_item_ipv6, hdr.dst_addr),
+ },
+ {
+ /*
+ * This locator is used only for building supported fields mask.
+ * The field is handled by sfc_mae_rule_process_pattern_data().
+ */
+ SFC_MAE_FIELD_HANDLING_DEFERRED,
+ RTE_SIZEOF_FIELD(struct rte_flow_item_ipv6, hdr.proto),
+ offsetof(struct rte_flow_item_ipv6, hdr.proto),
+ },
+ {
+ EFX_MAE_FIELD_IP_TTL,
+ RTE_SIZEOF_FIELD(struct rte_flow_item_ipv6, hdr.hop_limits),
+ offsetof(struct rte_flow_item_ipv6, hdr.hop_limits),
+ },
+};
+
+static int
+sfc_mae_rule_parse_item_ipv6(const struct rte_flow_item *item,
+ struct sfc_flow_parse_ctx *ctx,
+ struct rte_flow_error *error)
+{
+ rte_be16_t ethertype_ipv6_be = RTE_BE16(RTE_ETHER_TYPE_IPV6);
+ struct sfc_mae_parse_ctx *ctx_mae = ctx->mae;
+ const efx_mae_field_id_t *fremap = ctx_mae->field_ids_remap;
+ struct sfc_mae_pattern_data *pdata = &ctx_mae->pattern_data;
+ struct rte_flow_item_ipv6 supp_mask;
+ const uint8_t *spec = NULL;
+ const uint8_t *mask = NULL;
+ rte_be32_t vtc_flow_be;
+ uint32_t vtc_flow;
+ uint8_t tc_value;
+ uint8_t tc_mask;
+ int rc;
+
+ sfc_mae_item_build_supp_mask(flocs_ipv6, RTE_DIM(flocs_ipv6),
+ &supp_mask, sizeof(supp_mask));
+
+ vtc_flow_be = RTE_BE32(RTE_IPV6_HDR_TC_MASK);
+ memcpy(&supp_mask, &vtc_flow_be, sizeof(vtc_flow_be));
+
+ rc = sfc_flow_parse_init(item,
+ (const void **)&spec, (const void **)&mask,
+ (const void *)&supp_mask,
+ &rte_flow_item_ipv6_mask,
+ sizeof(struct rte_flow_item_ipv6), error);
+ if (rc != 0)
+ return rc;
+
+ pdata->innermost_ethertype_restriction.value = ethertype_ipv6_be;
+ pdata->innermost_ethertype_restriction.mask = RTE_BE16(0xffff);
+
+ if (spec != NULL) {
+ const struct rte_flow_item_ipv6 *item_spec;
+ const struct rte_flow_item_ipv6 *item_mask;
+
+ item_spec = (const struct rte_flow_item_ipv6 *)spec;
+ item_mask = (const struct rte_flow_item_ipv6 *)mask;
+
+ pdata->l3_next_proto_value = item_spec->hdr.proto;
+ pdata->l3_next_proto_mask = item_mask->hdr.proto;
+ } else {
+ return 0;
+ }
+
+ rc = sfc_mae_parse_item(flocs_ipv6, RTE_DIM(flocs_ipv6), spec, mask,
+ ctx_mae, error);
+ if (rc != 0)
+ return rc;
+
+ memcpy(&vtc_flow_be, spec, sizeof(vtc_flow_be));
+ vtc_flow = rte_be_to_cpu_32(vtc_flow_be);
+ tc_value = (vtc_flow & RTE_IPV6_HDR_TC_MASK) >> RTE_IPV6_HDR_TC_SHIFT;
+
+ memcpy(&vtc_flow_be, mask, sizeof(vtc_flow_be));
+ vtc_flow = rte_be_to_cpu_32(vtc_flow_be);
+ tc_mask = (vtc_flow & RTE_IPV6_HDR_TC_MASK) >> RTE_IPV6_HDR_TC_SHIFT;
+
+ rc = efx_mae_match_spec_field_set(ctx_mae->match_spec,
+ fremap[EFX_MAE_FIELD_IP_TOS],
+ sizeof(tc_value), &tc_value,
+ sizeof(tc_mask), &tc_mask);
+ if (rc != 0) {
+ return rte_flow_error_set(error, rc, RTE_FLOW_ERROR_TYPE_ITEM,
+ NULL, "Failed to process item fields");
+ }
+
+ return 0;
+}
+
+static const struct sfc_mae_field_locator flocs_tcp[] = {
+ {
+ EFX_MAE_FIELD_L4_SPORT_BE,
+ RTE_SIZEOF_FIELD(struct rte_flow_item_tcp, hdr.src_port),
+ offsetof(struct rte_flow_item_tcp, hdr.src_port),
+ },
+ {
+ EFX_MAE_FIELD_L4_DPORT_BE,
+ RTE_SIZEOF_FIELD(struct rte_flow_item_tcp, hdr.dst_port),
+ offsetof(struct rte_flow_item_tcp, hdr.dst_port),
+ },
+ {
+ EFX_MAE_FIELD_TCP_FLAGS_BE,
+ /*
+ * The values have been picked intentionally since the
+ * target MAE field is oversize (16 bit). This mapping
+ * relies on the fact that the MAE field is big-endian.
+ */
+ RTE_SIZEOF_FIELD(struct rte_flow_item_tcp, hdr.data_off) +
+ RTE_SIZEOF_FIELD(struct rte_flow_item_tcp, hdr.tcp_flags),
+ offsetof(struct rte_flow_item_tcp, hdr.data_off),
+ },
+};
+
+static int
+sfc_mae_rule_parse_item_tcp(const struct rte_flow_item *item,
+ struct sfc_flow_parse_ctx *ctx,
+ struct rte_flow_error *error)
+{
+ struct sfc_mae_parse_ctx *ctx_mae = ctx->mae;
+ struct sfc_mae_pattern_data *pdata = &ctx_mae->pattern_data;
+ struct rte_flow_item_tcp supp_mask;
+ const uint8_t *spec = NULL;
+ const uint8_t *mask = NULL;
+ int rc;
+
+ /*
+ * When encountered among outermost items, item TCP is invalid.
+ * Check which match specification is being constructed now.
+ */
+ if (ctx_mae->match_spec != ctx_mae->match_spec_action) {
+ return rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM, item,
+ "TCP in outer frame is invalid");
+ }
+
+ sfc_mae_item_build_supp_mask(flocs_tcp, RTE_DIM(flocs_tcp),
+ &supp_mask, sizeof(supp_mask));
+
+ rc = sfc_flow_parse_init(item,
+ (const void **)&spec, (const void **)&mask,
+ (const void *)&supp_mask,
+ &rte_flow_item_tcp_mask,
+ sizeof(struct rte_flow_item_tcp), error);
+ if (rc != 0)
+ return rc;
+
+ pdata->l3_next_proto_restriction_value = IPPROTO_TCP;
+ pdata->l3_next_proto_restriction_mask = 0xff;
+
+ if (spec == NULL)
+ return 0;
+
+ return sfc_mae_parse_item(flocs_tcp, RTE_DIM(flocs_tcp), spec, mask,
+ ctx_mae, error);
+}
+
+static const struct sfc_mae_field_locator flocs_udp[] = {
+ {
+ EFX_MAE_FIELD_L4_SPORT_BE,
+ RTE_SIZEOF_FIELD(struct rte_flow_item_udp, hdr.src_port),
+ offsetof(struct rte_flow_item_udp, hdr.src_port),
+ },
+ {
+ EFX_MAE_FIELD_L4_DPORT_BE,
+ RTE_SIZEOF_FIELD(struct rte_flow_item_udp, hdr.dst_port),
+ offsetof(struct rte_flow_item_udp, hdr.dst_port),
+ },
+};
+
+static int
+sfc_mae_rule_parse_item_udp(const struct rte_flow_item *item,
+ struct sfc_flow_parse_ctx *ctx,
+ struct rte_flow_error *error)
+{
+ struct sfc_mae_parse_ctx *ctx_mae = ctx->mae;
+ struct sfc_mae_pattern_data *pdata = &ctx_mae->pattern_data;
+ struct rte_flow_item_udp supp_mask;
+ const uint8_t *spec = NULL;
+ const uint8_t *mask = NULL;
+ int rc;
+
+ sfc_mae_item_build_supp_mask(flocs_udp, RTE_DIM(flocs_udp),
+ &supp_mask, sizeof(supp_mask));
+
+ rc = sfc_flow_parse_init(item,
+ (const void **)&spec, (const void **)&mask,
+ (const void *)&supp_mask,
+ &rte_flow_item_udp_mask,
+ sizeof(struct rte_flow_item_udp), error);
+ if (rc != 0)
+ return rc;
+
+ pdata->l3_next_proto_restriction_value = IPPROTO_UDP;
+ pdata->l3_next_proto_restriction_mask = 0xff;
+
+ if (spec == NULL)
+ return 0;
+
+ return sfc_mae_parse_item(flocs_udp, RTE_DIM(flocs_udp), spec, mask,
+ ctx_mae, error);
+}
+
+static const struct sfc_mae_field_locator flocs_tunnel[] = {
+ {
+ /*
+ * The size and offset values are relevant
+ * for Geneve and NVGRE, too.
+ */
+ .size = RTE_SIZEOF_FIELD(struct rte_flow_item_vxlan, vni),
+ .ofst = offsetof(struct rte_flow_item_vxlan, vni),
+ },
+};
+
+/*
+ * An auxiliary registry which allows using non-encap. field IDs
+ * directly when building a match specification of type ACTION.
+ *
+ * See sfc_mae_rule_parse_pattern() and sfc_mae_rule_parse_item_tunnel().
+ */
+static const efx_mae_field_id_t field_ids_no_remap[] = {
+#define FIELD_ID_NO_REMAP(_field) \
+ [EFX_MAE_FIELD_##_field] = EFX_MAE_FIELD_##_field
+
+ FIELD_ID_NO_REMAP(ETHER_TYPE_BE),
+ FIELD_ID_NO_REMAP(ETH_SADDR_BE),
+ FIELD_ID_NO_REMAP(ETH_DADDR_BE),
+ FIELD_ID_NO_REMAP(VLAN0_TCI_BE),
+ FIELD_ID_NO_REMAP(VLAN0_PROTO_BE),
+ FIELD_ID_NO_REMAP(VLAN1_TCI_BE),
+ FIELD_ID_NO_REMAP(VLAN1_PROTO_BE),
+ FIELD_ID_NO_REMAP(SRC_IP4_BE),
+ FIELD_ID_NO_REMAP(DST_IP4_BE),
+ FIELD_ID_NO_REMAP(IP_PROTO),
+ FIELD_ID_NO_REMAP(IP_TOS),
+ FIELD_ID_NO_REMAP(IP_TTL),
+ FIELD_ID_NO_REMAP(SRC_IP6_BE),
+ FIELD_ID_NO_REMAP(DST_IP6_BE),
+ FIELD_ID_NO_REMAP(L4_SPORT_BE),
+ FIELD_ID_NO_REMAP(L4_DPORT_BE),
+ FIELD_ID_NO_REMAP(TCP_FLAGS_BE),
+
+#undef FIELD_ID_NO_REMAP
+};
+
+/*
+ * An auxiliary registry which allows using "ENC" field IDs
+ * when building a match specification of type OUTER.
+ *
+ * See sfc_mae_rule_encap_parse_init().
+ */
+static const efx_mae_field_id_t field_ids_remap_to_encap[] = {
+#define FIELD_ID_REMAP_TO_ENCAP(_field) \
+ [EFX_MAE_FIELD_##_field] = EFX_MAE_FIELD_ENC_##_field
+
+ FIELD_ID_REMAP_TO_ENCAP(ETHER_TYPE_BE),
+ FIELD_ID_REMAP_TO_ENCAP(ETH_SADDR_BE),
+ FIELD_ID_REMAP_TO_ENCAP(ETH_DADDR_BE),
+ FIELD_ID_REMAP_TO_ENCAP(VLAN0_TCI_BE),
+ FIELD_ID_REMAP_TO_ENCAP(VLAN0_PROTO_BE),
+ FIELD_ID_REMAP_TO_ENCAP(VLAN1_TCI_BE),
+ FIELD_ID_REMAP_TO_ENCAP(VLAN1_PROTO_BE),
+ FIELD_ID_REMAP_TO_ENCAP(SRC_IP4_BE),
+ FIELD_ID_REMAP_TO_ENCAP(DST_IP4_BE),
+ FIELD_ID_REMAP_TO_ENCAP(IP_PROTO),
+ FIELD_ID_REMAP_TO_ENCAP(IP_TOS),
+ FIELD_ID_REMAP_TO_ENCAP(IP_TTL),
+ FIELD_ID_REMAP_TO_ENCAP(SRC_IP6_BE),
+ FIELD_ID_REMAP_TO_ENCAP(DST_IP6_BE),
+ FIELD_ID_REMAP_TO_ENCAP(L4_SPORT_BE),
+ FIELD_ID_REMAP_TO_ENCAP(L4_DPORT_BE),
+
+#undef FIELD_ID_REMAP_TO_ENCAP
+};
+
+static int
+sfc_mae_rule_parse_item_tunnel(const struct rte_flow_item *item,
+ struct sfc_flow_parse_ctx *ctx,
+ struct rte_flow_error *error)
+{
+ struct sfc_mae_parse_ctx *ctx_mae = ctx->mae;
+ uint8_t vnet_id_v[sizeof(uint32_t)] = {0};
+ uint8_t vnet_id_m[sizeof(uint32_t)] = {0};
+ const struct rte_flow_item_vxlan *vxp;
+ uint8_t supp_mask[sizeof(uint64_t)];
+ const uint8_t *spec = NULL;
+ const uint8_t *mask = NULL;
+ const void *def_mask;
+ int rc;
+
+ /*
+ * We're about to start processing inner frame items.
+ * Process pattern data that has been deferred so far
+ * and reset pattern data storage.
+ */
+ rc = sfc_mae_rule_process_pattern_data(ctx_mae, error);
+ if (rc != 0)
+ return rc;
+
+ memset(&ctx_mae->pattern_data, 0, sizeof(ctx_mae->pattern_data));
+
+ sfc_mae_item_build_supp_mask(flocs_tunnel, RTE_DIM(flocs_tunnel),
+ &supp_mask, sizeof(supp_mask));
+
+ /*
+ * This tunnel item was preliminarily detected by
+ * sfc_mae_rule_encap_parse_init(). Default mask
+ * was also picked by that helper. Use it here.
+ */
+ def_mask = ctx_mae->tunnel_def_mask;
+
+ rc = sfc_flow_parse_init(item,
+ (const void **)&spec, (const void **)&mask,
+ (const void *)&supp_mask, def_mask,
+ sizeof(def_mask), error);
+ if (rc != 0)
+ return rc;
+
+ /*
+ * This item and later ones comprise a
+ * match specification of type ACTION.
+ */
+ ctx_mae->match_spec = ctx_mae->match_spec_action;
+
+ /* This item and later ones use non-encap. EFX MAE field IDs. */
+ ctx_mae->field_ids_remap = field_ids_no_remap;
+
+ if (spec == NULL)
+ return 0;
+
+ /*
+ * Field EFX_MAE_FIELD_ENC_VNET_ID_BE is a 32-bit one.
+ * Copy 24-bit VNI, which is BE, at offset 1 in it.
+ * The extra byte is 0 both in the mask and in the value.
+ */
+ vxp = (const struct rte_flow_item_vxlan *)spec;
+ memcpy(vnet_id_v + 1, &vxp->vni, sizeof(vxp->vni));
+
+ vxp = (const struct rte_flow_item_vxlan *)mask;
+ memcpy(vnet_id_m + 1, &vxp->vni, sizeof(vxp->vni));
+
+ rc = efx_mae_match_spec_field_set(ctx_mae->match_spec,
+ EFX_MAE_FIELD_ENC_VNET_ID_BE,
+ sizeof(vnet_id_v), vnet_id_v,
+ sizeof(vnet_id_m), vnet_id_m);
+ if (rc != 0) {
+ rc = rte_flow_error_set(error, rc, RTE_FLOW_ERROR_TYPE_ITEM,
+ item, "Failed to set VXLAN VNI");
+ }
+
+ return rc;