+ * @param priv
+ * Pointer to private structure.
+ * @param[in, out] parser
+ * Internal parser structure.
+ */
+static void
+priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser)
+{
+ const unsigned int ipv4 =
+ hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
+ const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6;
+ const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
+ const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4;
+ const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4;
+ const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
+ unsigned int i;
+
+ (void)priv;
+ if (parser->layer == HASH_RXQ_ETH) {
+ goto fill;
+ } else {
+ /*
+ * This layer becomes useless as the pattern define under
+ * layers.
+ */
+ rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr);
+ parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
+ }
+ /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */
+ for (i = ohmin; i != (ohmax + 1); ++i) {
+ if (!parser->queue[i].ibv_attr)
+ continue;
+ rte_free(parser->queue[i].ibv_attr);
+ parser->queue[i].ibv_attr = NULL;
+ }
+ /* Remove impossible flow according to the RSS configuration. */
+ if (hash_rxq_init[parser->layer].dpdk_rss_hf &
+ parser->rss_conf.rss_hf) {
+ /* Remove any other flow. */
+ for (i = hmin; i != (hmax + 1); ++i) {
+ if ((i == parser->layer) ||
+ (!parser->queue[i].ibv_attr))
+ continue;
+ rte_free(parser->queue[i].ibv_attr);
+ parser->queue[i].ibv_attr = NULL;
+ }
+ } else if (!parser->queue[ip].ibv_attr) {
+ /* no RSS possible with the current configuration. */
+ parser->queues_n = 1;
+ return;
+ }
+fill:
+ /*
+ * Fill missing layers in verbs specifications, or compute the correct
+ * offset to allocate the memory space for the attributes and
+ * specifications.
+ */
+ for (i = 0; i != hash_rxq_init_n - 1; ++i) {
+ union {
+ struct ibv_flow_spec_ipv4_ext ipv4;
+ struct ibv_flow_spec_ipv6 ipv6;
+ struct ibv_flow_spec_tcp_udp udp_tcp;
+ } specs;
+ void *dst;
+ uint16_t size;
+
+ if (i == parser->layer)
+ continue;
+ if (parser->layer == HASH_RXQ_ETH) {
+ if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
+ size = sizeof(struct ibv_flow_spec_ipv4_ext);
+ specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
+ .type = IBV_FLOW_SPEC_IPV4_EXT,
+ .size = size,
+ };
+ } else {
+ size = sizeof(struct ibv_flow_spec_ipv6);
+ specs.ipv6 = (struct ibv_flow_spec_ipv6){
+ .type = IBV_FLOW_SPEC_IPV6,
+ .size = size,
+ };
+ }
+ if (parser->queue[i].ibv_attr) {
+ dst = (void *)((uintptr_t)
+ parser->queue[i].ibv_attr +
+ parser->queue[i].offset);
+ memcpy(dst, &specs, size);
+ ++parser->queue[i].ibv_attr->num_of_specs;
+ }
+ parser->queue[i].offset += size;
+ }
+ if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
+ (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
+ size = sizeof(struct ibv_flow_spec_tcp_udp);
+ specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
+ .type = ((i == HASH_RXQ_UDPV4 ||
+ i == HASH_RXQ_UDPV6) ?
+ IBV_FLOW_SPEC_UDP :
+ IBV_FLOW_SPEC_TCP),
+ .size = size,
+ };
+ if (parser->queue[i].ibv_attr) {
+ dst = (void *)((uintptr_t)
+ parser->queue[i].ibv_attr +
+ parser->queue[i].offset);
+ memcpy(dst, &specs, size);
+ ++parser->queue[i].ibv_attr->num_of_specs;
+ }
+ parser->queue[i].offset += size;
+ }
+ }
+}
+
+/**
+ * Validate and convert a flow supported by the NIC.
+ *
+ * @param priv
+ * Pointer to private structure.
+ * @param[in] attr
+ * Flow rule attributes.
+ * @param[in] pattern
+ * Pattern specification (list terminated by the END pattern item).
+ * @param[in] actions
+ * Associated actions (list terminated by the END action).
+ * @param[out] error
+ * Perform verbose error reporting if not NULL.
+ * @param[in, out] parser
+ * Internal parser structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+priv_flow_convert(struct priv *priv,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ struct rte_flow_error *error,
+ struct mlx5_flow_parse *parser)
+{
+ const struct mlx5_flow_items *cur_item = mlx5_flow_items;
+ unsigned int i;
+ int ret;
+
+ /* First step. Validate the attributes, items and actions. */
+ *parser = (struct mlx5_flow_parse){
+ .create = parser->create,
+ .layer = HASH_RXQ_ETH,
+ .mark_id = MLX5_FLOW_MARK_DEFAULT,
+ };
+ ret = priv_flow_convert_attributes(priv, attr, error, parser);
+ if (ret)
+ return ret;
+ ret = priv_flow_convert_actions(priv, actions, error, parser);
+ if (ret)
+ return ret;
+ ret = priv_flow_convert_items_validate(priv, items, error, parser);
+ if (ret)
+ return ret;
+ priv_flow_convert_finalise(priv, parser);
+ /*
+ * Second step.
+ * Allocate the memory space to store verbs specifications.
+ */
+ if (parser->drop || parser->queues_n == 1) {
+ unsigned int priority =
+ attr->priority +
+ hash_rxq_init[HASH_RXQ_ETH].flow_priority;
+ unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
+
+ parser->queue[HASH_RXQ_ETH].ibv_attr =
+ priv_flow_convert_allocate(priv, priority,
+ offset, error);
+ if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
+ return ENOMEM;
+ parser->queue[HASH_RXQ_ETH].offset =
+ sizeof(struct ibv_flow_attr);
+ } else {
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ unsigned int priority =
+ attr->priority +
+ hash_rxq_init[i].flow_priority;
+ unsigned int offset;
+
+ if (!(parser->rss_conf.rss_hf &
+ hash_rxq_init[i].dpdk_rss_hf) &&
+ (i != HASH_RXQ_ETH))
+ continue;
+ offset = parser->queue[i].offset;
+ parser->queue[i].ibv_attr =
+ priv_flow_convert_allocate(priv, priority,
+ offset, error);
+ if (!parser->queue[i].ibv_attr)
+ goto exit_enomem;
+ parser->queue[i].offset = sizeof(struct ibv_flow_attr);
+ }
+ }
+ /* Third step. Conversion parse, fill the specifications. */
+ parser->inner = 0;
+ for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
+ if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
+ continue;
+ cur_item = &mlx5_flow_items[items->type];
+ ret = cur_item->convert(items,
+ (cur_item->default_mask ?
+ cur_item->default_mask :
+ cur_item->mask),
+ parser);
+ if (ret) {
+ rte_flow_error_set(error, ret,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ items, "item not supported");
+ goto exit_free;
+ }
+ }
+ if (parser->mark)
+ mlx5_flow_create_flag_mark(parser, parser->mark_id);
+ if (parser->count && parser->create) {
+ mlx5_flow_create_count(priv, parser);
+ if (!parser->cs)
+ goto exit_count_error;
+ }
+ /*
+ * Last step. Complete missing specification to reach the RSS
+ * configuration.
+ */
+ if (parser->queues_n > 1) {
+ priv_flow_convert_finalise(priv, parser);
+ } else {
+ /*
+ * Action queue have their priority overridden with
+ * Ethernet priority, this priority needs to be adjusted to
+ * their most specific layer priority.
+ */
+ parser->queue[HASH_RXQ_ETH].ibv_attr->priority =
+ attr->priority +
+ hash_rxq_init[parser->layer].flow_priority;
+ }
+exit_free:
+ /* Only verification is expected, all resources should be released. */
+ if (!parser->create) {
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ if (parser->queue[i].ibv_attr) {
+ rte_free(parser->queue[i].ibv_attr);
+ parser->queue[i].ibv_attr = NULL;
+ }
+ }
+ }
+ if (parser->allmulti &&
+ parser->layer == HASH_RXQ_ETH) {
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ if (!parser->queue[i].ibv_attr)
+ continue;
+ if (parser->queue[i].ibv_attr->num_of_specs != 1)
+ break;
+ parser->queue[i].ibv_attr->type =
+ IBV_FLOW_ATTR_MC_DEFAULT;
+ }
+ }
+ return ret;
+exit_enomem:
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ if (parser->queue[i].ibv_attr) {
+ rte_free(parser->queue[i].ibv_attr);
+ parser->queue[i].ibv_attr = NULL;
+ }
+ }
+ rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL, "cannot allocate verbs spec attributes.");
+ return ret;
+exit_count_error:
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL, "cannot create counter.");
+ return rte_errno;
+}
+
+/**
+ * Copy the specification created into the flow.
+ *
+ * @param parser
+ * Internal parser structure.
+ * @param src
+ * Create specification.
+ * @param size
+ * Size in bytes of the specification to copy.
+ */
+static void
+mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
+ unsigned int size)
+{
+ unsigned int i;
+ void *dst;
+
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ if (!parser->queue[i].ibv_attr)
+ continue;
+ /* Specification must be the same l3 type or none. */
+ if (parser->layer == HASH_RXQ_ETH ||
+ (hash_rxq_init[parser->layer].ip_version ==
+ hash_rxq_init[i].ip_version) ||
+ (hash_rxq_init[i].ip_version == 0)) {
+ dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
+ parser->queue[i].offset);
+ memcpy(dst, src, size);
+ ++parser->queue[i].ibv_attr->num_of_specs;
+ parser->queue[i].offset += size;
+ }
+ }
+}
+
+/**
+ * Convert Ethernet item to Verbs specification.
+ *
+ * @param item[in]
+ * Item specification.
+ * @param default_mask[in]
+ * Default bit-masks to use when item->mask is not provided.
+ * @param data[in, out]
+ * User structure.
+ */
+static int
+mlx5_flow_create_eth(const struct rte_flow_item *item,
+ const void *default_mask,
+ void *data)
+{
+ const struct rte_flow_item_eth *spec = item->spec;
+ const struct rte_flow_item_eth *mask = item->mask;
+ struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
+ const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
+ struct ibv_flow_spec_eth eth = {
+ .type = parser->inner | IBV_FLOW_SPEC_ETH,
+ .size = eth_size,
+ };
+
+ /* Don't update layer for the inner pattern. */
+ if (!parser->inner)
+ parser->layer = HASH_RXQ_ETH;
+ if (spec) {
+ unsigned int i;
+
+ if (!mask)
+ mask = default_mask;
+ memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
+ memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
+ eth.val.ether_type = spec->type;
+ memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
+ memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
+ eth.mask.ether_type = mask->type;
+ /* Remove unwanted bits from values. */
+ for (i = 0; i < ETHER_ADDR_LEN; ++i) {
+ eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
+ eth.val.src_mac[i] &= eth.mask.src_mac[i];
+ }
+ eth.val.ether_type &= eth.mask.ether_type;
+ }
+ mlx5_flow_create_copy(parser, ð, eth_size);
+ parser->allmulti = eth.val.dst_mac[0] & 1;
+ return 0;
+}
+
+/**
+ * Convert VLAN item to Verbs specification.
+ *
+ * @param item[in]
+ * Item specification.
+ * @param default_mask[in]
+ * Default bit-masks to use when item->mask is not provided.
+ * @param data[in, out]
+ * User structure.
+ */
+static int
+mlx5_flow_create_vlan(const struct rte_flow_item *item,
+ const void *default_mask,
+ void *data)
+{
+ const struct rte_flow_item_vlan *spec = item->spec;
+ const struct rte_flow_item_vlan *mask = item->mask;
+ struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
+ struct ibv_flow_spec_eth *eth;
+ const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
+
+ if (spec) {
+ unsigned int i;
+ if (!mask)
+ mask = default_mask;
+
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ if (!parser->queue[i].ibv_attr)
+ continue;
+
+ eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
+ parser->queue[i].offset - eth_size);
+ eth->val.vlan_tag = spec->tci;
+ eth->mask.vlan_tag = mask->tci;
+ eth->val.vlan_tag &= eth->mask.vlan_tag;
+ }
+ }
+ return 0;
+}
+
+/**
+ * Convert IPv4 item to Verbs specification.
+ *
+ * @param item[in]