-/**
- * Check support for a given item.
- *
- * @param item[in]
- * Item specification.
- * @param mask[in]
- * Bit-masks covering supported fields to compare with spec, last and mask in
- * \item.
- * @param size
- * Bit-Mask size in bytes.
- *
- * @return
- * 0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_item_validate(const struct rte_flow_item *item,
- const uint8_t *mask, unsigned int size)
-{
- if (!item->spec && (item->mask || item->last)) {
- rte_errno = EINVAL;
- return -rte_errno;
- }
- if (item->spec && !item->mask) {
- unsigned int i;
- const uint8_t *spec = item->spec;
-
- for (i = 0; i < size; ++i)
- if ((spec[i] | mask[i]) != mask[i]) {
- rte_errno = EINVAL;
- return -rte_errno;
- }
- }
- if (item->last && !item->mask) {
- unsigned int i;
- const uint8_t *spec = item->last;
-
- for (i = 0; i < size; ++i)
- if ((spec[i] | mask[i]) != mask[i]) {
- rte_errno = EINVAL;
- return -rte_errno;
- }
- }
- if (item->mask) {
- unsigned int i;
- const uint8_t *spec = item->spec;
-
- for (i = 0; i < size; ++i)
- if ((spec[i] | mask[i]) != mask[i]) {
- rte_errno = EINVAL;
- return -rte_errno;
- }
- }
- if (item->spec && item->last) {
- uint8_t spec[size];
- uint8_t last[size];
- const uint8_t *apply = mask;
- unsigned int i;
- int ret;
-
- if (item->mask)
- apply = item->mask;
- for (i = 0; i < size; ++i) {
- spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
- last[i] = ((const uint8_t *)item->last)[i] & apply[i];
- }
- ret = memcmp(spec, last, size);
- if (ret != 0) {
- rte_errno = EINVAL;
- return -rte_errno;
- }
- }
- return 0;
-}
-
-/**
- * Copy the RSS configuration from the user ones, of the rss_conf is null,
- * uses the driver one.
- *
- * @param parser
- * Internal parser structure.
- * @param rss_conf
- * User RSS configuration to save.
- *
- * @return
- * 0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_rss_conf(struct mlx5_flow_parse *parser,
- const struct rte_eth_rss_conf *rss_conf)
-{
- /*
- * This function is also called at the beginning of
- * mlx5_flow_convert_actions() to initialize the parser with the
- * device default RSS configuration.
- */
- if (rss_conf) {
- if (rss_conf->rss_hf & MLX5_RSS_HF_MASK) {
- rte_errno = EINVAL;
- return -rte_errno;
- }
- if (rss_conf->rss_key_len != 40) {
- rte_errno = EINVAL;
- return -rte_errno;
- }
- if (rss_conf->rss_key_len && rss_conf->rss_key) {
- parser->rss_conf.rss_key_len = rss_conf->rss_key_len;
- memcpy(parser->rss_key, rss_conf->rss_key,
- rss_conf->rss_key_len);
- parser->rss_conf.rss_key = parser->rss_key;
- }
- parser->rss_conf.rss_hf = rss_conf->rss_hf;
- }
- return 0;
-}
-
-/**
- * Extract attribute to the parser.
- *
- * @param[in] attr
- * Flow rule attributes.
- * @param[out] error
- * Perform verbose error reporting if not NULL.
- *
- * @return
- * 0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_attributes(const struct rte_flow_attr *attr,
- struct rte_flow_error *error)
-{
- if (attr->group) {
- rte_flow_error_set(error, ENOTSUP,
- RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
- NULL,
- "groups are not supported");
- return -rte_errno;
- }
- if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
- rte_flow_error_set(error, ENOTSUP,
- RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
- NULL,
- "priorities are not supported");
- return -rte_errno;
- }
- if (attr->egress) {
- rte_flow_error_set(error, ENOTSUP,
- RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
- NULL,
- "egress is not supported");
- return -rte_errno;
- }
- if (!attr->ingress) {
- rte_flow_error_set(error, ENOTSUP,
- RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
- NULL,
- "only ingress is supported");
- return -rte_errno;
- }
- return 0;
-}
-
-/**
- * Extract actions request to the parser.
- *
- * @param dev
- * Pointer to Ethernet device.
- * @param[in] actions
- * Associated actions (list terminated by the END action).
- * @param[out] error
- * Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- * Internal parser structure.
- *
- * @return
- * 0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_actions(struct rte_eth_dev *dev,
- const struct rte_flow_action actions[],
- struct rte_flow_error *error,
- struct mlx5_flow_parse *parser)
-{
- struct priv *priv = dev->data->dev_private;
- int ret;
-
- /*
- * Add default RSS configuration necessary for Verbs to create QP even
- * if no RSS is necessary.
- */
- ret = mlx5_flow_convert_rss_conf(parser,
- (const struct rte_eth_rss_conf *)
- &priv->rss_conf);
- if (ret)
- return ret;
- for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
- if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
- continue;
- } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
- parser->drop = 1;
- } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
- const struct rte_flow_action_queue *queue =
- (const struct rte_flow_action_queue *)
- actions->conf;
- uint16_t n;
- uint16_t found = 0;
-
- if (!queue || (queue->index > (priv->rxqs_n - 1)))
- goto exit_action_not_supported;
- for (n = 0; n < parser->queues_n; ++n) {
- if (parser->queues[n] == queue->index) {
- found = 1;
- break;
- }
- }
- if (parser->queues_n > 1 && !found) {
- rte_flow_error_set(error, ENOTSUP,
- RTE_FLOW_ERROR_TYPE_ACTION,
- actions,
- "queue action not in RSS queues");
- return -rte_errno;
- }
- if (!found) {
- parser->queues_n = 1;
- parser->queues[0] = queue->index;
- }
- } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
- const struct rte_flow_action_rss *rss =
- (const struct rte_flow_action_rss *)
- actions->conf;
- uint16_t n;
-
- if (!rss || !rss->num) {
- rte_flow_error_set(error, EINVAL,
- RTE_FLOW_ERROR_TYPE_ACTION,
- actions,
- "no valid queues");
- return -rte_errno;
- }
- if (parser->queues_n == 1) {
- uint16_t found = 0;
-
- assert(parser->queues_n);
- for (n = 0; n < rss->num; ++n) {
- if (parser->queues[0] ==
- rss->queue[n]) {
- found = 1;
- break;
- }
- }
- if (!found) {
- rte_flow_error_set(error, ENOTSUP,
- RTE_FLOW_ERROR_TYPE_ACTION,
- actions,
- "queue action not in RSS"
- " queues");
- return -rte_errno;
- }
- }
- for (n = 0; n < rss->num; ++n) {
- if (rss->queue[n] >= priv->rxqs_n) {
- rte_flow_error_set(error, EINVAL,
- RTE_FLOW_ERROR_TYPE_ACTION,
- actions,
- "queue id > number of"
- " queues");
- return -rte_errno;
- }
- }
- for (n = 0; n < rss->num; ++n)
- parser->queues[n] = rss->queue[n];
- parser->queues_n = rss->num;
- if (mlx5_flow_convert_rss_conf(parser, rss->rss_conf)) {
- rte_flow_error_set(error, EINVAL,
- RTE_FLOW_ERROR_TYPE_ACTION,
- actions,
- "wrong RSS configuration");
- return -rte_errno;
- }
- } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
- const struct rte_flow_action_mark *mark =
- (const struct rte_flow_action_mark *)
- actions->conf;
-
- if (!mark) {
- rte_flow_error_set(error, EINVAL,
- RTE_FLOW_ERROR_TYPE_ACTION,
- actions,
- "mark must be defined");
- return -rte_errno;
- } else if (mark->id >= MLX5_FLOW_MARK_MAX) {
- rte_flow_error_set(error, ENOTSUP,
- RTE_FLOW_ERROR_TYPE_ACTION,
- actions,
- "mark must be between 0"
- " and 16777199");
- return -rte_errno;
- }
- parser->mark = 1;
- parser->mark_id = mark->id;
- } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
- parser->mark = 1;
- } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
- priv->config.flow_counter_en) {
- parser->count = 1;
- } else {
- goto exit_action_not_supported;
- }
- }
- if (parser->drop && parser->mark)
- parser->mark = 0;
- if (!parser->queues_n && !parser->drop) {
- rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
- NULL, "no valid action");
- return -rte_errno;
- }
- return 0;
-exit_action_not_supported:
- rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
- actions, "action not supported");
- return -rte_errno;
-}
-
-/**
- * Validate items.
- *
- * @param[in] items
- * Pattern specification (list terminated by the END pattern item).
- * @param[out] error
- * Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- * Internal parser structure.
- *
- * @return
- * 0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_items_validate(const struct rte_flow_item items[],
- struct rte_flow_error *error,
- struct mlx5_flow_parse *parser)
-{
- const struct mlx5_flow_items *cur_item = mlx5_flow_items;
- unsigned int i;
- int ret = 0;
-
- /* Initialise the offsets to start after verbs attribute. */
- for (i = 0; i != hash_rxq_init_n; ++i)
- parser->queue[i].offset = sizeof(struct ibv_flow_attr);
- for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
- const struct mlx5_flow_items *token = NULL;
- unsigned int n;
-
- if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
- continue;
- for (i = 0;
- cur_item->items &&
- cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
- ++i) {
- if (cur_item->items[i] == items->type) {
- token = &mlx5_flow_items[items->type];
- break;
- }
- }
- if (!token)
- goto exit_item_not_supported;
- cur_item = token;
- ret = mlx5_flow_item_validate(items,
- (const uint8_t *)cur_item->mask,
- cur_item->mask_sz);
- if (ret)
- goto exit_item_not_supported;
- if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
- if (parser->inner) {
- rte_flow_error_set(error, ENOTSUP,
- RTE_FLOW_ERROR_TYPE_ITEM,
- items,
- "cannot recognize multiple"
- " VXLAN encapsulations");
- return -rte_errno;
- }
- parser->inner = IBV_FLOW_SPEC_INNER;
- }
- if (parser->drop) {
- parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
- } else {
- for (n = 0; n != hash_rxq_init_n; ++n)
- parser->queue[n].offset += cur_item->dst_sz;
- }
- }
- if (parser->drop) {
- parser->queue[HASH_RXQ_ETH].offset +=
- sizeof(struct ibv_flow_spec_action_drop);
- }
- if (parser->mark) {
- for (i = 0; i != hash_rxq_init_n; ++i)
- parser->queue[i].offset +=
- sizeof(struct ibv_flow_spec_action_tag);
- }
- if (parser->count) {
- unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
-
- for (i = 0; i != hash_rxq_init_n; ++i)
- parser->queue[i].offset += size;
- }
- return 0;
-exit_item_not_supported:
- return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
- items, "item not supported");
-}
-
-/**
- * Allocate memory space to store verbs flow attributes.
- *
- * @param[in] size
- * Amount of byte to allocate.
- * @param[out] error
- * Perform verbose error reporting if not NULL.
- *
- * @return
- * A verbs flow attribute on success, NULL otherwise and rte_errno is set.
- */
-static struct ibv_flow_attr *
-mlx5_flow_convert_allocate(unsigned int size, struct rte_flow_error *error)
-{
- struct ibv_flow_attr *ibv_attr;
-
- ibv_attr = rte_calloc(__func__, 1, size, 0);
- if (!ibv_attr) {
- rte_flow_error_set(error, ENOMEM,
- RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
- NULL,
- "cannot allocate verbs spec attributes");
- return NULL;
- }
- return ibv_attr;
-}
-
-/**
- * Make inner packet matching with an higher priority from the non Inner
- * matching.
- *
- * @param[in, out] parser
- * Internal parser structure.
- * @param attr
- * User flow attribute.
- */
-static void
-mlx5_flow_update_priority(struct mlx5_flow_parse *parser,
- const struct rte_flow_attr *attr)
-{
- unsigned int i;
-
- if (parser->drop) {
- parser->queue[HASH_RXQ_ETH].ibv_attr->priority =
- attr->priority +
- hash_rxq_init[HASH_RXQ_ETH].flow_priority;
- return;
- }
- for (i = 0; i != hash_rxq_init_n; ++i) {
- if (parser->queue[i].ibv_attr) {
- parser->queue[i].ibv_attr->priority =
- attr->priority +
- hash_rxq_init[i].flow_priority -
- (parser->inner ? 1 : 0);
- }
- }
-}
-
-/**
- * Finalise verbs flow attributes.
- *
- * @param[in, out] parser
- * Internal parser structure.
- */
-static void
-mlx5_flow_convert_finalise(struct mlx5_flow_parse *parser)
-{
- const unsigned int ipv4 =
- hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
- const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6;
- const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
- const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4;
- const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4;
- const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
- unsigned int i;
-
- /* Remove any other flow not matching the pattern. */
- if (parser->queues_n == 1 && !parser->rss_conf.rss_hf) {
- for (i = 0; i != hash_rxq_init_n; ++i) {
- if (i == HASH_RXQ_ETH)
- continue;
- rte_free(parser->queue[i].ibv_attr);
- parser->queue[i].ibv_attr = NULL;
- }
- return;
- }
- if (parser->layer == HASH_RXQ_ETH) {
- goto fill;
- } else {
- /*
- * This layer becomes useless as the pattern define under
- * layers.
- */
- rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr);
- parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
- }
- /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */
- for (i = ohmin; i != (ohmax + 1); ++i) {
- if (!parser->queue[i].ibv_attr)
- continue;
- rte_free(parser->queue[i].ibv_attr);
- parser->queue[i].ibv_attr = NULL;
- }
- /* Remove impossible flow according to the RSS configuration. */
- if (hash_rxq_init[parser->layer].dpdk_rss_hf &
- parser->rss_conf.rss_hf) {
- /* Remove any other flow. */
- for (i = hmin; i != (hmax + 1); ++i) {
- if ((i == parser->layer) ||
- (!parser->queue[i].ibv_attr))
- continue;
- rte_free(parser->queue[i].ibv_attr);
- parser->queue[i].ibv_attr = NULL;
- }
- } else if (!parser->queue[ip].ibv_attr) {
- /* no RSS possible with the current configuration. */
- parser->queues_n = 1;
- return;
- }
-fill:
- /*
- * Fill missing layers in verbs specifications, or compute the correct
- * offset to allocate the memory space for the attributes and
- * specifications.
- */
- for (i = 0; i != hash_rxq_init_n - 1; ++i) {
- union {
- struct ibv_flow_spec_ipv4_ext ipv4;
- struct ibv_flow_spec_ipv6 ipv6;
- struct ibv_flow_spec_tcp_udp udp_tcp;
- } specs;
- void *dst;
- uint16_t size;
-
- if (i == parser->layer)
- continue;
- if (parser->layer == HASH_RXQ_ETH) {
- if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
- size = sizeof(struct ibv_flow_spec_ipv4_ext);
- specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
- .type = IBV_FLOW_SPEC_IPV4_EXT,
- .size = size,
- };
- } else {
- size = sizeof(struct ibv_flow_spec_ipv6);
- specs.ipv6 = (struct ibv_flow_spec_ipv6){
- .type = IBV_FLOW_SPEC_IPV6,
- .size = size,
- };
- }
- if (parser->queue[i].ibv_attr) {
- dst = (void *)((uintptr_t)
- parser->queue[i].ibv_attr +
- parser->queue[i].offset);
- memcpy(dst, &specs, size);
- ++parser->queue[i].ibv_attr->num_of_specs;
- }
- parser->queue[i].offset += size;
- }
- if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
- (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
- size = sizeof(struct ibv_flow_spec_tcp_udp);
- specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
- .type = ((i == HASH_RXQ_UDPV4 ||
- i == HASH_RXQ_UDPV6) ?
- IBV_FLOW_SPEC_UDP :
- IBV_FLOW_SPEC_TCP),
- .size = size,
- };
- if (parser->queue[i].ibv_attr) {
- dst = (void *)((uintptr_t)
- parser->queue[i].ibv_attr +
- parser->queue[i].offset);
- memcpy(dst, &specs, size);
- ++parser->queue[i].ibv_attr->num_of_specs;
- }
- parser->queue[i].offset += size;
- }
- }
-}
-
-/**
- * Validate and convert a flow supported by the NIC.
- *
- * @param dev
- * Pointer to Ethernet device.
- * @param[in] attr
- * Flow rule attributes.
- * @param[in] pattern
- * Pattern specification (list terminated by the END pattern item).
- * @param[in] actions
- * Associated actions (list terminated by the END action).
- * @param[out] error
- * Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- * Internal parser structure.
- *
- * @return
- * 0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert(struct rte_eth_dev *dev,
- const struct rte_flow_attr *attr,
- const struct rte_flow_item items[],
- const struct rte_flow_action actions[],
- struct rte_flow_error *error,
- struct mlx5_flow_parse *parser)