X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_flow.c;h=3c4ef6d7d220bde98c14731ebaad9814d7119298;hb=082becbf1f35;hp=4a031a523b0a3d2608edcf8db815574afe3359d2;hpb=3c78124f0ac62692c5e5af45a316ce1ec854bfc6;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 4a031a523b..d7cb1eb89b 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -3,26 +3,16 @@ * Copyright 2016 Mellanox Technologies, Ltd */ -#include -#include #include #include #include #include - -/* Verbs header. */ -/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif -#include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif +#include #include #include -#include +#include +#include #include #include #include @@ -32,36 +22,528 @@ #include #include #include +#include #include "mlx5_defs.h" #include "mlx5.h" #include "mlx5_flow.h" -#include "mlx5_rxtx.h" +#include "mlx5_flow_os.h" +#include "mlx5_rx.h" +#include "mlx5_tx.h" +#include "mlx5_common_os.h" +#include "rte_pmd_mlx5.h" + +struct tunnel_default_miss_ctx { + uint16_t *queue; + __extension__ + union { + struct rte_flow_action_rss action_rss; + struct rte_flow_action_queue miss_queue; + struct rte_flow_action_jump miss_jump; + uint8_t raw[0]; + }; +}; + +static int +flow_tunnel_add_default_miss(struct rte_eth_dev *dev, + struct rte_flow *flow, + const struct rte_flow_attr *attr, + const struct rte_flow_action *app_actions, + uint32_t flow_idx, + const struct mlx5_flow_tunnel *tunnel, + struct tunnel_default_miss_ctx *ctx, + struct rte_flow_error *error); +static struct mlx5_flow_tunnel * +mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); +static void +mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); +static uint32_t +tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, + const struct mlx5_flow_tunnel *tunnel, + uint32_t group, uint32_t *table, + struct rte_flow_error *error); + +static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); +static void mlx5_flow_pop_thread_workspace(void); + /** Device flow drivers. */ -#ifdef HAVE_IBV_FLOW_DV_SUPPORT -extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops; -#endif extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; const struct mlx5_flow_driver_ops *flow_drv_ops[] = { [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, -#ifdef HAVE_IBV_FLOW_DV_SUPPORT +#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, #endif [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops }; +/** Helper macro to build input graph for mlx5_flow_expand_rss(). */ +#define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ + (const int []){ \ + __VA_ARGS__, 0, \ + } + +/** Node object of input graph for mlx5_flow_expand_rss(). */ +struct mlx5_flow_expand_node { + const int *const next; + /**< + * List of next node indexes. Index 0 is interpreted as a terminator. + */ + const enum rte_flow_item_type type; + /**< Pattern item type of current node. */ + uint64_t rss_types; + /**< + * RSS types bit-field associated with this node + * (see RTE_ETH_RSS_* definitions). + */ + uint64_t node_flags; + /**< + * Bit-fields that define how the node is used in the expansion. + * (see MLX5_EXPANSION_NODE_* definitions). + */ +}; + +/* Optional expand field. The expansion alg will not go deeper. */ +#define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) + +/* The node is not added implicitly as expansion to the flow pattern. + * If the node type does not match the flow pattern item type, the + * expansion alg will go deeper to its next items. + * In the current implementation, the list of next nodes indexes can + * have up to one node with this flag set and it has to be the last + * node index (before the list terminator). + */ +#define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) + +/** Object returned by mlx5_flow_expand_rss(). */ +struct mlx5_flow_expand_rss { + uint32_t entries; + /**< Number of entries @p patterns and @p priorities. */ + struct { + struct rte_flow_item *pattern; /**< Expanded pattern array. */ + uint32_t priority; /**< Priority offset for each expansion. */ + } entry[]; +}; + +static void +mlx5_dbg__print_pattern(const struct rte_flow_item *item); + +static const struct mlx5_flow_expand_node * +mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, + unsigned int item_idx, + const struct mlx5_flow_expand_node graph[], + const struct mlx5_flow_expand_node *node); + +static bool +mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) +{ + switch (item->type) { + case RTE_FLOW_ITEM_TYPE_ETH: + case RTE_FLOW_ITEM_TYPE_VLAN: + case RTE_FLOW_ITEM_TYPE_IPV4: + case RTE_FLOW_ITEM_TYPE_IPV6: + case RTE_FLOW_ITEM_TYPE_UDP: + case RTE_FLOW_ITEM_TYPE_TCP: + case RTE_FLOW_ITEM_TYPE_VXLAN: + case RTE_FLOW_ITEM_TYPE_NVGRE: + case RTE_FLOW_ITEM_TYPE_GRE: + case RTE_FLOW_ITEM_TYPE_GENEVE: + case RTE_FLOW_ITEM_TYPE_MPLS: + case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: + case RTE_FLOW_ITEM_TYPE_GRE_KEY: + case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: + case RTE_FLOW_ITEM_TYPE_GTP: + return true; + default: + break; + } + return false; +} + +/** + * Network Service Header (NSH) and its next protocol values + * are described in RFC-8393. + */ +static enum rte_flow_item_type +mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) +{ + enum rte_flow_item_type type; + + switch (proto_mask & proto_spec) { + case RTE_VXLAN_GPE_TYPE_IPV4: + type = RTE_FLOW_ITEM_TYPE_IPV4; + break; + case RTE_VXLAN_GPE_TYPE_IPV6: + type = RTE_VXLAN_GPE_TYPE_IPV6; + break; + case RTE_VXLAN_GPE_TYPE_ETH: + type = RTE_FLOW_ITEM_TYPE_ETH; + break; + default: + type = RTE_FLOW_ITEM_TYPE_END; + } + return type; +} + +static enum rte_flow_item_type +mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) +{ + enum rte_flow_item_type type; + + switch (proto_mask & proto_spec) { + case IPPROTO_UDP: + type = RTE_FLOW_ITEM_TYPE_UDP; + break; + case IPPROTO_TCP: + type = RTE_FLOW_ITEM_TYPE_TCP; + break; + case IPPROTO_IP: + type = RTE_FLOW_ITEM_TYPE_IPV4; + break; + case IPPROTO_IPV6: + type = RTE_FLOW_ITEM_TYPE_IPV6; + break; + default: + type = RTE_FLOW_ITEM_TYPE_END; + } + return type; +} + +static enum rte_flow_item_type +mlx5_ethertype_to_item_type(rte_be16_t type_spec, + rte_be16_t type_mask, bool is_tunnel) +{ + enum rte_flow_item_type type; + + switch (rte_be_to_cpu_16(type_spec & type_mask)) { + case RTE_ETHER_TYPE_TEB: + type = is_tunnel ? + RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; + break; + case RTE_ETHER_TYPE_VLAN: + type = !is_tunnel ? + RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; + break; + case RTE_ETHER_TYPE_IPV4: + type = RTE_FLOW_ITEM_TYPE_IPV4; + break; + case RTE_ETHER_TYPE_IPV6: + type = RTE_FLOW_ITEM_TYPE_IPV6; + break; + default: + type = RTE_FLOW_ITEM_TYPE_END; + } + return type; +} + +static enum rte_flow_item_type +mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) +{ +#define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ + do { \ + const void *m = item->mask; \ + const void *s = item->spec; \ + mask = m ? \ + ((const struct rte_flow_item_##type *)m)->fld : \ + rte_flow_item_##type##_mask.fld; \ + spec = ((const struct rte_flow_item_##type *)s)->fld; \ + } while (0) + + enum rte_flow_item_type ret; + uint16_t spec, mask; + + if (item == NULL || item->spec == NULL) + return RTE_FLOW_ITEM_TYPE_VOID; + switch (item->type) { + case RTE_FLOW_ITEM_TYPE_ETH: + MLX5_XSET_ITEM_MASK_SPEC(eth, type); + if (!mask) + return RTE_FLOW_ITEM_TYPE_VOID; + ret = mlx5_ethertype_to_item_type(spec, mask, false); + break; + case RTE_FLOW_ITEM_TYPE_VLAN: + MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); + if (!mask) + return RTE_FLOW_ITEM_TYPE_VOID; + ret = mlx5_ethertype_to_item_type(spec, mask, false); + break; + case RTE_FLOW_ITEM_TYPE_IPV4: + MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); + if (!mask) + return RTE_FLOW_ITEM_TYPE_VOID; + ret = mlx5_inet_proto_to_item_type(spec, mask); + break; + case RTE_FLOW_ITEM_TYPE_IPV6: + MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); + if (!mask) + return RTE_FLOW_ITEM_TYPE_VOID; + ret = mlx5_inet_proto_to_item_type(spec, mask); + break; + case RTE_FLOW_ITEM_TYPE_GENEVE: + MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); + ret = mlx5_ethertype_to_item_type(spec, mask, true); + break; + case RTE_FLOW_ITEM_TYPE_GRE: + MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); + ret = mlx5_ethertype_to_item_type(spec, mask, true); + break; + case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: + MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); + ret = mlx5_nsh_proto_to_item_type(spec, mask); + break; + default: + ret = RTE_FLOW_ITEM_TYPE_VOID; + break; + } + return ret; +#undef MLX5_XSET_ITEM_MASK_SPEC +} + +static const int * +mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], + const int *next_node) +{ + const struct mlx5_flow_expand_node *node = NULL; + const int *next = next_node; + + while (next && *next) { + /* + * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT + * flag set, because they were not found in the flow pattern. + */ + node = &graph[*next]; + if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) + break; + next = node->next; + } + return next; +} + +#define MLX5_RSS_EXP_ELT_N 16 + +/** + * Expand RSS flows into several possible flows according to the RSS hash + * fields requested and the driver capabilities. + * + * @param[out] buf + * Buffer to store the result expansion. + * @param[in] size + * Buffer size in bytes. If 0, @p buf can be NULL. + * @param[in] pattern + * User flow pattern. + * @param[in] types + * RSS types to expand (see RTE_ETH_RSS_* definitions). + * @param[in] graph + * Input graph to expand @p pattern according to @p types. + * @param[in] graph_root_index + * Index of root node in @p graph, typically 0. + * + * @return + * A positive value representing the size of @p buf in bytes regardless of + * @p size on success, a negative errno value otherwise and rte_errno is + * set, the following errors are defined: + * + * -E2BIG: graph-depth @p graph is too deep. + * -EINVAL: @p size has not enough space for expanded pattern. + */ +static int +mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, + const struct rte_flow_item *pattern, uint64_t types, + const struct mlx5_flow_expand_node graph[], + int graph_root_index) +{ + const struct rte_flow_item *item; + const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; + const int *next_node; + const int *stack[MLX5_RSS_EXP_ELT_N]; + int stack_pos = 0; + struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; + unsigned int i, item_idx, last_expand_item_idx = 0; + size_t lsize; + size_t user_pattern_size = 0; + void *addr = NULL; + const struct mlx5_flow_expand_node *next = NULL; + struct rte_flow_item missed_item; + int missed = 0; + int elt = 0; + const struct rte_flow_item *last_expand_item = NULL; + + memset(&missed_item, 0, sizeof(missed_item)); + lsize = offsetof(struct mlx5_flow_expand_rss, entry) + + MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); + if (lsize > size) + return -EINVAL; + buf->entry[0].priority = 0; + buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; + buf->entries = 0; + addr = buf->entry[0].pattern; + for (item = pattern, item_idx = 0; + item->type != RTE_FLOW_ITEM_TYPE_END; + item++, item_idx++) { + if (!mlx5_flow_is_rss_expandable_item(item)) { + user_pattern_size += sizeof(*item); + continue; + } + last_expand_item = item; + last_expand_item_idx = item_idx; + i = 0; + while (node->next && node->next[i]) { + next = &graph[node->next[i]]; + if (next->type == item->type) + break; + if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { + node = next; + i = 0; + } else { + ++i; + } + } + if (next) + node = next; + user_pattern_size += sizeof(*item); + } + user_pattern_size += sizeof(*item); /* Handle END item. */ + lsize += user_pattern_size; + if (lsize > size) + return -EINVAL; + /* Copy the user pattern in the first entry of the buffer. */ + rte_memcpy(addr, pattern, user_pattern_size); + addr = (void *)(((uintptr_t)addr) + user_pattern_size); + buf->entries = 1; + /* Start expanding. */ + memset(flow_items, 0, sizeof(flow_items)); + user_pattern_size -= sizeof(*item); + /* + * Check if the last valid item has spec set, need complete pattern, + * and the pattern can be used for expansion. + */ + missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); + if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { + /* Item type END indicates expansion is not required. */ + return lsize; + } + if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { + next = NULL; + missed = 1; + i = 0; + while (node->next && node->next[i]) { + next = &graph[node->next[i]]; + if (next->type == missed_item.type) { + flow_items[0].type = missed_item.type; + flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; + break; + } + if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { + node = next; + i = 0; + } else { + ++i; + } + next = NULL; + } + } + if (next && missed) { + elt = 2; /* missed item + item end. */ + node = next; + lsize += elt * sizeof(*item) + user_pattern_size; + if (lsize > size) + return -EINVAL; + if (node->rss_types & types) { + buf->entry[buf->entries].priority = 1; + buf->entry[buf->entries].pattern = addr; + buf->entries++; + rte_memcpy(addr, buf->entry[0].pattern, + user_pattern_size); + addr = (void *)(((uintptr_t)addr) + user_pattern_size); + rte_memcpy(addr, flow_items, elt * sizeof(*item)); + addr = (void *)(((uintptr_t)addr) + + elt * sizeof(*item)); + } + } else if (last_expand_item != NULL) { + node = mlx5_flow_expand_rss_adjust_node(pattern, + last_expand_item_idx, graph, node); + } + memset(flow_items, 0, sizeof(flow_items)); + next_node = mlx5_flow_expand_rss_skip_explicit(graph, + node->next); + stack[stack_pos] = next_node; + node = next_node ? &graph[*next_node] : NULL; + while (node) { + flow_items[stack_pos].type = node->type; + if (node->rss_types & types) { + size_t n; + /* + * compute the number of items to copy from the + * expansion and copy it. + * When the stack_pos is 0, there are 1 element in it, + * plus the addition END item. + */ + elt = stack_pos + 2; + flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; + lsize += elt * sizeof(*item) + user_pattern_size; + if (lsize > size) + return -EINVAL; + n = elt * sizeof(*item); + buf->entry[buf->entries].priority = + stack_pos + 1 + missed; + buf->entry[buf->entries].pattern = addr; + buf->entries++; + rte_memcpy(addr, buf->entry[0].pattern, + user_pattern_size); + addr = (void *)(((uintptr_t)addr) + + user_pattern_size); + rte_memcpy(addr, &missed_item, + missed * sizeof(*item)); + addr = (void *)(((uintptr_t)addr) + + missed * sizeof(*item)); + rte_memcpy(addr, flow_items, n); + addr = (void *)(((uintptr_t)addr) + n); + } + /* Go deeper. */ + if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && + node->next) { + next_node = mlx5_flow_expand_rss_skip_explicit(graph, + node->next); + if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { + rte_errno = E2BIG; + return -rte_errno; + } + stack[stack_pos] = next_node; + } else if (*(next_node + 1)) { + /* Follow up with the next possibility. */ + next_node = mlx5_flow_expand_rss_skip_explicit(graph, + ++next_node); + } else if (!stack_pos) { + /* + * Completing the traverse over the different paths. + * The next_node is advanced to the terminator. + */ + ++next_node; + } else { + /* Move to the next path. */ + while (stack_pos) { + next_node = stack[--stack_pos]; + next_node++; + if (*next_node) + break; + } + next_node = mlx5_flow_expand_rss_skip_explicit(graph, + next_node); + stack[stack_pos] = next_node; + } + node = next_node && *next_node ? &graph[*next_node] : NULL; + }; + return lsize; +} + enum mlx5_expansion { MLX5_EXPANSION_ROOT, MLX5_EXPANSION_ROOT_OUTER, - MLX5_EXPANSION_ROOT_ETH_VLAN, - MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, MLX5_EXPANSION_OUTER_ETH, - MLX5_EXPANSION_OUTER_ETH_VLAN, MLX5_EXPANSION_OUTER_VLAN, MLX5_EXPANSION_OUTER_IPV4, MLX5_EXPANSION_OUTER_IPV4_UDP, @@ -70,11 +552,14 @@ enum mlx5_expansion { MLX5_EXPANSION_OUTER_IPV6_UDP, MLX5_EXPANSION_OUTER_IPV6_TCP, MLX5_EXPANSION_VXLAN, + MLX5_EXPANSION_STD_VXLAN, + MLX5_EXPANSION_L3_VXLAN, MLX5_EXPANSION_VXLAN_GPE, MLX5_EXPANSION_GRE, + MLX5_EXPANSION_NVGRE, + MLX5_EXPANSION_GRE_KEY, MLX5_EXPANSION_MPLS, MLX5_EXPANSION_ETH, - MLX5_EXPANSION_ETH_VLAN, MLX5_EXPANSION_VLAN, MLX5_EXPANSION_IPV4, MLX5_EXPANSION_IPV4_UDP, @@ -82,155 +567,244 @@ enum mlx5_expansion { MLX5_EXPANSION_IPV6, MLX5_EXPANSION_IPV6_UDP, MLX5_EXPANSION_IPV6_TCP, + MLX5_EXPANSION_IPV6_FRAG_EXT, + MLX5_EXPANSION_GTP, + MLX5_EXPANSION_GENEVE, }; /** Supported expansion of items. */ -static const struct rte_flow_expand_node mlx5_support_expansion[] = { +static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { [MLX5_EXPANSION_ROOT] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, - MLX5_EXPANSION_IPV4, - MLX5_EXPANSION_IPV6), + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, + MLX5_EXPANSION_IPV4, + MLX5_EXPANSION_IPV6), .type = RTE_FLOW_ITEM_TYPE_END, }, [MLX5_EXPANSION_ROOT_OUTER] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, - MLX5_EXPANSION_OUTER_IPV4, - MLX5_EXPANSION_OUTER_IPV6), - .type = RTE_FLOW_ITEM_TYPE_END, - }, - [MLX5_EXPANSION_ROOT_ETH_VLAN] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), - .type = RTE_FLOW_ITEM_TYPE_END, - }, - [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN), + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, + MLX5_EXPANSION_OUTER_IPV4, + MLX5_EXPANSION_OUTER_IPV6), .type = RTE_FLOW_ITEM_TYPE_END, }, [MLX5_EXPANSION_OUTER_ETH] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, - MLX5_EXPANSION_OUTER_IPV6, - MLX5_EXPANSION_MPLS), - .type = RTE_FLOW_ITEM_TYPE_ETH, - .rss_types = 0, - }, - [MLX5_EXPANSION_OUTER_ETH_VLAN] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), .type = RTE_FLOW_ITEM_TYPE_ETH, .rss_types = 0, }, [MLX5_EXPANSION_OUTER_VLAN] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, - MLX5_EXPANSION_OUTER_IPV6), + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, + MLX5_EXPANSION_OUTER_IPV6), .type = RTE_FLOW_ITEM_TYPE_VLAN, + .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, }, [MLX5_EXPANSION_OUTER_IPV4] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT + .next = MLX5_FLOW_EXPAND_RSS_NEXT (MLX5_EXPANSION_OUTER_IPV4_UDP, MLX5_EXPANSION_OUTER_IPV4_TCP, MLX5_EXPANSION_GRE, + MLX5_EXPANSION_NVGRE, MLX5_EXPANSION_IPV4, MLX5_EXPANSION_IPV6), .type = RTE_FLOW_ITEM_TYPE_IPV4, - .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | - ETH_RSS_NONFRAG_IPV4_OTHER, + .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | + RTE_ETH_RSS_NONFRAG_IPV4_OTHER, }, [MLX5_EXPANSION_OUTER_IPV4_UDP] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, - MLX5_EXPANSION_VXLAN_GPE), + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, + MLX5_EXPANSION_VXLAN_GPE, + MLX5_EXPANSION_MPLS, + MLX5_EXPANSION_GENEVE, + MLX5_EXPANSION_GTP), .type = RTE_FLOW_ITEM_TYPE_UDP, - .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, + .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, }, [MLX5_EXPANSION_OUTER_IPV4_TCP] = { .type = RTE_FLOW_ITEM_TYPE_TCP, - .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, + .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, }, [MLX5_EXPANSION_OUTER_IPV6] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT + .next = MLX5_FLOW_EXPAND_RSS_NEXT (MLX5_EXPANSION_OUTER_IPV6_UDP, MLX5_EXPANSION_OUTER_IPV6_TCP, MLX5_EXPANSION_IPV4, - MLX5_EXPANSION_IPV6), + MLX5_EXPANSION_IPV6, + MLX5_EXPANSION_GRE, + MLX5_EXPANSION_NVGRE), .type = RTE_FLOW_ITEM_TYPE_IPV6, - .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | - ETH_RSS_NONFRAG_IPV6_OTHER, + .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | + RTE_ETH_RSS_NONFRAG_IPV6_OTHER, }, [MLX5_EXPANSION_OUTER_IPV6_UDP] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, - MLX5_EXPANSION_VXLAN_GPE), + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, + MLX5_EXPANSION_VXLAN_GPE, + MLX5_EXPANSION_MPLS, + MLX5_EXPANSION_GENEVE, + MLX5_EXPANSION_GTP), .type = RTE_FLOW_ITEM_TYPE_UDP, - .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, + .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, }, [MLX5_EXPANSION_OUTER_IPV6_TCP] = { .type = RTE_FLOW_ITEM_TYPE_TCP, - .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, + .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, }, [MLX5_EXPANSION_VXLAN] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, - MLX5_EXPANSION_IPV4, - MLX5_EXPANSION_IPV6), + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, + MLX5_EXPANSION_IPV4, + MLX5_EXPANSION_IPV6), .type = RTE_FLOW_ITEM_TYPE_VXLAN, }, + [MLX5_EXPANSION_STD_VXLAN] = { + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), + .type = RTE_FLOW_ITEM_TYPE_VXLAN, + }, + [MLX5_EXPANSION_L3_VXLAN] = { + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, + MLX5_EXPANSION_IPV6), + .type = RTE_FLOW_ITEM_TYPE_VXLAN, + }, [MLX5_EXPANSION_VXLAN_GPE] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, - MLX5_EXPANSION_IPV4, - MLX5_EXPANSION_IPV6), + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, + MLX5_EXPANSION_IPV4, + MLX5_EXPANSION_IPV6), .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, }, [MLX5_EXPANSION_GRE] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, + MLX5_EXPANSION_IPV4, + MLX5_EXPANSION_IPV6, + MLX5_EXPANSION_GRE_KEY, + MLX5_EXPANSION_MPLS), .type = RTE_FLOW_ITEM_TYPE_GRE, }, + [MLX5_EXPANSION_GRE_KEY] = { + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, + MLX5_EXPANSION_IPV6, + MLX5_EXPANSION_MPLS), + .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, + .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, + }, + [MLX5_EXPANSION_NVGRE] = { + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), + .type = RTE_FLOW_ITEM_TYPE_NVGRE, + }, [MLX5_EXPANSION_MPLS] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, - MLX5_EXPANSION_IPV6), + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, + MLX5_EXPANSION_IPV6, + MLX5_EXPANSION_ETH), .type = RTE_FLOW_ITEM_TYPE_MPLS, + .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, }, [MLX5_EXPANSION_ETH] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, - MLX5_EXPANSION_IPV6), - .type = RTE_FLOW_ITEM_TYPE_ETH, - }, - [MLX5_EXPANSION_ETH_VLAN] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), .type = RTE_FLOW_ITEM_TYPE_ETH, }, [MLX5_EXPANSION_VLAN] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, - MLX5_EXPANSION_IPV6), + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, + MLX5_EXPANSION_IPV6), .type = RTE_FLOW_ITEM_TYPE_VLAN, + .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, }, [MLX5_EXPANSION_IPV4] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, - MLX5_EXPANSION_IPV4_TCP), + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, + MLX5_EXPANSION_IPV4_TCP), .type = RTE_FLOW_ITEM_TYPE_IPV4, - .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | - ETH_RSS_NONFRAG_IPV4_OTHER, + .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | + RTE_ETH_RSS_NONFRAG_IPV4_OTHER, }, [MLX5_EXPANSION_IPV4_UDP] = { .type = RTE_FLOW_ITEM_TYPE_UDP, - .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, + .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, }, [MLX5_EXPANSION_IPV4_TCP] = { .type = RTE_FLOW_ITEM_TYPE_TCP, - .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, + .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, }, [MLX5_EXPANSION_IPV6] = { - .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, - MLX5_EXPANSION_IPV6_TCP), + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, + MLX5_EXPANSION_IPV6_TCP, + MLX5_EXPANSION_IPV6_FRAG_EXT), .type = RTE_FLOW_ITEM_TYPE_IPV6, - .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | - ETH_RSS_NONFRAG_IPV6_OTHER, + .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | + RTE_ETH_RSS_NONFRAG_IPV6_OTHER, }, [MLX5_EXPANSION_IPV6_UDP] = { .type = RTE_FLOW_ITEM_TYPE_UDP, - .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, + .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, }, [MLX5_EXPANSION_IPV6_TCP] = { .type = RTE_FLOW_ITEM_TYPE_TCP, - .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, + .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, + }, + [MLX5_EXPANSION_IPV6_FRAG_EXT] = { + .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, + }, + [MLX5_EXPANSION_GTP] = { + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, + MLX5_EXPANSION_IPV6), + .type = RTE_FLOW_ITEM_TYPE_GTP, + }, + [MLX5_EXPANSION_GENEVE] = { + .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, + MLX5_EXPANSION_IPV4, + MLX5_EXPANSION_IPV6), + .type = RTE_FLOW_ITEM_TYPE_GENEVE, }, }; +static struct rte_flow_action_handle * +mlx5_action_handle_create(struct rte_eth_dev *dev, + const struct rte_flow_indir_action_conf *conf, + const struct rte_flow_action *action, + struct rte_flow_error *error); +static int mlx5_action_handle_destroy + (struct rte_eth_dev *dev, + struct rte_flow_action_handle *handle, + struct rte_flow_error *error); +static int mlx5_action_handle_update + (struct rte_eth_dev *dev, + struct rte_flow_action_handle *handle, + const void *update, + struct rte_flow_error *error); +static int mlx5_action_handle_query + (struct rte_eth_dev *dev, + const struct rte_flow_action_handle *handle, + void *data, + struct rte_flow_error *error); +static int +mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, + struct rte_flow_tunnel *app_tunnel, + struct rte_flow_action **actions, + uint32_t *num_of_actions, + struct rte_flow_error *error); +static int +mlx5_flow_tunnel_match(struct rte_eth_dev *dev, + struct rte_flow_tunnel *app_tunnel, + struct rte_flow_item **items, + uint32_t *num_of_items, + struct rte_flow_error *error); +static int +mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, + struct rte_flow_item *pmd_items, + uint32_t num_items, struct rte_flow_error *err); +static int +mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, + struct rte_flow_action *pmd_actions, + uint32_t num_actions, + struct rte_flow_error *err); +static int +mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, + struct rte_mbuf *m, + struct rte_flow_restore_info *info, + struct rte_flow_error *err); +static struct rte_flow_item_flex_handle * +mlx5_flow_flex_item_create(struct rte_eth_dev *dev, + const struct rte_flow_item_flex_conf *conf, + struct rte_flow_error *error); +static int +mlx5_flow_flex_item_release(struct rte_eth_dev *dev, + const struct rte_flow_item_flex_handle *handle, + struct rte_flow_error *error); + static const struct rte_flow_ops mlx5_flow_ops = { .validate = mlx5_flow_validate, .create = mlx5_flow_create, @@ -240,43 +814,17 @@ static const struct rte_flow_ops mlx5_flow_ops = { .query = mlx5_flow_query, .dev_dump = mlx5_flow_dev_dump, .get_aged_flows = mlx5_flow_get_aged_flows, -}; - -/* Convert FDIR request to Generic flow. */ -struct mlx5_fdir { - struct rte_flow_attr attr; - struct rte_flow_item items[4]; - struct rte_flow_item_eth l2; - struct rte_flow_item_eth l2_mask; - union { - struct rte_flow_item_ipv4 ipv4; - struct rte_flow_item_ipv6 ipv6; - } l3; - union { - struct rte_flow_item_ipv4 ipv4; - struct rte_flow_item_ipv6 ipv6; - } l3_mask; - union { - struct rte_flow_item_udp udp; - struct rte_flow_item_tcp tcp; - } l4; - union { - struct rte_flow_item_udp udp; - struct rte_flow_item_tcp tcp; - } l4_mask; - struct rte_flow_action actions[2]; - struct rte_flow_action_queue queue; -}; - -/* Map of Verbs to Flow priority with 8 Verbs priorities. */ -static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { - { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, -}; - -/* Map of Verbs to Flow priority with 16 Verbs priorities. */ -static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { - { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, - { 9, 10, 11 }, { 12, 13, 14 }, + .action_handle_create = mlx5_action_handle_create, + .action_handle_destroy = mlx5_action_handle_destroy, + .action_handle_update = mlx5_action_handle_update, + .action_handle_query = mlx5_action_handle_query, + .tunnel_decap_set = mlx5_flow_tunnel_decap_set, + .tunnel_match = mlx5_flow_tunnel_match, + .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, + .tunnel_item_release = mlx5_flow_tunnel_item_release, + .get_restore_info = mlx5_flow_tunnel_get_restore_info, + .flex_item_create = mlx5_flow_flex_item_create, + .flex_item_release = mlx5_flow_flex_item_release, }; /* Tunnel information. */ @@ -328,6 +876,8 @@ static struct mlx5_flow_tunnel_info tunnels_info[] = { }, }; + + /** * Translate tag ID to register. * @@ -375,7 +925,7 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev, case MLX5_METADATA_FDB: switch (config->dv_xmeta_en) { case MLX5_XMETA_MODE_LEGACY: - return REG_NONE; + return REG_NON; case MLX5_XMETA_MODE_META16: return REG_C_0; case MLX5_XMETA_MODE_META32: @@ -385,16 +935,16 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev, case MLX5_FLOW_MARK: switch (config->dv_xmeta_en) { case MLX5_XMETA_MODE_LEGACY: - return REG_NONE; + return REG_NON; case MLX5_XMETA_MODE_META16: return REG_C_1; case MLX5_XMETA_MODE_META32: return REG_C_0; } break; - case MLX5_MTR_SFX: + case MLX5_MTR_ID: /* - * If meter color and flow match share one register, flow match + * If meter color and meter id share one register, flow match * should use the meter color register for match. */ if (priv->mtr_reg_share) @@ -403,7 +953,11 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev, return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; case MLX5_MTR_COLOR: - MLX5_ASSERT(priv->mtr_color_reg != REG_NONE); + case MLX5_ASO_FLOW_HIT: + case MLX5_ASO_CONNTRACK: + case MLX5_SAMPLE_ID: + /* All features use the same REG_C. */ + MLX5_ASSERT(priv->mtr_color_reg != REG_NON); return priv->mtr_color_reg; case MLX5_COPY_MARK: /* @@ -422,11 +976,11 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev, start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : (priv->mtr_reg_share ? REG_C_3 : REG_C_4); skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); - if (id > (REG_C_7 - start_reg)) + if (id > (uint32_t)(REG_C_7 - start_reg)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "invalid tag id"); - if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NONE) + if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unsupported tag id"); @@ -436,21 +990,21 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev, * If the available index REG_C_y >= REG_C_x, skip the * color register. */ - if (skip_mtr_reg && config->flow_mreg_c + if (skip_mtr_reg && priv->sh->flow_mreg_c [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { - if (id >= (REG_C_7 - start_reg)) + if (id >= (uint32_t)(REG_C_7 - start_reg)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "invalid tag id"); - if (config->flow_mreg_c - [id + 1 + start_reg - REG_C_0] != REG_NONE) - return config->flow_mreg_c + if (priv->sh->flow_mreg_c + [id + 1 + start_reg - REG_C_0] != REG_NON) + return priv->sh->flow_mreg_c [id + 1 + start_reg - REG_C_0]; return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unsupported tag id"); } - return config->flow_mreg_c[id + start_reg - REG_C_0]; + return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; } MLX5_ASSERT(false); return rte_flow_error_set(error, EINVAL, @@ -471,7 +1025,6 @@ bool mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_config *config = &priv->config; /* * Having available reg_c can be regarded inclusively as supporting @@ -481,107 +1034,64 @@ mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) * - reg_c's are preserved across different domain (FDB and NIC) on * packet loopback by flow lookup miss. */ - return config->flow_mreg_c[2] != REG_NONE; + return priv->sh->flow_mreg_c[2] != REG_NON; } /** - * Discover the maximum number of priority available. + * Get the lowest priority. * * @param[in] dev * Pointer to the Ethernet device structure. + * @param[in] attributes + * Pointer to device flow rule attributes. * * @return - * number of supported flow priority on success, a negative errno - * value otherwise and rte_errno is set. + * The value of lowest priority of flow. */ -int -mlx5_flow_discover_priorities(struct rte_eth_dev *dev) +uint32_t +mlx5_get_lowest_priority(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr) { struct mlx5_priv *priv = dev->data->dev_private; - struct { - struct ibv_flow_attr attr; - struct ibv_flow_spec_eth eth; - struct ibv_flow_spec_action_drop drop; - } flow_attr = { - .attr = { - .num_of_specs = 2, - .port = (uint8_t)priv->dev_port, - }, - .eth = { - .type = IBV_FLOW_SPEC_ETH, - .size = sizeof(struct ibv_flow_spec_eth), - }, - .drop = { - .size = sizeof(struct ibv_flow_spec_action_drop), - .type = IBV_FLOW_SPEC_ACTION_DROP, - }, - }; - struct ibv_flow *flow; - struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev); - uint16_t vprio[] = { 8, 16 }; - int i; - int priority = 0; - if (!drop) { - rte_errno = ENOTSUP; - return -rte_errno; - } - for (i = 0; i != RTE_DIM(vprio); i++) { - flow_attr.attr.priority = vprio[i] - 1; - flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); - if (!flow) - break; - claim_zero(mlx5_glue->destroy_flow(flow)); - priority = vprio[i]; - } - mlx5_hrxq_drop_release(dev); - switch (priority) { - case 8: - priority = RTE_DIM(priority_map_3); - break; - case 16: - priority = RTE_DIM(priority_map_5); - break; - default: - rte_errno = ENOTSUP; - DRV_LOG(ERR, - "port %u verbs maximum priority: %d expected 8/16", - dev->data->port_id, priority); - return -rte_errno; - } - DRV_LOG(INFO, "port %u flow maximum priority: %d", - dev->data->port_id, priority); - return priority; + if (!attr->group && !attr->transfer) + return priv->sh->flow_max_priority - 2; + return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; } /** - * Adjust flow priority based on the highest layer and the request priority. + * Calculate matcher priority of the flow. * * @param[in] dev * Pointer to the Ethernet device structure. - * @param[in] priority - * The rule base priority. + * @param[in] attr + * Pointer to device flow rule attributes. * @param[in] subpriority * The priority based on the items. - * + * @param[in] external + * Flow is user flow. * @return - * The new priority. + * The matcher priority of the flow. */ -uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, - uint32_t subpriority) +uint16_t +mlx5_get_matcher_priority(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + uint32_t subpriority, bool external) { - uint32_t res = 0; + uint16_t priority = (uint16_t)attr->priority; struct mlx5_priv *priv = dev->data->dev_private; - switch (priv->config.flow_prio) { - case RTE_DIM(priority_map_3): - res = priority_map_3[priority][subpriority]; - break; - case RTE_DIM(priority_map_5): - res = priority_map_5[priority][subpriority]; - break; + if (!attr->group && !attr->transfer) { + if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) + priority = priv->sh->flow_max_priority - 1; + return mlx5_os_flow_adjust_priority(dev, priority, subpriority); + } else if (!external && attr->transfer && attr->group == 0 && + attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { + return (priv->sh->flow_max_priority - 1) * 3; } - return res; + if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) + priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; + return priority * 3 + subpriority; } /** @@ -596,6 +1106,8 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, * Bit-masks covering supported fields by the NIC to compare with user mask. * @param[in] size * Bit-masks size in bytes. + * @param[in] range_accepted + * True if range of values is accepted for specific fields, false otherwise. * @param[out] error * Pointer to error structure. * @@ -607,6 +1119,7 @@ mlx5_flow_item_acceptable(const struct rte_flow_item *item, const uint8_t *mask, const uint8_t *nic_mask, unsigned int size, + bool range_accepted, struct rte_flow_error *error) { unsigned int i; @@ -624,7 +1137,7 @@ mlx5_flow_item_acceptable(const struct rte_flow_item *item, RTE_FLOW_ERROR_TYPE_ITEM, item, "mask/last without a spec is not" " supported"); - if (item->spec && item->last) { + if (item->spec && item->last && !range_accepted) { uint8_t spec[size]; uint8_t last[size]; unsigned int i; @@ -652,7 +1165,7 @@ mlx5_flow_item_acceptable(const struct rte_flow_item *item, * @param[in] tunnel * 1 when the hash field is for a tunnel item. * @param[in] layer_types - * ETH_RSS_* types. + * RTE_ETH_RSS_* types. * @param[in] hash_fields * Item hash fields. * @@ -708,7 +1221,7 @@ flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) } /** - * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive + * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device * flow. * * @param[in] dev @@ -716,42 +1229,45 @@ flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) * @param[in] dev_handle * Pointer to device flow handle structure. */ -static void +void flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow_handle *dev_handle) { struct mlx5_priv *priv = dev->data->dev_private; - const int mark = dev_handle->mark; const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); - struct mlx5_hrxq *hrxq; + struct mlx5_ind_table_obj *ind_tbl = NULL; unsigned int i; - if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) - return; - hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], + if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { + struct mlx5_hrxq *hrxq; + + hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], dev_handle->rix_hrxq); - if (!hrxq) + if (hrxq) + ind_tbl = hrxq->ind_table; + } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { + struct mlx5_shared_action_rss *shared_rss; + + shared_rss = mlx5_ipool_get + (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], + dev_handle->rix_srss); + if (shared_rss) + ind_tbl = shared_rss->ind_tbl; + } + if (!ind_tbl) return; - for (i = 0; i != hrxq->ind_table->queues_n; ++i) { - int idx = hrxq->ind_table->queues[i]; - struct mlx5_rxq_ctrl *rxq_ctrl = - container_of((*priv->rxqs)[idx], - struct mlx5_rxq_ctrl, rxq); + for (i = 0; i != ind_tbl->queues_n; ++i) { + int idx = ind_tbl->queues[i]; + struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); + MLX5_ASSERT(rxq_ctrl != NULL); + if (rxq_ctrl == NULL) + continue; /* * To support metadata register copy on Tx loopback, * this must be always enabled (metadata may arive * from other port - not from local flows only. */ - if (priv->config.dv_flow_en && - priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && - mlx5_flow_ext_mreg_supported(dev)) { - rxq_ctrl->rxq.mark = 1; - rxq_ctrl->flow_mark_n = 1; - } else if (mark) { - rxq_ctrl->rxq.mark = 1; - rxq_ctrl->flow_mark_n++; - } if (tunnel) { unsigned int j; @@ -769,6 +1285,20 @@ flow_drv_rxq_flags_set(struct rte_eth_dev *dev, } } +static void +flow_rxq_mark_flag_set(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_rxq_ctrl *rxq_ctrl; + + if (priv->mark_enabled) + return; + LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { + rxq_ctrl->rxq.mark = 1; + } + priv->mark_enabled = 1; +} + /** * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow * @@ -783,7 +1313,11 @@ flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) struct mlx5_priv *priv = dev->data->dev_private; uint32_t handle_idx; struct mlx5_flow_handle *dev_handle; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); + MLX5_ASSERT(wks); + if (wks->mark) + flow_rxq_mark_flag_set(dev); SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, handle_idx, dev_handle, next) flow_drv_rxq_flags_set(dev, dev_handle); @@ -803,33 +1337,36 @@ flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow_handle *dev_handle) { struct mlx5_priv *priv = dev->data->dev_private; - const int mark = dev_handle->mark; const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); - struct mlx5_hrxq *hrxq; + struct mlx5_ind_table_obj *ind_tbl = NULL; unsigned int i; - if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) - return; - hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], + if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { + struct mlx5_hrxq *hrxq; + + hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], dev_handle->rix_hrxq); - if (!hrxq) + if (hrxq) + ind_tbl = hrxq->ind_table; + } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { + struct mlx5_shared_action_rss *shared_rss; + + shared_rss = mlx5_ipool_get + (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], + dev_handle->rix_srss); + if (shared_rss) + ind_tbl = shared_rss->ind_tbl; + } + if (!ind_tbl) return; MLX5_ASSERT(dev->data->dev_started); - for (i = 0; i != hrxq->ind_table->queues_n; ++i) { - int idx = hrxq->ind_table->queues[i]; - struct mlx5_rxq_ctrl *rxq_ctrl = - container_of((*priv->rxqs)[idx], - struct mlx5_rxq_ctrl, rxq); - - if (priv->config.dv_flow_en && - priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && - mlx5_flow_ext_mreg_supported(dev)) { - rxq_ctrl->rxq.mark = 1; - rxq_ctrl->flow_mark_n = 1; - } else if (mark) { - rxq_ctrl->flow_mark_n--; - rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; - } + for (i = 0; i != ind_tbl->queues_n; ++i) { + int idx = ind_tbl->queues[i]; + struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); + + MLX5_ASSERT(rxq_ctrl != NULL); + if (rxq_ctrl == NULL) + continue; if (tunnel) { unsigned int j; @@ -881,19 +1418,17 @@ flow_rxq_flags_clear(struct rte_eth_dev *dev) unsigned int i; for (i = 0; i != priv->rxqs_n; ++i) { - struct mlx5_rxq_ctrl *rxq_ctrl; + struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); unsigned int j; - if (!(*priv->rxqs)[i]) + if (rxq == NULL || rxq->ctrl == NULL) continue; - rxq_ctrl = container_of((*priv->rxqs)[i], - struct mlx5_rxq_ctrl, rxq); - rxq_ctrl->flow_mark_n = 0; - rxq_ctrl->rxq.mark = 0; + rxq->ctrl->rxq.mark = 0; for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) - rxq_ctrl->flow_tunnels_n[j] = 0; - rxq_ctrl->rxq.tunnel = 0; + rxq->ctrl->flow_tunnels_n[j] = 0; + rxq->ctrl->rxq.tunnel = 0; } + priv->mark_enabled = 0; } /** @@ -906,21 +1441,25 @@ void mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_rxq_data *data; unsigned int i; for (i = 0; i != priv->rxqs_n; ++i) { - if (!(*priv->rxqs)[i]) + struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); + struct mlx5_rxq_data *data; + + if (rxq == NULL || rxq->ctrl == NULL) continue; - data = (*priv->rxqs)[i]; + data = &rxq->ctrl->rxq; if (!rte_flow_dynf_metadata_avail()) { data->dynf_meta = 0; data->flow_meta_mask = 0; data->flow_meta_offset = -1; + data->flow_meta_port_mask = 0; } else { data->dynf_meta = 1; data->flow_meta_mask = rte_flow_dynf_metadata_mask; data->flow_meta_offset = rte_flow_dynf_metadata_offs; + data->flow_meta_port_mask = priv->sh->dv_meta_mask; } } } @@ -1101,7 +1640,7 @@ mlx5_flow_validate_action_queue(const struct rte_flow_action *action, RTE_FLOW_ERROR_TYPE_ACTION_CONF, &queue->index, "queue index out of range"); - if (!(*priv->rxqs)[queue->index]) + if (mlx5_rxq_get(dev, queue->index) == NULL) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, &queue->index, @@ -1114,19 +1653,64 @@ mlx5_flow_validate_action_queue(const struct rte_flow_action *action, return 0; } -/* - * Validate the rss action. +/** + * Validate queue numbers for device RSS. * - * @param[in] action - * Pointer to the queue action. - * @param[in] action_flags - * Bit-fields that holds the actions detected until now. * @param[in] dev - * Pointer to the Ethernet device structure. - * @param[in] attr - * Attributes of flow that includes this action. - * @param[in] item_flags - * Items that were detected. + * Configured device. + * @param[in] queues + * Array of queue numbers. + * @param[in] queues_n + * Size of the @p queues array. + * @param[out] error + * On error, filled with a textual error description. + * @param[out] queue + * On error, filled with an offending queue index in @p queues array. + * + * @return + * 0 on success, a negative errno code on error. + */ +static int +mlx5_validate_rss_queues(struct rte_eth_dev *dev, + const uint16_t *queues, uint32_t queues_n, + const char **error, uint32_t *queue_idx) +{ + const struct mlx5_priv *priv = dev->data->dev_private; + enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; + uint32_t i; + + for (i = 0; i != queues_n; ++i) { + struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, + queues[i]); + + if (queues[i] >= priv->rxqs_n) { + *error = "queue index out of range"; + *queue_idx = i; + return -EINVAL; + } + if (rxq_ctrl == NULL) { + *error = "queue is not configured"; + *queue_idx = i; + return -EINVAL; + } + if (i == 0) + rxq_type = rxq_ctrl->type; + if (rxq_type != rxq_ctrl->type) { + *error = "combining hairpin and regular RSS queues is not supported"; + *queue_idx = i; + return -ENOTSUP; + } + } + return 0; +} + +/* + * Validate the rss action. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] action + * Pointer to the queue action. * @param[out] error * Pointer to error structure. * @@ -1134,23 +1718,16 @@ mlx5_flow_validate_action_queue(const struct rte_flow_action *action, * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -mlx5_flow_validate_action_rss(const struct rte_flow_action *action, - uint64_t action_flags, - struct rte_eth_dev *dev, - const struct rte_flow_attr *attr, - uint64_t item_flags, - struct rte_flow_error *error) +mlx5_validate_action_rss(struct rte_eth_dev *dev, + const struct rte_flow_action *action, + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_action_rss *rss = action->conf; - int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); - unsigned int i; + int ret; + const char *message; + uint32_t queue_idx; - if (action_flags & MLX5_FLOW_FATE_ACTIONS) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can't have 2 fate actions" - " in same flow"); if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) return rte_flow_error_set(error, ENOTSUP, @@ -1193,14 +1770,14 @@ mlx5_flow_validate_action_rss(const struct rte_flow_action *action, &rss->types, "some RSS protocols are not" " supported"); - if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && - !(rss->types & ETH_RSS_IP)) + if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && + !(rss->types & RTE_ETH_RSS_IP)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, "L3 partial RSS requested but L3 RSS" " type not specified"); - if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && - !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) + if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && + !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, "L4 partial RSS requested but L4 RSS" @@ -1213,27 +1790,78 @@ mlx5_flow_validate_action_rss(const struct rte_flow_action *action, return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, "No queues configured"); - for (i = 0; i != rss->queue_num; ++i) { - if (rss->queue[i] >= priv->rxqs_n) - return rte_flow_error_set - (error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION_CONF, - &rss->queue[i], "queue index out of range"); - if (!(*priv->rxqs)[rss->queue[i]]) - return rte_flow_error_set - (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, - &rss->queue[i], "queue is not configured"); + ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, + &message, &queue_idx); + if (ret != 0) { + return rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, + &rss->queue[queue_idx], message); } + return 0; +} + +/* + * Validate the rss action. + * + * @param[in] action + * Pointer to the queue action. + * @param[in] action_flags + * Bit-fields that holds the actions detected until now. + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] attr + * Attributes of flow that includes this action. + * @param[in] item_flags + * Items that were detected. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_flow_validate_action_rss(const struct rte_flow_action *action, + uint64_t action_flags, + struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + uint64_t item_flags, + struct rte_flow_error *error) +{ + const struct rte_flow_action_rss *rss = action->conf; + int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); + int ret; + + if (action_flags & MLX5_FLOW_FATE_ACTIONS) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "can't have 2 fate actions" + " in same flow"); + ret = mlx5_validate_action_rss(dev, action, error); + if (ret) + return ret; if (attr->egress) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, "rss action not supported for " "egress"); - if (rss->level > 1 && !tunnel) + if (rss->level > 1 && !tunnel) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, "inner RSS is not supported for " "non-tunnel flows"); + if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && + !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, + "RSS on eCPRI is not supported now"); + } + if ((item_flags & MLX5_FLOW_LAYER_MPLS) && + !(item_flags & + (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && + rss->level > 1) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, NULL, + "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); return 0; } @@ -1300,6 +1928,37 @@ mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, return 0; } +/* + * Validate the ASO CT action. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] conntrack + * Pointer to the CT action profile. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_validate_action_ct(struct rte_eth_dev *dev, + const struct rte_flow_action_conntrack *conntrack, + struct rte_flow_error *error) +{ + RTE_SET_USED(dev); + + if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Invalid CT state"); + if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Invalid last TCP packet flag"); + return 0; +} + /** * Verify the @p attributes will be correctly understood by the NIC and store * them in the @p flow if everything is correct. @@ -1320,13 +1979,13 @@ mlx5_flow_validate_attributes(struct rte_eth_dev *dev, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - uint32_t priority_max = priv->config.flow_prio - 1; + uint32_t priority_max = priv->sh->flow_max_priority - 1; if (attributes->group) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, "groups is not supported"); - if (attributes->priority != MLX5_FLOW_PRIO_RSVD && + if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && attributes->priority >= priority_max) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, @@ -1354,6 +2013,8 @@ mlx5_flow_validate_attributes(struct rte_eth_dev *dev, * Item specification. * @param[in] item_flags * Bit-fields that holds the items detected until now. + * @param[in] ext_vlan_sup + * Whether extended VLAN features are supported or not. * @param[out] error * Pointer to error structure. * @@ -1393,7 +2054,8 @@ mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, ret = mlx5_flow_item_acceptable (item, (const uint8_t *)mask, (const uint8_t *)&rte_flow_item_icmp6_mask, - sizeof(struct rte_flow_item_icmp6), error); + sizeof(struct rte_flow_item_icmp6), + MLX5_ITEM_RANGE_NOT_ACCEPTED, error); if (ret < 0) return ret; return 0; @@ -1419,6 +2081,12 @@ mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, struct rte_flow_error *error) { const struct rte_flow_item_icmp *mask = item->mask; + const struct rte_flow_item_icmp nic_mask = { + .hdr.icmp_type = 0xff, + .hdr.icmp_code = 0xff, + .hdr.icmp_ident = RTE_BE16(0xffff), + .hdr.icmp_seq_nb = RTE_BE16(0xffff), + }; const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : MLX5_FLOW_LAYER_OUTER_L3_IPV4; @@ -1441,11 +2109,12 @@ mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, RTE_FLOW_ERROR_TYPE_ITEM, item, "multiple L4 layers not supported"); if (!mask) - mask = &rte_flow_item_icmp_mask; + mask = &nic_mask; ret = mlx5_flow_item_acceptable (item, (const uint8_t *)mask, - (const uint8_t *)&rte_flow_item_icmp_mask, - sizeof(struct rte_flow_item_icmp), error); + (const uint8_t *)&nic_mask, + sizeof(struct rte_flow_item_icmp), + MLX5_ITEM_RANGE_NOT_ACCEPTED, error); if (ret < 0) return ret; return 0; @@ -1466,7 +2135,7 @@ mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, */ int mlx5_flow_validate_item_eth(const struct rte_flow_item *item, - uint64_t item_flags, + uint64_t item_flags, bool ext_vlan_sup, struct rte_flow_error *error) { const struct rte_flow_item_eth *mask = item->mask; @@ -1474,6 +2143,7 @@ mlx5_flow_validate_item_eth(const struct rte_flow_item *item, .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", .type = RTE_BE16(0xffff), + .has_vlan = ext_vlan_sup ? 1 : 0, }; int ret; int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); @@ -1495,12 +2165,16 @@ mlx5_flow_validate_item_eth(const struct rte_flow_item *item, return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, "L2 layer should not follow VLAN"); + if (item_flags & MLX5_FLOW_LAYER_GTP) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "L2 layer should not follow GTP"); if (!mask) mask = &rte_flow_item_eth_mask; ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, (const uint8_t *)&nic_mask, sizeof(struct rte_flow_item_eth), - error); + MLX5_ITEM_RANGE_NOT_ACCEPTED, error); return ret; } @@ -1554,7 +2228,7 @@ mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, (const uint8_t *)&nic_mask, sizeof(struct rte_flow_item_vlan), - error); + MLX5_ITEM_RANGE_NOT_ACCEPTED, error); if (ret) return ret; if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { @@ -1599,9 +2273,15 @@ mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, * Item specification. * @param[in] item_flags * Bit-fields that holds the items detected until now. + * @param[in] last_item + * Previous validated item in the pattern items. + * @param[in] ether_type + * Type in the ethernet layer header (including dot1q). * @param[in] acc_mask * Acceptable mask, if NULL default internal default mask * will be used to check whether item fields are supported. + * @param[in] range_accepted + * True if range of values is accepted for specific fields, false otherwise. * @param[out] error * Pointer to error structure. * @@ -1614,6 +2294,7 @@ mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, uint64_t last_item, uint16_t ether_type, const struct rte_flow_item_ipv4 *acc_mask, + bool range_accepted, struct rte_flow_error *error) { const struct rte_flow_item_ipv4 *mask = item->mask; @@ -1643,7 +2324,7 @@ mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, RTE_FLOW_ERROR_TYPE_ITEM, item, "IPv4 cannot follow L2/VLAN layer " "which ether type is not IPv4"); - if (item_flags & MLX5_FLOW_LAYER_IPIP) { + if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { if (mask && spec) next_proto = mask->hdr.next_proto_id & spec->hdr.next_proto_id; @@ -1684,7 +2365,7 @@ mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, acc_mask ? (const uint8_t *)acc_mask : (const uint8_t *)&nic_mask, sizeof(struct rte_flow_item_ipv4), - error); + range_accepted, error); if (ret < 0) return ret; return 0; @@ -1697,6 +2378,10 @@ mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, * Item specification. * @param[in] item_flags * Bit-fields that holds the items detected until now. + * @param[in] last_item + * Previous validated item in the pattern items. + * @param[in] ether_type + * Type in the ethernet layer header (including dot1q). * @param[in] acc_mask * Acceptable mask, if NULL default internal default mask * will be used to check whether item fields are supported. @@ -1745,9 +2430,9 @@ mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, RTE_FLOW_ERROR_TYPE_ITEM, item, "IPv6 cannot follow L2/VLAN layer " "which ether type is not IPv6"); - if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { - if (mask && spec) - next_proto = mask->hdr.proto & spec->hdr.proto; + if (mask && mask->hdr.proto == UINT8_MAX && spec) + next_proto = spec->hdr.proto; + if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, @@ -1755,6 +2440,16 @@ mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, "multiple tunnel " "not supported"); } + if (next_proto == IPPROTO_HOPOPTS || + next_proto == IPPROTO_ROUTING || + next_proto == IPPROTO_FRAGMENT || + next_proto == IPPROTO_ESP || + next_proto == IPPROTO_AH || + next_proto == IPPROTO_DSTOPTS) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "IPv6 proto (next header) should " + "not be set as extension header"); if (item_flags & MLX5_FLOW_LAYER_IPIP) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, @@ -1779,7 +2474,7 @@ mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, acc_mask ? (const uint8_t *)acc_mask : (const uint8_t *)&nic_mask, sizeof(struct rte_flow_item_ipv6), - error); + MLX5_ITEM_RANGE_NOT_ACCEPTED, error); if (ret < 0) return ret; return 0; @@ -1834,7 +2529,8 @@ mlx5_flow_validate_item_udp(const struct rte_flow_item *item, ret = mlx5_flow_item_acceptable (item, (const uint8_t *)mask, (const uint8_t *)&rte_flow_item_udp_mask, - sizeof(struct rte_flow_item_udp), error); + sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, + error); if (ret < 0) return ret; return 0; @@ -1889,7 +2585,8 @@ mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, ret = mlx5_flow_item_acceptable (item, (const uint8_t *)mask, (const uint8_t *)flow_mask, - sizeof(struct rte_flow_item_tcp), error); + sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, + error); if (ret < 0) return ret; return 0; @@ -1898,12 +2595,16 @@ mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, /** * Validate VXLAN item. * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] udp_dport + * UDP destination port * @param[in] item * Item specification. * @param[in] item_flags * Bit-fields that holds the items detected until now. - * @param[in] target_protocol - * The next protocol in the previous item. + * @param[in] attr + * Flow rule attributes. * @param[out] error * Pointer to error structure. * @@ -1911,24 +2612,33 @@ mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, +mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, + uint16_t udp_dport, + const struct rte_flow_item *item, uint64_t item_flags, + const struct rte_flow_attr *attr, struct rte_flow_error *error) { const struct rte_flow_item_vxlan *spec = item->spec; const struct rte_flow_item_vxlan *mask = item->mask; int ret; + struct mlx5_priv *priv = dev->data->dev_private; union vni { uint32_t vlan_id; uint8_t vni[4]; } id = { .vlan_id = 0, }; - + const struct rte_flow_item_vxlan nic_mask = { + .vni = "\xff\xff\xff", + .rsvd1 = 0xff, + }; + const struct rte_flow_item_vxlan *valid_mask; if (item_flags & MLX5_FLOW_LAYER_TUNNEL) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, "multiple tunnel layers not" " supported"); + valid_mask = &rte_flow_item_vxlan_mask; /* * Verify only UDPv4 is present as defined in * https://tools.ietf.org/html/rfc7348 @@ -1939,11 +2649,23 @@ mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, "no outer UDP layer found"); if (!mask) mask = &rte_flow_item_vxlan_mask; + + if (priv->sh->steering_format_version != + MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || + !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { + /* FDB domain & NIC domain non-zero group */ + if ((attr->transfer || attr->group) && priv->sh->misc5_cap) + valid_mask = &nic_mask; + /* Group zero in NIC domain */ + if (!attr->group && !attr->transfer && + priv->sh->tunnel_header_0_1) + valid_mask = &nic_mask; + } ret = mlx5_flow_item_acceptable (item, (const uint8_t *)mask, - (const uint8_t *)&rte_flow_item_vxlan_mask, + (const uint8_t *)valid_mask, sizeof(struct rte_flow_item_vxlan), - error); + MLX5_ITEM_RANGE_NOT_ACCEPTED, error); if (ret < 0) return ret; if (spec) { @@ -2014,7 +2736,7 @@ mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, (item, (const uint8_t *)mask, (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, sizeof(struct rte_flow_item_vxlan_gpe), - error); + MLX5_ITEM_RANGE_NOT_ACCEPTED, error); if (ret < 0) return ret; if (spec) { @@ -2088,7 +2810,7 @@ mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, ret = mlx5_flow_item_acceptable (item, (const uint8_t *)mask, (const uint8_t *)&gre_key_default_mask, - sizeof(rte_be32_t), error); + sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); return ret; } @@ -2140,7 +2862,8 @@ mlx5_flow_validate_item_gre(const struct rte_flow_item *item, ret = mlx5_flow_item_acceptable (item, (const uint8_t *)mask, (const uint8_t *)&nic_mask, - sizeof(struct rte_flow_item_gre), error); + sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, + error); if (ret < 0) return ret; #ifndef HAVE_MLX5DV_DR @@ -2215,7 +2938,8 @@ mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, ret = mlx5_flow_item_acceptable (item, (const uint8_t *)mask, (const uint8_t *)&nic_mask, - sizeof(struct rte_flow_item_geneve), error); + sizeof(struct rte_flow_item_geneve), + MLX5_ITEM_RANGE_NOT_ACCEPTED, error); if (ret) return ret; if (spec) { @@ -2243,6 +2967,149 @@ mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, return 0; } +/** + * Validate Geneve TLV option item. + * + * @param[in] item + * Item specification. + * @param[in] last_item + * Previous validated item in the pattern items. + * @param[in] geneve_item + * Previous GENEVE item specification. + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, + uint64_t last_item, + const struct rte_flow_item *geneve_item, + struct rte_eth_dev *dev, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_ctx_shared *sh = priv->sh; + struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; + struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr; + uint8_t data_max_supported = + hca_attr->max_geneve_tlv_option_data_len * 4; + struct mlx5_dev_config *config = &priv->config; + const struct rte_flow_item_geneve *geneve_spec; + const struct rte_flow_item_geneve *geneve_mask; + const struct rte_flow_item_geneve_opt *spec = item->spec; + const struct rte_flow_item_geneve_opt *mask = item->mask; + unsigned int i; + unsigned int data_len; + uint8_t tlv_option_len; + uint16_t optlen_m, optlen_v; + const struct rte_flow_item_geneve_opt full_mask = { + .option_class = RTE_BE16(0xffff), + .option_type = 0xff, + .option_len = 0x1f, + }; + + if (!mask) + mask = &rte_flow_item_geneve_opt_mask; + if (!spec) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, + "Geneve TLV opt class/type/length must be specified"); + if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, + "Geneve TLV opt length exceeds the limit (31)"); + /* Check if class type and length masks are full. */ + if (full_mask.option_class != mask->option_class || + full_mask.option_type != mask->option_type || + full_mask.option_len != (mask->option_len & full_mask.option_len)) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, + "Geneve TLV opt class/type/length masks must be full"); + /* Check if length is supported */ + if ((uint32_t)spec->option_len > + config->hca_attr.max_geneve_tlv_option_data_len) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, + "Geneve TLV opt length not supported"); + if (config->hca_attr.max_geneve_tlv_options > 1) + DRV_LOG(DEBUG, + "max_geneve_tlv_options supports more than 1 option"); + /* Check GENEVE item preceding. */ + if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, + "Geneve opt item must be preceded with Geneve item"); + geneve_spec = geneve_item->spec; + geneve_mask = geneve_item->mask ? geneve_item->mask : + &rte_flow_item_geneve_mask; + /* Check if GENEVE TLV option size doesn't exceed option length */ + if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || + geneve_spec->ver_opt_len_o_c_rsvd0)) { + tlv_option_len = spec->option_len & mask->option_len; + optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); + optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); + optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); + optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); + if ((optlen_v & optlen_m) <= tlv_option_len) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, + "GENEVE TLV option length exceeds optlen"); + } + /* Check if length is 0 or data is 0. */ + if (spec->data == NULL || spec->option_len == 0) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, + "Geneve TLV opt with zero data/length not supported"); + /* Check not all data & mask are 0. */ + data_len = spec->option_len * 4; + if (mask->data == NULL) { + for (i = 0; i < data_len; i++) + if (spec->data[i]) + break; + if (i == data_len) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Can't match on Geneve option data 0"); + } else { + for (i = 0; i < data_len; i++) + if (spec->data[i] & mask->data[i]) + break; + if (i == data_len) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Can't match on Geneve option data and mask 0"); + /* Check data mask supported. */ + for (i = data_max_supported; i < data_len ; i++) + if (mask->data[i]) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Data mask is of unsupported size"); + } + /* Check GENEVE option is supported in NIC. */ + if (!config->hca_attr.geneve_tlv_opt) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, + "Geneve TLV opt not supported"); + /* Check if we already have geneve option with different type/class. */ + rte_spinlock_lock(&sh->geneve_tlv_opt_sl); + geneve_opt_resource = sh->geneve_tlv_option_resource; + if (geneve_opt_resource != NULL) + if (geneve_opt_resource->option_class != spec->option_class || + geneve_opt_resource->option_type != spec->option_type || + geneve_opt_resource->length != spec->option_len) { + rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Only one Geneve TLV option supported"); + } + rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); + return 0; +} + /** * Validate MPLS item. * @@ -2278,10 +3145,10 @@ mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, "MPLS not supported or" " disabled in firmware" " configuration."); - /* MPLS over IP, UDP, GRE is allowed */ - if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | - MLX5_FLOW_LAYER_OUTER_L4_UDP | - MLX5_FLOW_LAYER_GRE))) + /* MPLS over UDP, GRE is allowed */ + if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | + MLX5_FLOW_LAYER_GRE | + MLX5_FLOW_LAYER_GRE_KEY))) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, "protocol filtering not compatible" @@ -2298,7 +3165,8 @@ mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, ret = mlx5_flow_item_acceptable (item, (const uint8_t *)mask, (const uint8_t *)&rte_flow_item_mpls_mask, - sizeof(struct rte_flow_item_mpls), error); + sizeof(struct rte_flow_item_mpls), + MLX5_ITEM_RANGE_NOT_ACCEPTED, error); if (ret < 0) return ret; return 0; @@ -2353,56 +3221,102 @@ mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, ret = mlx5_flow_item_acceptable (item, (const uint8_t *)mask, (const uint8_t *)&rte_flow_item_nvgre_mask, - sizeof(struct rte_flow_item_nvgre), error); + sizeof(struct rte_flow_item_nvgre), + MLX5_ITEM_RANGE_NOT_ACCEPTED, error); if (ret < 0) return ret; return 0; } -/* Allocate unique ID for the split Q/RSS subflows. */ -static uint32_t -flow_qrss_get_id(struct rte_eth_dev *dev) -{ - struct mlx5_priv *priv = dev->data->dev_private; - uint32_t qrss_id, ret; - - ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id); - if (ret) - return 0; - MLX5_ASSERT(qrss_id); - return qrss_id; -} - -/* Free unique ID for the split Q/RSS subflows. */ -static void -flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id) -{ - struct mlx5_priv *priv = dev->data->dev_private; - - if (qrss_id) - mlx5_flow_id_release(priv->qrss_id_pool, qrss_id); -} - /** - * Release resource related QUEUE/RSS action split. + * Validate eCPRI item. * - * @param dev - * Pointer to Ethernet device. - * @param flow - * Flow to release id's from. + * @param[in] item + * Item specification. + * @param[in] item_flags + * Bit-fields that holds the items detected until now. + * @param[in] last_item + * Previous validated item in the pattern items. + * @param[in] ether_type + * Type in the ethernet layer header (including dot1q). + * @param[in] acc_mask + * Acceptable mask, if NULL default internal default mask + * will be used to check whether item fields are supported. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ -static void -flow_mreg_split_qrss_release(struct rte_eth_dev *dev, - struct rte_flow *flow) +int +mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, + uint64_t item_flags, + uint64_t last_item, + uint16_t ether_type, + const struct rte_flow_item_ecpri *acc_mask, + struct rte_flow_error *error) { - struct mlx5_priv *priv = dev->data->dev_private; - uint32_t handle_idx; - struct mlx5_flow_handle *dev_handle; + const struct rte_flow_item_ecpri *mask = item->mask; + const struct rte_flow_item_ecpri nic_mask = { + .hdr = { + .common = { + .u32 = + RTE_BE32(((const struct rte_ecpri_common_hdr) { + .type = 0xFF, + }).u32), + }, + .dummy[0] = 0xFFFFFFFF, + }, + }; + const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | + MLX5_FLOW_LAYER_OUTER_VLAN); + struct rte_flow_item_ecpri mask_lo; - SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, - handle_idx, dev_handle, next) - if (dev_handle->split_flow_id) - flow_qrss_free_id(dev, dev_handle->split_flow_id); + if (!(last_item & outer_l2_vlan) && + last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "eCPRI can only follow L2/VLAN layer or UDP layer"); + if ((last_item & outer_l2_vlan) && ether_type && + ether_type != RTE_ETHER_TYPE_ECPRI) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); + if (item_flags & MLX5_FLOW_LAYER_TUNNEL) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "eCPRI with tunnel is not supported right now"); + if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "multiple L3 layers not supported"); + else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "eCPRI cannot coexist with a TCP layer"); + /* In specification, eCPRI could be over UDP layer. */ + else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "eCPRI over UDP layer is not yet supported right now"); + /* Mask for type field in common header could be zero. */ + if (!mask) + mask = &rte_flow_item_ecpri_mask; + mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); + /* Input mask is in big-endian format. */ + if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, + "partial mask is not supported for protocol"); + else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, + "message header mask must be after a type mask"); + return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, + acc_mask ? (const uint8_t *)acc_mask + : (const uint8_t *)&nic_mask, + sizeof(struct rte_flow_item_ecpri), + MLX5_ITEM_RANGE_NOT_ACCEPTED, error); } static int @@ -2474,6 +3388,14 @@ flow_null_query(struct rte_eth_dev *dev __rte_unused, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); } +static int +flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, + uint32_t domains __rte_unused, + uint32_t flags __rte_unused) +{ + return 0; +} + /* Void driver to protect from null pointer reference. */ const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { .validate = flow_null_validate, @@ -2483,6 +3405,7 @@ const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { .remove = flow_null_remove, .destroy = flow_null_destroy, .query = flow_null_query, + .sync_domain = flow_null_sync_domain, }; /** @@ -2501,8 +3424,12 @@ static enum mlx5_flow_drv_type flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) { struct mlx5_priv *priv = dev->data->dev_private; - enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX; + /* The OS can determine first a specific flow type (DV, VERBS) */ + enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); + if (type != MLX5_FLOW_TYPE_MAX) + return type; + /* If no OS specific type - continue with DV/VERBS selection */ if (attr->transfer && priv->config.dv_esw_en) type = MLX5_FLOW_TYPE_DV; if (!attr->transfer) @@ -2672,10 +3599,9 @@ flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, } /** - * Flow driver remove API. This abstracts calling driver specific functions. + * Flow driver destroy API. This abstracts calling driver specific functions. * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow - * on device. All the resources of the flow should be freed by calling - * flow_drv_destroy(). + * on device and releases resources of the flow. * * @param[in] dev * Pointer to Ethernet device. @@ -2683,77 +3609,352 @@ flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, * Pointer to flow structure. */ static inline void -flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) +flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { const struct mlx5_flow_driver_ops *fops; enum mlx5_flow_drv_type type = flow->drv_type; MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); fops = flow_get_drv_ops(type); - fops->remove(dev, flow); + fops->destroy(dev, flow); } /** - * Flow driver destroy API. This abstracts calling driver specific functions. - * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow - * on device and releases resources of the flow. + * Flow driver find RSS policy tbl API. This abstracts calling driver + * specific functions. Parent flow (rte_flow) should have driver + * type (drv_type). It will find the RSS policy table that has the rss_desc. * * @param[in] dev * Pointer to Ethernet device. * @param[in, out] flow * Pointer to flow structure. + * @param[in] policy + * Pointer to meter policy table. + * @param[in] rss_desc + * Pointer to rss_desc */ -static inline void -flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) +static struct mlx5_flow_meter_sub_policy * +flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct mlx5_flow_meter_policy *policy, + struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) { const struct mlx5_flow_driver_ops *fops; enum mlx5_flow_drv_type type = flow->drv_type; - flow_mreg_split_qrss_release(dev, flow); MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); fops = flow_get_drv_ops(type); - fops->destroy(dev, flow); + return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); +} + +/** + * Flow driver color tag rule API. This abstracts calling driver + * specific functions. Parent flow (rte_flow) should have driver + * type (drv_type). It will create the color tag rules in hierarchy meter. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in, out] flow + * Pointer to flow structure. + * @param[in] fm + * Pointer to flow meter structure. + * @param[in] src_port + * The src port this extra rule should use. + * @param[in] item + * The src port id match item. + * @param[out] error + * Pointer to error structure. + */ +static int +flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct mlx5_flow_meter_info *fm, + int32_t src_port, + const struct rte_flow_item *item, + struct rte_flow_error *error) +{ + const struct mlx5_flow_driver_ops *fops; + enum mlx5_flow_drv_type type = flow->drv_type; + + MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); + fops = flow_get_drv_ops(type); + return fops->meter_hierarchy_rule_create(dev, fm, + src_port, item, error); } /** * Get RSS action from the action list. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] actions * Pointer to the list of actions. + * @param[in] flow + * Parent flow structure pointer. * * @return * Pointer to the RSS action if exist, else return NULL. */ static const struct rte_flow_action_rss* -flow_get_rss_action(const struct rte_flow_action actions[]) +flow_get_rss_action(struct rte_eth_dev *dev, + const struct rte_flow_action actions[]) { + struct mlx5_priv *priv = dev->data->dev_private; + const struct rte_flow_action_rss *rss = NULL; + struct mlx5_meter_policy_action_container *acg; + struct mlx5_meter_policy_action_container *acy; + for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { switch (actions->type) { case RTE_FLOW_ACTION_TYPE_RSS: - return (const struct rte_flow_action_rss *) - actions->conf; + rss = actions->conf; + break; + case RTE_FLOW_ACTION_TYPE_SAMPLE: + { + const struct rte_flow_action_sample *sample = + actions->conf; + const struct rte_flow_action *act = sample->actions; + for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) + if (act->type == RTE_FLOW_ACTION_TYPE_RSS) + rss = act->conf; + break; + } + case RTE_FLOW_ACTION_TYPE_METER: + { + uint32_t mtr_idx; + struct mlx5_flow_meter_info *fm; + struct mlx5_flow_meter_policy *policy; + const struct rte_flow_action_meter *mtr = actions->conf; + + fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); + if (fm && !fm->def_policy) { + policy = mlx5_flow_meter_policy_find(dev, + fm->policy_id, NULL); + MLX5_ASSERT(policy); + if (policy->is_hierarchy) { + policy = + mlx5_flow_meter_hierarchy_get_final_policy(dev, + policy); + if (!policy) + return NULL; + } + if (policy->is_rss) { + acg = + &policy->act_cnt[RTE_COLOR_GREEN]; + acy = + &policy->act_cnt[RTE_COLOR_YELLOW]; + if (acg->fate_action == + MLX5_FLOW_FATE_SHARED_RSS) + rss = acg->rss->conf; + else if (acy->fate_action == + MLX5_FLOW_FATE_SHARED_RSS) + rss = acy->rss->conf; + } + } + break; + } default: break; } } - return NULL; + return rss; } -static unsigned int -find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) +/** + * Get ASO age action by index. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] age_idx + * Index to the ASO age action. + * + * @return + * The specified ASO age action. + */ +struct mlx5_aso_age_action* +flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) { - const struct rte_flow_item *item; - unsigned int has_vlan = 0; + uint16_t pool_idx = age_idx & UINT16_MAX; + uint16_t offset = (age_idx >> 16) & UINT16_MAX; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; + struct mlx5_aso_age_pool *pool; + + rte_rwlock_read_lock(&mng->resize_rwl); + pool = mng->pools[pool_idx]; + rte_rwlock_read_unlock(&mng->resize_rwl); + return &pool->actions[offset - 1]; +} + +/* maps indirect action to translated direct in some actions array */ +struct mlx5_translated_action_handle { + struct rte_flow_action_handle *action; /**< Indirect action handle. */ + int index; /**< Index in related array of rte_flow_action. */ +}; + +/** + * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related + * direct action if translation possible. + * This functionality used to run same execution path for both direct and + * indirect actions on flow create. All necessary preparations for indirect + * action handling should be performed on *handle* actions list returned + * from this call. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] actions + * List of actions to translate. + * @param[out] handle + * List to store translated indirect action object handles. + * @param[in, out] indir_n + * Size of *handle* array. On return should be updated with number of + * indirect actions retrieved from the *actions* list. + * @param[out] translated_actions + * List of actions where all indirect actions were translated to direct + * if possible. NULL if no translation took place. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_action_handles_translate(struct rte_eth_dev *dev, + const struct rte_flow_action actions[], + struct mlx5_translated_action_handle *handle, + int *indir_n, + struct rte_flow_action **translated_actions, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct rte_flow_action *translated = NULL; + size_t actions_size; + int n; + int copied_n = 0; + struct mlx5_translated_action_handle *handle_end = NULL; + + for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { + if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) + continue; + if (copied_n == *indir_n) { + return rte_flow_error_set + (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "too many shared actions"); + } + rte_memcpy(&handle[copied_n].action, &actions[n].conf, + sizeof(actions[n].conf)); + handle[copied_n].index = n; + copied_n++; + } + n++; + *indir_n = copied_n; + if (!copied_n) + return 0; + actions_size = sizeof(struct rte_flow_action) * n; + translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); + if (!translated) { + rte_errno = ENOMEM; + return -ENOMEM; + } + memcpy(translated, actions, actions_size); + for (handle_end = handle + copied_n; handle < handle_end; handle++) { + struct mlx5_shared_action_rss *shared_rss; + uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; + uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; + uint32_t idx = act_idx & + ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); + + switch (type) { + case MLX5_INDIRECT_ACTION_TYPE_RSS: + shared_rss = mlx5_ipool_get + (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); + translated[handle->index].type = + RTE_FLOW_ACTION_TYPE_RSS; + translated[handle->index].conf = + &shared_rss->origin; + break; + case MLX5_INDIRECT_ACTION_TYPE_COUNT: + translated[handle->index].type = + (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_COUNT; + translated[handle->index].conf = (void *)(uintptr_t)idx; + break; + case MLX5_INDIRECT_ACTION_TYPE_AGE: + if (priv->sh->flow_hit_aso_en) { + translated[handle->index].type = + (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_AGE; + translated[handle->index].conf = + (void *)(uintptr_t)idx; + break; + } + /* Fall-through */ + case MLX5_INDIRECT_ACTION_TYPE_CT: + if (priv->sh->ct_aso_en) { + translated[handle->index].type = + RTE_FLOW_ACTION_TYPE_CONNTRACK; + translated[handle->index].conf = + (void *)(uintptr_t)idx; + break; + } + /* Fall-through */ + default: + mlx5_free(translated); + return rte_flow_error_set + (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "invalid indirect action type"); + } + } + *translated_actions = translated; + return 0; +} - for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { - if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { - has_vlan = 1; +/** + * Get Shared RSS action from the action list. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] shared + * Pointer to the list of actions. + * @param[in] shared_n + * Actions list length. + * + * @return + * The MLX5 RSS action ID if exists, otherwise return 0. + */ +static uint32_t +flow_get_shared_rss_action(struct rte_eth_dev *dev, + struct mlx5_translated_action_handle *handle, + int shared_n) +{ + struct mlx5_translated_action_handle *handle_end; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_shared_action_rss *shared_rss; + + + for (handle_end = handle + shared_n; handle < handle_end; handle++) { + uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; + uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; + uint32_t idx = act_idx & + ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); + switch (type) { + case MLX5_INDIRECT_ACTION_TYPE_RSS: + shared_rss = mlx5_ipool_get + (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], + idx); + __atomic_add_fetch(&shared_rss->refcnt, 1, + __ATOMIC_RELAXED); + return idx; + default: break; } } - if (has_vlan) - return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : - MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; + return 0; +} + +static unsigned int +find_graph_root(uint32_t rss_level) +{ return rss_level < 2 ? MLX5_EXPANSION_ROOT : MLX5_EXPANSION_ROOT_OUTER; } @@ -2769,7 +3970,7 @@ find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) * subflow. * * @param[in] dev_flow - * Pointer the created preifx subflow. + * Pointer the created prefix subflow. * * @return * The layers get from prefix subflow. @@ -2856,32 +4057,104 @@ flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], return actions_n + 1; } +/** + * Check if the action will change packet. + * + * @param dev + * Pointer to Ethernet device. + * @param[in] type + * action type. + * + * @return + * true if action will change packet, false otherwise. + */ +static bool flow_check_modify_action_type(struct rte_eth_dev *dev, + enum rte_flow_action_type type) +{ + struct mlx5_priv *priv = dev->data->dev_private; + + switch (type) { + case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: + case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: + case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: + case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: + case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: + case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: + case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: + case RTE_FLOW_ACTION_TYPE_SET_TP_DST: + case RTE_FLOW_ACTION_TYPE_DEC_TTL: + case RTE_FLOW_ACTION_TYPE_SET_TTL: + case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: + case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: + case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: + case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: + case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: + case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: + case RTE_FLOW_ACTION_TYPE_SET_META: + case RTE_FLOW_ACTION_TYPE_SET_TAG: + case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: + case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: + case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: + case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: + case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: + case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: + case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: + case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: + case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: + case RTE_FLOW_ACTION_TYPE_RAW_DECAP: + case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: + return true; + case RTE_FLOW_ACTION_TYPE_FLAG: + case RTE_FLOW_ACTION_TYPE_MARK: + if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) + return true; + else + return false; + default: + return false; + } +} + /** * Check meter action from the action list. * + * @param dev + * Pointer to Ethernet device. * @param[in] actions * Pointer to the list of actions. - * @param[out] mtr + * @param[out] has_mtr * Pointer to the meter exist flag. + * @param[out] has_modify + * Pointer to the flag showing there's packet change action. + * @param[out] meter_id + * Pointer to the meter id. * * @return * Total number of actions. */ static int -flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) +flow_check_meter_action(struct rte_eth_dev *dev, + const struct rte_flow_action actions[], + bool *has_mtr, bool *has_modify, uint32_t *meter_id) { + const struct rte_flow_action_meter *mtr = NULL; int actions_n = 0; - MLX5_ASSERT(mtr); - *mtr = 0; + MLX5_ASSERT(has_mtr); + *has_mtr = false; for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: - *mtr = 1; + mtr = actions->conf; + *meter_id = mtr->mtr_id; + *has_mtr = true; break; default: break; } + if (!*has_mtr) + *has_modify |= flow_check_modify_action_type(dev, + actions->type); actions_n++; } /* Count RTE_FLOW_ACTION_TYPE_END. */ @@ -2889,10 +4162,10 @@ flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) } /** - * Check if the flow should be splited due to hairpin. + * Check if the flow should be split due to hairpin. * The reason for the split is that in current HW we can't - * support encap on Rx, so if a flow have encap we move it - * to Tx. + * support encap and push-vlan on Rx, so if a flow contains + * these actions we move it to Tx. * * @param dev * Pointer to Ethernet device. @@ -2912,10 +4185,11 @@ flow_check_hairpin_split(struct rte_eth_dev *dev, { int queue_action = 0; int action_n = 0; - int encap = 0; + int split = 0; const struct rte_flow_action_queue *queue; const struct rte_flow_action_rss *rss; const struct rte_flow_action_raw_encap *raw_encap; + const struct rte_eth_hairpin_conf *conf; if (!attr->ingress) return 0; @@ -2925,8 +4199,8 @@ flow_check_hairpin_split(struct rte_eth_dev *dev, queue = actions->conf; if (queue == NULL) return 0; - if (mlx5_rxq_get_type(dev, queue->index) != - MLX5_RXQ_TYPE_HAIRPIN) + conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); + if (conf == NULL || conf->tx_explicit != 0) return 0; queue_action = 1; action_n++; @@ -2935,23 +4209,24 @@ flow_check_hairpin_split(struct rte_eth_dev *dev, rss = actions->conf; if (rss == NULL || rss->queue_num == 0) return 0; - if (mlx5_rxq_get_type(dev, rss->queue[0]) != - MLX5_RXQ_TYPE_HAIRPIN) + conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); + if (conf == NULL || conf->tx_explicit != 0) return 0; queue_action = 1; action_n++; break; case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: - encap = 1; + case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: + case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: + case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: + split++; action_n++; break; case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: raw_encap = actions->conf; - if (raw_encap->size > - (sizeof(struct rte_flow_item_eth) + - sizeof(struct rte_flow_item_ipv4))) - encap = 1; + if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) + split++; action_n++; break; default: @@ -2959,53 +4234,45 @@ flow_check_hairpin_split(struct rte_eth_dev *dev, break; } } - if (encap == 1 && queue_action) + if (split && queue_action) return action_n; return 0; } /* Declare flow create/destroy prototype in advance. */ static uint32_t -flow_list_create(struct rte_eth_dev *dev, uint32_t *list, +flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, const struct rte_flow_attr *attr, const struct rte_flow_item items[], const struct rte_flow_action actions[], bool external, struct rte_flow_error *error); static void -flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, +flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, uint32_t flow_idx); -/** - * Add a flow of copying flow metadata registers in RX_CP_TBL. - * - * As mark_id is unique, if there's already a registered flow for the mark_id, - * return by increasing the reference counter of the resource. Otherwise, create - * the resource (mcp_res) and flow. - * - * Flow looks like, - * - If ingress port is ANY and reg_c[1] is mark_id, - * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. - * - * For default flow (zero mark_id), flow is like, - * - If ingress port is ANY, - * reg_b := reg_c[0] and jump to RX_ACT_TBL. - * - * @param dev - * Pointer to Ethernet device. - * @param mark_id - * ID of MARK action, zero means default flow for META. - * @param[out] error - * Perform verbose error reporting if not NULL. - * - * @return - * Associated resource on success, NULL otherwise and rte_errno is set. - */ -static struct mlx5_flow_mreg_copy_resource * -flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, - struct rte_flow_error *error) +int +flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry, void *cb_ctx) +{ + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_mreg_copy_resource *mcp_res = + container_of(entry, typeof(*mcp_res), hlist_ent); + + return mcp_res->mark_id != *(uint32_t *)(ctx->data); +} + +struct mlx5_list_entry * +flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) { + struct rte_eth_dev *dev = tool_ctx; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_mreg_copy_resource *mcp_res; + struct rte_flow_error *error = ctx->error; + uint32_t idx = 0; + int ret; + uint32_t mark_id = *(uint32_t *)(ctx->data); struct rte_flow_attr attr = { .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, .ingress = 1, @@ -3021,7 +4288,7 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, }; struct mlx5_flow_action_copy_mreg cp_mreg = { .dst = REG_B, - .src = 0, + .src = REG_NON, }; struct rte_flow_action_jump jump = { .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, @@ -3029,11 +4296,8 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, struct rte_flow_action actions[] = { [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, }; - struct mlx5_flow_mreg_copy_resource *mcp_res; - uint32_t idx = 0; - int ret; - /* Fill the register fileds in the flow. */ + /* Fill the register fields in the flow. */ ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); if (ret < 0) return NULL; @@ -3042,17 +4306,6 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, if (ret < 0) return NULL; cp_mreg.src = ret; - /* Check if already registered. */ - MLX5_ASSERT(priv->mreg_cp_tbl); - mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id); - if (mcp_res) { - /* For non-default rule. */ - if (mark_id != MLX5_DEFAULT_COPY_ID) - mcp_res->refcnt++; - MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID || - mcp_res->refcnt == 1); - return mcp_res; - } /* Provide the full width of FLAG specific value. */ if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) tag_spec.data = MLX5_FLOW_MARK_DEFAULT; @@ -3085,7 +4338,7 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, }; } else { /* Default rule, wildcard match. */ - attr.priority = MLX5_FLOW_PRIO_RSVD; + attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; items[0] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_END, }; @@ -3109,29 +4362,109 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, return NULL; } mcp_res->idx = idx; + mcp_res->mark_id = mark_id; /* * The copy Flows are not included in any list. There * ones are referenced from other Flows and can not - * be applied, removed, deleted in ardbitrary order + * be applied, removed, deleted in arbitrary order * by list traversing. */ - mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, - actions, false, error); - if (!mcp_res->rix_flow) - goto error; - mcp_res->refcnt++; - mcp_res->hlist_ent.key = mark_id; - ret = mlx5_hlist_insert(priv->mreg_cp_tbl, - &mcp_res->hlist_ent); - MLX5_ASSERT(!ret); - if (ret) - goto error; - return mcp_res; -error: - if (mcp_res->rix_flow) - flow_list_destroy(dev, NULL, mcp_res->rix_flow); + mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, + &attr, items, actions, false, error); + if (!mcp_res->rix_flow) { + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); + return NULL; + } + return &mcp_res->hlist_ent; +} + +struct mlx5_list_entry * +flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, + void *cb_ctx __rte_unused) +{ + struct rte_eth_dev *dev = tool_ctx; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_mreg_copy_resource *mcp_res; + uint32_t idx = 0; + + mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); + if (!mcp_res) { + rte_errno = ENOMEM; + return NULL; + } + memcpy(mcp_res, oentry, sizeof(*mcp_res)); + mcp_res->idx = idx; + return &mcp_res->hlist_ent; +} + +void +flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) +{ + struct mlx5_flow_mreg_copy_resource *mcp_res = + container_of(entry, typeof(*mcp_res), hlist_ent); + struct rte_eth_dev *dev = tool_ctx; + struct mlx5_priv *priv = dev->data->dev_private; + + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); +} + +/** + * Add a flow of copying flow metadata registers in RX_CP_TBL. + * + * As mark_id is unique, if there's already a registered flow for the mark_id, + * return by increasing the reference counter of the resource. Otherwise, create + * the resource (mcp_res) and flow. + * + * Flow looks like, + * - If ingress port is ANY and reg_c[1] is mark_id, + * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. + * + * For default flow (zero mark_id), flow is like, + * - If ingress port is ANY, + * reg_b := reg_c[0] and jump to RX_ACT_TBL. + * + * @param dev + * Pointer to Ethernet device. + * @param mark_id + * ID of MARK action, zero means default flow for META. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * Associated resource on success, NULL otherwise and rte_errno is set. + */ +static struct mlx5_flow_mreg_copy_resource * +flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_list_entry *entry; + struct mlx5_flow_cb_ctx ctx = { + .dev = dev, + .error = error, + .data = &mark_id, + }; + + /* Check if already registered. */ + MLX5_ASSERT(priv->mreg_cp_tbl); + entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); + if (!entry) + return NULL; + return container_of(entry, struct mlx5_flow_mreg_copy_resource, + hlist_ent); +} + +void +flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) +{ + struct mlx5_flow_mreg_copy_resource *mcp_res = + container_of(entry, typeof(*mcp_res), hlist_ent); + struct rte_eth_dev *dev = tool_ctx; + struct mlx5_priv *priv = dev->data->dev_private; + + MLX5_ASSERT(mcp_res->rix_flow); + flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); - return NULL; } /** @@ -3155,109 +4488,16 @@ flow_mreg_del_copy_action(struct rte_eth_dev *dev, flow->rix_mreg_copy); if (!mcp_res || !priv->mreg_cp_tbl) return; - if (flow->copy_applied) { - MLX5_ASSERT(mcp_res->appcnt); - flow->copy_applied = 0; - --mcp_res->appcnt; - if (!mcp_res->appcnt) { - struct rte_flow *mcp_flow = mlx5_ipool_get - (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], - mcp_res->rix_flow); - - if (mcp_flow) - flow_drv_remove(dev, mcp_flow); - } - } - /* - * We do not check availability of metadata registers here, - * because copy resources are not allocated in this case. - */ - if (--mcp_res->refcnt) - return; MLX5_ASSERT(mcp_res->rix_flow); - flow_list_destroy(dev, NULL, mcp_res->rix_flow); - mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); + mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); flow->rix_mreg_copy = 0; } /** - * Start flow in RX_CP_TBL. - * - * @param dev - * Pointer to Ethernet device. - * @flow - * Parent flow for wich copying is provided. - * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. - */ -static int -flow_mreg_start_copy_action(struct rte_eth_dev *dev, - struct rte_flow *flow) -{ - struct mlx5_flow_mreg_copy_resource *mcp_res; - struct mlx5_priv *priv = dev->data->dev_private; - int ret; - - if (!flow->rix_mreg_copy || flow->copy_applied) - return 0; - mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], - flow->rix_mreg_copy); - if (!mcp_res) - return 0; - if (!mcp_res->appcnt) { - struct rte_flow *mcp_flow = mlx5_ipool_get - (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], - mcp_res->rix_flow); - - if (mcp_flow) { - ret = flow_drv_apply(dev, mcp_flow, NULL); - if (ret) - return ret; - } - } - ++mcp_res->appcnt; - flow->copy_applied = 1; - return 0; -} - -/** - * Stop flow in RX_CP_TBL. - * - * @param dev - * Pointer to Ethernet device. - * @flow - * Parent flow for wich copying is provided. - */ -static void -flow_mreg_stop_copy_action(struct rte_eth_dev *dev, - struct rte_flow *flow) -{ - struct mlx5_flow_mreg_copy_resource *mcp_res; - struct mlx5_priv *priv = dev->data->dev_private; - - if (!flow->rix_mreg_copy || !flow->copy_applied) - return; - mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], - flow->rix_mreg_copy); - if (!mcp_res) - return; - MLX5_ASSERT(mcp_res->appcnt); - --mcp_res->appcnt; - flow->copy_applied = 0; - if (!mcp_res->appcnt) { - struct rte_flow *mcp_flow = mlx5_ipool_get - (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], - mcp_res->rix_flow); - - if (mcp_flow) - flow_drv_remove(dev, mcp_flow); - } -} - -/** - * Remove the default copy action from RX_CP_TBL. + * Remove the default copy action from RX_CP_TBL. + * + * This functions is called in the mlx5_dev_start(). No thread safe + * is guaranteed. * * @param dev * Pointer to Ethernet device. @@ -3265,25 +4505,28 @@ flow_mreg_stop_copy_action(struct rte_eth_dev *dev, static void flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) { - struct mlx5_flow_mreg_copy_resource *mcp_res; + struct mlx5_list_entry *entry; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_cb_ctx ctx; + uint32_t mark_id; /* Check if default flow is registered. */ if (!priv->mreg_cp_tbl) return; - mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, - MLX5_DEFAULT_COPY_ID); - if (!mcp_res) + mark_id = MLX5_DEFAULT_COPY_ID; + ctx.data = &mark_id; + entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); + if (!entry) return; - MLX5_ASSERT(mcp_res->rix_flow); - flow_list_destroy(dev, NULL, mcp_res->rix_flow); - mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); + mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); } /** * Add the default copy action in in RX_CP_TBL. * + * This functions is called in the mlx5_dev_start(). No thread safe + * is guaranteed. + * * @param dev * Pointer to Ethernet device. * @param[out] error @@ -3298,6 +4541,8 @@ flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_mreg_copy_resource *mcp_res; + struct mlx5_flow_cb_ctx ctx; + uint32_t mark_id; /* Check whether extensive metadata feature is engaged. */ if (!priv->config.dv_flow_en || @@ -3305,7 +4550,15 @@ flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, !mlx5_flow_ext_mreg_supported(dev) || !priv->sh->dv_regc0_mask) return 0; - mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); + /* + * Add default mreg copy flow may be called multiple time, but + * only be called once in stop. Avoid register it twice. + */ + mark_id = MLX5_DEFAULT_COPY_ID; + ctx.data = &mark_id; + if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) + return 0; + mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); if (!mcp_res) return -rte_errno; return 0; @@ -3371,10 +4624,6 @@ flow_mreg_update_copy_table(struct rte_eth_dev *dev, if (!mcp_res) return -rte_errno; flow->rix_mreg_copy = mcp_res->idx; - if (dev->data->dev_started) { - mcp_res->appcnt++; - flow->copy_applied = 1; - } return 0; case RTE_FLOW_ACTION_TYPE_MARK: mark = (const struct rte_flow_action_mark *) @@ -3384,10 +4633,6 @@ flow_mreg_update_copy_table(struct rte_eth_dev *dev, if (!mcp_res) return -rte_errno; flow->rix_mreg_copy = mcp_res->idx; - if (dev->data->dev_started) { - mcp_res->appcnt++; - flow->copy_applied = 1; - } return 0; default: break; @@ -3401,7 +4646,8 @@ flow_mreg_update_copy_table(struct rte_eth_dev *dev, /** * Split the hairpin flow. - * Since HW can't support encap on Rx we move the encap to Tx. + * Since HW can't support encap and push-vlan on Rx, we move these + * actions to Tx. * If the count action is after the encap then we also * move the count action. in this case the count will also measure * the outer bytes. @@ -3428,9 +4674,8 @@ flow_hairpin_split(struct rte_eth_dev *dev, struct rte_flow_action actions_rx[], struct rte_flow_action actions_tx[], struct rte_flow_item pattern_tx[], - uint32_t *flow_id) + uint32_t flow_id) { - struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_action_raw_encap *raw_encap; const struct rte_flow_action_raw_decap *raw_decap; struct mlx5_rte_flow_action_set_tag *set_tag; @@ -3440,11 +4685,13 @@ flow_hairpin_split(struct rte_eth_dev *dev, char *addr; int encap = 0; - mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id); for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { switch (actions->type) { case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: + case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: + case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: + case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); actions_tx++; @@ -3462,9 +4709,7 @@ flow_hairpin_split(struct rte_eth_dev *dev, break; case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: raw_encap = actions->conf; - if (raw_encap->size > - (sizeof(struct rte_flow_item_eth) + - sizeof(struct rte_flow_item_ipv4))) { + if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); actions_tx++; @@ -3477,9 +4722,7 @@ flow_hairpin_split(struct rte_eth_dev *dev, break; case RTE_FLOW_ACTION_TYPE_RAW_DECAP: raw_decap = actions->conf; - if (raw_decap->size < - (sizeof(struct rte_flow_item_eth) + - sizeof(struct rte_flow_item_ipv4))) { + if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); actions_tx++; @@ -3504,9 +4747,11 @@ flow_hairpin_split(struct rte_eth_dev *dev, rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); actions_rx++; set_tag = (void *)actions_rx; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); - MLX5_ASSERT(set_tag->id > REG_NONE); - set_tag->data = *flow_id; + *set_tag = (struct mlx5_rte_flow_action_set_tag) { + .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), + .data = flow_id, + }; + MLX5_ASSERT(set_tag->id > REG_NON); tag_action->conf = set_tag; /* Create Tx item list. */ rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); @@ -3515,16 +4760,15 @@ flow_hairpin_split(struct rte_eth_dev *dev, item->type = (enum rte_flow_item_type) MLX5_RTE_FLOW_ITEM_TYPE_TAG; tag_item = (void *)addr; - tag_item->data = *flow_id; + tag_item->data = flow_id; tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); - MLX5_ASSERT(set_tag->id > REG_NONE); + MLX5_ASSERT(set_tag->id > REG_NON); item->spec = tag_item; addr += sizeof(struct mlx5_rte_flow_item_tag); tag_item = (void *)addr; tag_item->data = UINT32_MAX; tag_item->id = UINT16_MAX; item->mask = tag_item; - addr += sizeof(struct mlx5_rte_flow_item_tag); item->last = NULL; item++; item->type = RTE_FLOW_ITEM_TYPE_END; @@ -3541,18 +4785,14 @@ flow_hairpin_split(struct rte_eth_dev *dev, * Parent flow structure pointer. * @param[in, out] sub_flow * Pointer to return the created subflow, may be NULL. - * @param[in] prefix_layers - * Prefix subflow layers, may be 0. * @param[in] attr * Flow rule attributes. * @param[in] items * Pattern specification (list terminated by the END pattern item). * @param[in] actions * Associated actions (list terminated by the END action). - * @param[in] external - * This flow rule is created by request external to PMD. - * @param[in] flow_idx - * This memory pool index to the flow. + * @param[in] flow_split_info + * Pointer to flow split info structure. * @param[out] error * Perform verbose error reporting if not NULL. * @return @@ -3562,35 +4802,163 @@ static int flow_create_split_inner(struct rte_eth_dev *dev, struct rte_flow *flow, struct mlx5_flow **sub_flow, - uint64_t prefix_layers, const struct rte_flow_attr *attr, const struct rte_flow_item items[], const struct rte_flow_action actions[], - bool external, uint32_t flow_idx, + struct mlx5_flow_split_info *flow_split_info, struct rte_flow_error *error) { struct mlx5_flow *dev_flow; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, - flow_idx, error); + flow_split_info->flow_idx, error); if (!dev_flow) return -rte_errno; dev_flow->flow = flow; - dev_flow->external = external; + dev_flow->external = flow_split_info->external; + dev_flow->skip_scale = flow_split_info->skip_scale; /* Subflow object was created, we must include one in the list. */ SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, dev_flow->handle, next); /* * If dev_flow is as one of the suffix flow, some actions in suffix - * flow may need some user defined item layer flags. + * flow may need some user defined item layer flags, and pass the + * Metadata rxq mark flag to suffix flow as well. */ - if (prefix_layers) - dev_flow->handle->layers = prefix_layers; + if (flow_split_info->prefix_layers) + dev_flow->handle->layers = flow_split_info->prefix_layers; + if (flow_split_info->prefix_mark) { + MLX5_ASSERT(wks); + wks->mark = 1; + } if (sub_flow) *sub_flow = dev_flow; +#ifdef HAVE_IBV_FLOW_DV_SUPPORT + dev_flow->dv.table_id = flow_split_info->table_id; +#endif return flow_drv_translate(dev, dev_flow, attr, items, actions, error); } +/** + * Get the sub policy of a meter. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] flow + * Parent flow structure pointer. + * @param wks + * Pointer to thread flow work space. + * @param[in] attr + * Flow rule attributes. + * @param[in] items + * Pattern specification (list terminated by the END pattern item). + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. + */ +static struct mlx5_flow_meter_sub_policy * +get_meter_sub_policy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct mlx5_flow_workspace *wks, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + struct rte_flow_error *error) +{ + struct mlx5_flow_meter_policy *policy; + struct mlx5_flow_meter_policy *final_policy; + struct mlx5_flow_meter_sub_policy *sub_policy = NULL; + + policy = wks->policy; + final_policy = policy->is_hierarchy ? wks->final_policy : policy; + if (final_policy->is_rss || final_policy->is_queue) { + struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; + struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; + uint32_t i; + + /* + * This is a tmp dev_flow, + * no need to register any matcher for it in translate. + */ + wks->skip_matcher_reg = 1; + for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { + struct mlx5_flow dev_flow = {0}; + struct mlx5_flow_handle dev_handle = { {0} }; + uint8_t fate = final_policy->act_cnt[i].fate_action; + + if (fate == MLX5_FLOW_FATE_SHARED_RSS) { + const struct rte_flow_action_rss *rss_act = + final_policy->act_cnt[i].rss->conf; + struct rte_flow_action rss_actions[2] = { + [0] = { + .type = RTE_FLOW_ACTION_TYPE_RSS, + .conf = rss_act, + }, + [1] = { + .type = RTE_FLOW_ACTION_TYPE_END, + .conf = NULL, + } + }; + + dev_flow.handle = &dev_handle; + dev_flow.ingress = attr->ingress; + dev_flow.flow = flow; + dev_flow.external = 0; +#ifdef HAVE_IBV_FLOW_DV_SUPPORT + dev_flow.dv.transfer = attr->transfer; +#endif + /** + * Translate RSS action to get rss hash fields. + */ + if (flow_drv_translate(dev, &dev_flow, attr, + items, rss_actions, error)) + goto exit; + rss_desc_v[i] = wks->rss_desc; + rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; + rss_desc_v[i].hash_fields = + dev_flow.hash_fields; + rss_desc_v[i].queue_num = + rss_desc_v[i].hash_fields ? + rss_desc_v[i].queue_num : 1; + rss_desc_v[i].tunnel = + !!(dev_flow.handle->layers & + MLX5_FLOW_LAYER_TUNNEL); + /* Use the RSS queues in the containers. */ + rss_desc_v[i].queue = + (uint16_t *)(uintptr_t)rss_act->queue; + rss_desc[i] = &rss_desc_v[i]; + } else if (fate == MLX5_FLOW_FATE_QUEUE) { + /* This is queue action. */ + rss_desc_v[i] = wks->rss_desc; + rss_desc_v[i].key_len = 0; + rss_desc_v[i].hash_fields = 0; + rss_desc_v[i].queue = + &final_policy->act_cnt[i].queue; + rss_desc_v[i].queue_num = 1; + rss_desc[i] = &rss_desc_v[i]; + } else { + rss_desc[i] = NULL; + } + } + sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, + flow, policy, rss_desc); + } else { + enum mlx5_meter_domain mtr_domain = + attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : + (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : + MLX5_MTR_DOMAIN_INGRESS); + sub_policy = policy->sub_policys[mtr_domain][0]; + } + if (!sub_policy) + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to get meter sub-policy."); +exit: + return sub_policy; +} + /** * Split the meter flow. * @@ -3603,8 +4971,14 @@ flow_create_split_inner(struct rte_eth_dev *dev, * header will be in the prefix sub flow, as not to take the * L3 tunnel header into account. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. + * @param[in] flow + * Parent flow structure pointer. + * @param wks + * Pointer to thread flow work space. + * @param[in] attr + * Flow rule attributes. * @param[in] items * Pattern specification (list terminated by the END pattern item). * @param[out] sfx_items @@ -3615,125 +4989,248 @@ flow_create_split_inner(struct rte_eth_dev *dev, * Suffix flow actions. * @param[out] actions_pre * Prefix flow actions. - * @param[out] pattern_sfx - * The pattern items for the suffix flow. - * @param[out] tag_sfx - * Pointer to suffix flow tag. + * @param[out] mtr_flow_id + * Pointer to meter flow id. + * @param[out] error + * Perform verbose error reporting if not NULL. * * @return - * 0 on success. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int flow_meter_split_prep(struct rte_eth_dev *dev, - const struct rte_flow_item items[], - struct rte_flow_item sfx_items[], - const struct rte_flow_action actions[], - struct rte_flow_action actions_sfx[], - struct rte_flow_action actions_pre[]) + struct rte_flow *flow, + struct mlx5_flow_workspace *wks, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + struct rte_flow_item sfx_items[], + const struct rte_flow_action actions[], + struct rte_flow_action actions_sfx[], + struct rte_flow_action actions_pre[], + uint32_t *mtr_flow_id, + struct rte_flow_error *error) { + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter_info *fm = wks->fm; struct rte_flow_action *tag_action = NULL; struct rte_flow_item *tag_item; struct mlx5_rte_flow_action_set_tag *set_tag; - struct rte_flow_error error; const struct rte_flow_action_raw_encap *raw_encap; const struct rte_flow_action_raw_decap *raw_decap; - struct mlx5_rte_flow_item_tag *tag_spec; - struct mlx5_rte_flow_item_tag *tag_mask; - uint32_t tag_id; - bool copy_vlan = false; + struct mlx5_rte_flow_item_tag *tag_item_spec; + struct mlx5_rte_flow_item_tag *tag_item_mask; + uint32_t tag_id = 0; + struct rte_flow_item *vlan_item_dst = NULL; + const struct rte_flow_item *vlan_item_src = NULL; + struct rte_flow_action *hw_mtr_action; + struct rte_flow_action *action_pre_head = NULL; + int32_t flow_src_port = priv->representor_id; + bool mtr_first; + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; + uint32_t flow_id = 0; + uint32_t flow_id_reversed = 0; + uint8_t flow_id_bits = 0; + int shift; + + /* Prepare the suffix subflow items. */ + tag_item = sfx_items++; + for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { + struct mlx5_priv *port_priv; + const struct rte_flow_item_port_id *pid_v; + int item_type = items->type; + switch (item_type) { + case RTE_FLOW_ITEM_TYPE_PORT_ID: + pid_v = items->spec; + MLX5_ASSERT(pid_v); + port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); + if (!port_priv) + return rte_flow_error_set(error, + rte_errno, + RTE_FLOW_ERROR_TYPE_ITEM_SPEC, + pid_v, + "Failed to get port info."); + flow_src_port = port_priv->representor_id; + if (!fm->def_policy && wks->policy->is_hierarchy && + flow_src_port != priv->representor_id) { + if (flow_drv_mtr_hierarchy_rule_create(dev, + flow, fm, + flow_src_port, + items, + error)) + return -rte_errno; + } + memcpy(sfx_items, items, sizeof(*sfx_items)); + sfx_items++; + break; + case RTE_FLOW_ITEM_TYPE_VLAN: + /* Determine if copy vlan item below. */ + vlan_item_src = items; + vlan_item_dst = sfx_items++; + vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; + break; + default: + break; + } + } + sfx_items->type = RTE_FLOW_ITEM_TYPE_END; + sfx_items++; + mtr_first = priv->sh->meter_aso_en && + (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); + /* For ASO meter, meter must be before tag in TX direction. */ + if (mtr_first) { + action_pre_head = actions_pre++; + /* Leave space for tag action. */ + tag_action = actions_pre++; + } /* Prepare the actions for prefix and suffix flow. */ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { - struct rte_flow_action **action_cur = NULL; + struct rte_flow_action *action_cur = NULL; switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: - /* Add the extra tag action first. */ - tag_action = actions_pre; - tag_action->type = (enum rte_flow_action_type) - MLX5_RTE_FLOW_ACTION_TYPE_TAG; - actions_pre++; - action_cur = &actions_pre; + if (mtr_first) { + action_cur = action_pre_head; + } else { + /* Leave space for tag action. */ + tag_action = actions_pre++; + action_cur = actions_pre++; + } break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: - action_cur = &actions_pre; + action_cur = actions_pre++; break; case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: raw_encap = actions->conf; if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) - action_cur = &actions_pre; + action_cur = actions_pre++; break; case RTE_FLOW_ACTION_TYPE_RAW_DECAP: raw_decap = actions->conf; if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) - action_cur = &actions_pre; + action_cur = actions_pre++; break; case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: - copy_vlan = true; + if (vlan_item_dst && vlan_item_src) { + memcpy(vlan_item_dst, vlan_item_src, + sizeof(*vlan_item_dst)); + /* + * Convert to internal match item, it is used + * for vlan push and set vid. + */ + vlan_item_dst->type = (enum rte_flow_item_type) + MLX5_RTE_FLOW_ITEM_TYPE_VLAN; + } break; default: break; } if (!action_cur) - action_cur = &actions_sfx; - memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); - (*action_cur)++; + action_cur = (fm->def_policy) ? + actions_sfx++ : actions_pre++; + memcpy(action_cur, actions, sizeof(struct rte_flow_action)); } /* Add end action to the actions. */ actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; + if (priv->sh->meter_aso_en) { + /** + * For ASO meter, need to add an extra jump action explicitly, + * to jump from meter to policer table. + */ + struct mlx5_flow_meter_sub_policy *sub_policy; + struct mlx5_flow_tbl_data_entry *tbl_data; + + if (!fm->def_policy) { + sub_policy = get_meter_sub_policy(dev, flow, wks, + attr, items, error); + if (!sub_policy) + return -rte_errno; + } else { + enum mlx5_meter_domain mtr_domain = + attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : + (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : + MLX5_MTR_DOMAIN_INGRESS); + + sub_policy = + &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; + } + tbl_data = container_of(sub_policy->tbl_rsc, + struct mlx5_flow_tbl_data_entry, tbl); + hw_mtr_action = actions_pre++; + hw_mtr_action->type = (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_JUMP; + hw_mtr_action->conf = tbl_data->jump.action; + } actions_pre->type = RTE_FLOW_ACTION_TYPE_END; actions_pre++; - /* Set the tag. */ - set_tag = (void *)actions_pre; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); + if (!tag_action) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "No tag action space."); + if (!mtr_flow_id) { + tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; + goto exit; + } + /* Only default-policy Meter creates mtr flow id. */ + if (fm->def_policy) { + mlx5_ipool_malloc(fm->flow_ipool, &tag_id); + if (!tag_id) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to allocate meter flow id."); + flow_id = tag_id - 1; + flow_id_bits = (!flow_id) ? 1 : + (MLX5_REG_BITS - __builtin_clz(flow_id)); + if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > + mtr_reg_bits) { + mlx5_ipool_free(fm->flow_ipool, tag_id); + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Meter flow id exceeds max limit."); + } + if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) + priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; + } + /* Build tag actions and items for meter_id/meter flow_id. */ + set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; + tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; + tag_item_mask = tag_item_spec + 1; + /* Both flow_id and meter_id share the same register. */ + *set_tag = (struct mlx5_rte_flow_action_set_tag) { + .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, + 0, error), + .offset = mtr_id_offset, + .length = mtr_reg_bits, + .data = flow->meter, + }; /* - * Get the id from the qrss_pool to make qrss share the id with meter. + * The color Reg bits used by flow_id are growing from + * msb to lsb, so must do bit reverse for flow_id val in RegC. */ - tag_id = flow_qrss_get_id(dev); - set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; - assert(tag_action); + for (shift = 0; shift < flow_id_bits; shift++) + flow_id_reversed = (flow_id_reversed << 1) | + ((flow_id >> shift) & 0x1); + set_tag->data |= + flow_id_reversed << (mtr_reg_bits - flow_id_bits); + tag_item_spec->id = set_tag->id; + tag_item_spec->data = set_tag->data << mtr_id_offset; + tag_item_mask->data = UINT32_MAX << mtr_id_offset; + tag_action->type = (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_TAG; tag_action->conf = set_tag; - /* Prepare the suffix subflow items. */ - tag_item = sfx_items++; - for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { - int item_type = items->type; - - switch (item_type) { - case RTE_FLOW_ITEM_TYPE_PORT_ID: - memcpy(sfx_items, items, sizeof(*sfx_items)); - sfx_items++; - break; - case RTE_FLOW_ITEM_TYPE_VLAN: - if (copy_vlan) { - memcpy(sfx_items, items, sizeof(*sfx_items)); - /* - * Convert to internal match item, it is used - * for vlan push and set vid. - */ - sfx_items->type = (enum rte_flow_item_type) - MLX5_RTE_FLOW_ITEM_TYPE_VLAN; - sfx_items++; - } - break; - default: - break; - } - } - sfx_items->type = RTE_FLOW_ITEM_TYPE_END; - sfx_items++; - tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; - tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; - tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - tag_mask = tag_spec + 1; - tag_mask->data = 0xffffff00; tag_item->type = (enum rte_flow_item_type) - MLX5_RTE_FLOW_ITEM_TYPE_TAG; - tag_item->spec = tag_spec; + MLX5_RTE_FLOW_ITEM_TYPE_TAG; + tag_item->spec = tag_item_spec; tag_item->last = NULL; - tag_item->mask = tag_mask; - return tag_id; + tag_item->mask = tag_item_mask; +exit: + if (mtr_flow_id) + *mtr_flow_id = tag_id; + return 0; } /** @@ -3770,6 +5267,8 @@ flow_meter_split_prep(struct rte_eth_dev *dev, * Pointer to the Q/RSS action. * @param[in] actions_n * Number of original actions. + * @param[in] mtr_sfx + * Check if it is in meter suffix table. * @param[out] error * Perform verbose error reporting if not NULL. * @@ -3782,8 +5281,10 @@ flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, struct rte_flow_action *split_actions, const struct rte_flow_action *actions, const struct rte_flow_action *qrss, - int actions_n, struct rte_flow_error *error) + int actions_n, int mtr_sfx, + struct rte_flow_error *error) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_rte_flow_action_set_tag *set_tag; struct rte_flow_action_jump *jump; const int qrss_idx = qrss - actions; @@ -3796,15 +5297,15 @@ flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, * - Add jump to mreg CP_TBL. * As a result, there will be one more action. */ - ++actions_n; memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); + /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ + ++actions_n; set_tag = (void *)(split_actions + actions_n); /* - * If tag action is not set to void(it means we are not the meter - * suffix flow), add the tag action. Since meter suffix flow already - * has the tag added. + * If we are not the meter suffix flow, add the tag action. + * Since meter suffix flow already has the tag added. */ - if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { + if (!mtr_sfx) { /* * Allocate the new subflow ID. This one is unique within * device and not shared with representors. Otherwise, @@ -3815,7 +5316,8 @@ flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, * representors) domain even if they have coinciding * IDs. */ - flow_id = flow_qrss_get_id(dev); + mlx5_ipool_malloc(priv->sh->ipool + [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); if (!flow_id) return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_ACTION, @@ -3836,6 +5338,12 @@ flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, MLX5_RTE_FLOW_ACTION_TYPE_TAG, .conf = set_tag, }; + } else { + /* + * If we are the suffix flow of meter, tag already exist. + * Set the QUEUE/RSS action to void. + */ + split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; } /* JUMP action to jump to mreg copy table (CP_TBL). */ jump = (void *)(set_tag + 1); @@ -3867,7 +5375,7 @@ flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, * @param[out] error * Perform verbose error reporting if not NULL. * @param[in] encap_idx - * The encap action inndex. + * The encap action index. * * @return * 0 on success, negative value otherwise @@ -3916,77 +5424,352 @@ flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, } /** - * The splitting for metadata feature. - * - * - Q/RSS action on NIC Rx should be split in order to pass by - * the mreg copy table (RX_CP_TBL) and then it jumps to the - * action table (RX_ACT_TBL) which has the split Q/RSS action. - * - * - All the actions on NIC Tx should have a mreg copy action to - * copy reg_a from WQE to reg_c[0]. + * Check the match action from the action list. * - * @param dev - * Pointer to Ethernet device. - * @param[in] flow - * Parent flow structure pointer. - * @param[in] prefix_layers - * Prefix flow layer flags. + * @param[in] actions + * Pointer to the list of actions. * @param[in] attr * Flow rule attributes. - * @param[in] items - * Pattern specification (list terminated by the END pattern item). - * @param[in] actions - * Associated actions (list terminated by the END action). - * @param[in] external - * This flow rule is created by request external to PMD. - * @param[in] flow_idx - * This memory pool index to the flow. - * @param[out] error - * Perform verbose error reporting if not NULL. + * @param[in] action + * The action to be check if exist. + * @param[out] match_action_pos + * Pointer to the position of the matched action if exists, otherwise is -1. + * @param[out] qrss_action_pos + * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. + * @param[out] modify_after_mirror + * Pointer to the flag of modify action after FDB mirroring. + * * @return - * 0 on success, negative value otherwise + * > 0 the total number of actions. + * 0 if not found match action in action list. */ static int -flow_create_split_metadata(struct rte_eth_dev *dev, - struct rte_flow *flow, - uint64_t prefix_layers, - const struct rte_flow_attr *attr, - const struct rte_flow_item items[], - const struct rte_flow_action actions[], - bool external, uint32_t flow_idx, - struct rte_flow_error *error) +flow_check_match_action(const struct rte_flow_action actions[], + const struct rte_flow_attr *attr, + enum rte_flow_action_type action, + int *match_action_pos, int *qrss_action_pos, + int *modify_after_mirror) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_config *config = &priv->config; - const struct rte_flow_action *qrss = NULL; - struct rte_flow_action *ext_actions = NULL; - struct mlx5_flow *dev_flow = NULL; - uint32_t qrss_id = 0; - int mtr_sfx = 0; - size_t act_size; - int actions_n; - int encap_idx; - int ret; - - /* Check whether extensive metadata feature is engaged. */ - if (!config->dv_flow_en || - config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || - !mlx5_flow_ext_mreg_supported(dev)) - return flow_create_split_inner(dev, flow, NULL, prefix_layers, - attr, items, actions, external, - flow_idx, error); - actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, - &encap_idx); - if (qrss) { - /* Exclude hairpin flows from splitting. */ - if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { - const struct rte_flow_action_queue *queue; + const struct rte_flow_action_sample *sample; + const struct rte_flow_action_raw_decap *decap; + int actions_n = 0; + uint32_t ratio = 0; + int sub_type = 0; + int flag = 0; + int fdb_mirror = 0; - queue = qrss->conf; - if (mlx5_rxq_get_type(dev, queue->index) == - MLX5_RXQ_TYPE_HAIRPIN) - qrss = NULL; - } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { + *match_action_pos = -1; + *qrss_action_pos = -1; + for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { + if (actions->type == action) { + flag = 1; + *match_action_pos = actions_n; + } + switch (actions->type) { + case RTE_FLOW_ACTION_TYPE_QUEUE: + case RTE_FLOW_ACTION_TYPE_RSS: + *qrss_action_pos = actions_n; + break; + case RTE_FLOW_ACTION_TYPE_SAMPLE: + sample = actions->conf; + ratio = sample->ratio; + sub_type = ((const struct rte_flow_action *) + (sample->actions))->type; + if (ratio == 1 && attr->transfer) + fdb_mirror = 1; + break; + case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: + case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: + case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: + case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: + case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: + case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: + case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: + case RTE_FLOW_ACTION_TYPE_SET_TP_DST: + case RTE_FLOW_ACTION_TYPE_DEC_TTL: + case RTE_FLOW_ACTION_TYPE_SET_TTL: + case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: + case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: + case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: + case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: + case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: + case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: + case RTE_FLOW_ACTION_TYPE_FLAG: + case RTE_FLOW_ACTION_TYPE_MARK: + case RTE_FLOW_ACTION_TYPE_SET_META: + case RTE_FLOW_ACTION_TYPE_SET_TAG: + case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: + case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: + case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: + case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: + case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: + case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: + case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: + case RTE_FLOW_ACTION_TYPE_METER: + if (fdb_mirror) + *modify_after_mirror = 1; + break; + case RTE_FLOW_ACTION_TYPE_RAW_DECAP: + decap = actions->conf; + while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) + ; + actions_n++; + if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { + const struct rte_flow_action_raw_encap *encap = + actions->conf; + if (decap->size <= + MLX5_ENCAPSULATION_DECISION_SIZE && + encap->size > + MLX5_ENCAPSULATION_DECISION_SIZE) + /* L3 encap. */ + break; + } + if (fdb_mirror) + *modify_after_mirror = 1; + break; + default: + break; + } + actions_n++; + } + if (flag && fdb_mirror && !*modify_after_mirror) { + /* FDB mirroring uses the destination array to implement + * instead of FLOW_SAMPLER object. + */ + if (sub_type != RTE_FLOW_ACTION_TYPE_END) + flag = 0; + } + /* Count RTE_FLOW_ACTION_TYPE_END. */ + return flag ? actions_n + 1 : 0; +} + +#define SAMPLE_SUFFIX_ITEM 2 + +/** + * Split the sample flow. + * + * As sample flow will split to two sub flow, sample flow with + * sample action, the other actions will move to new suffix flow. + * + * Also add unique tag id with tag action in the sample flow, + * the same tag id will be as match in the suffix flow. + * + * @param dev + * Pointer to Ethernet device. + * @param[in] add_tag + * Add extra tag action flag. + * @param[out] sfx_items + * Suffix flow match items (list terminated by the END pattern item). + * @param[in] actions + * Associated actions (list terminated by the END action). + * @param[out] actions_sfx + * Suffix flow actions. + * @param[out] actions_pre + * Prefix flow actions. + * @param[in] actions_n + * The total number of actions. + * @param[in] sample_action_pos + * The sample action position. + * @param[in] qrss_action_pos + * The Queue/RSS action position. + * @param[in] jump_table + * Add extra jump action flag. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, or unique flow_id, a negative errno value + * otherwise and rte_errno is set. + */ +static int +flow_sample_split_prep(struct rte_eth_dev *dev, + int add_tag, + struct rte_flow_item sfx_items[], + const struct rte_flow_action actions[], + struct rte_flow_action actions_sfx[], + struct rte_flow_action actions_pre[], + int actions_n, + int sample_action_pos, + int qrss_action_pos, + int jump_table, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_rte_flow_action_set_tag *set_tag; + struct mlx5_rte_flow_item_tag *tag_spec; + struct mlx5_rte_flow_item_tag *tag_mask; + struct rte_flow_action_jump *jump_action; + uint32_t tag_id = 0; + int index; + int append_index = 0; + int ret; + + if (sample_action_pos < 0) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "invalid position of sample " + "action in list"); + /* Prepare the actions for prefix and suffix flow. */ + if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { + index = qrss_action_pos; + /* Put the preceding the Queue/RSS action into prefix flow. */ + if (index != 0) + memcpy(actions_pre, actions, + sizeof(struct rte_flow_action) * index); + /* Put others preceding the sample action into prefix flow. */ + if (sample_action_pos > index + 1) + memcpy(actions_pre + index, actions + index + 1, + sizeof(struct rte_flow_action) * + (sample_action_pos - index - 1)); + index = sample_action_pos - 1; + /* Put Queue/RSS action into Suffix flow. */ + memcpy(actions_sfx, actions + qrss_action_pos, + sizeof(struct rte_flow_action)); + actions_sfx++; + } else { + index = sample_action_pos; + if (index != 0) + memcpy(actions_pre, actions, + sizeof(struct rte_flow_action) * index); + } + /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. + * For CX6DX and above, metadata registers Cx preserve their value, + * add an extra tag action for NIC-RX and E-Switch Domain. + */ + if (add_tag) { + /* Prepare the prefix tag action. */ + append_index++; + set_tag = (void *)(actions_pre + actions_n + append_index); + ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); + if (ret < 0) + return ret; + mlx5_ipool_malloc(priv->sh->ipool + [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); + *set_tag = (struct mlx5_rte_flow_action_set_tag) { + .id = ret, + .data = tag_id, + }; + /* Prepare the suffix subflow items. */ + tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); + tag_spec->data = tag_id; + tag_spec->id = set_tag->id; + tag_mask = tag_spec + 1; + tag_mask->data = UINT32_MAX; + sfx_items[0] = (struct rte_flow_item){ + .type = (enum rte_flow_item_type) + MLX5_RTE_FLOW_ITEM_TYPE_TAG, + .spec = tag_spec, + .last = NULL, + .mask = tag_mask, + }; + sfx_items[1] = (struct rte_flow_item){ + .type = (enum rte_flow_item_type) + RTE_FLOW_ITEM_TYPE_END, + }; + /* Prepare the tag action in prefix subflow. */ + actions_pre[index++] = + (struct rte_flow_action){ + .type = (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_TAG, + .conf = set_tag, + }; + } + memcpy(actions_pre + index, actions + sample_action_pos, + sizeof(struct rte_flow_action)); + index += 1; + /* For the modify action after the sample action in E-Switch mirroring, + * Add the extra jump action in prefix subflow and jump into the next + * table, then do the modify action in the new table. + */ + if (jump_table) { + /* Prepare the prefix jump action. */ + append_index++; + jump_action = (void *)(actions_pre + actions_n + append_index); + jump_action->group = jump_table; + actions_pre[index++] = + (struct rte_flow_action){ + .type = (enum rte_flow_action_type) + RTE_FLOW_ACTION_TYPE_JUMP, + .conf = jump_action, + }; + } + actions_pre[index] = (struct rte_flow_action){ + .type = (enum rte_flow_action_type) + RTE_FLOW_ACTION_TYPE_END, + }; + /* Put the actions after sample into Suffix flow. */ + memcpy(actions_sfx, actions + sample_action_pos + 1, + sizeof(struct rte_flow_action) * + (actions_n - sample_action_pos - 1)); + return tag_id; +} + +/** + * The splitting for metadata feature. + * + * - Q/RSS action on NIC Rx should be split in order to pass by + * the mreg copy table (RX_CP_TBL) and then it jumps to the + * action table (RX_ACT_TBL) which has the split Q/RSS action. + * + * - All the actions on NIC Tx should have a mreg copy action to + * copy reg_a from WQE to reg_c[0]. + * + * @param dev + * Pointer to Ethernet device. + * @param[in] flow + * Parent flow structure pointer. + * @param[in] attr + * Flow rule attributes. + * @param[in] items + * Pattern specification (list terminated by the END pattern item). + * @param[in] actions + * Associated actions (list terminated by the END action). + * @param[in] flow_split_info + * Pointer to flow split info structure. + * @param[out] error + * Perform verbose error reporting if not NULL. + * @return + * 0 on success, negative value otherwise + */ +static int +flow_create_split_metadata(struct rte_eth_dev *dev, + struct rte_flow *flow, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct mlx5_flow_split_info *flow_split_info, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_config *config = &priv->config; + const struct rte_flow_action *qrss = NULL; + struct rte_flow_action *ext_actions = NULL; + struct mlx5_flow *dev_flow = NULL; + uint32_t qrss_id = 0; + int mtr_sfx = 0; + size_t act_size; + int actions_n; + int encap_idx; + int ret; + + /* Check whether extensive metadata feature is engaged. */ + if (!config->dv_flow_en || + config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || + !mlx5_flow_ext_mreg_supported(dev)) + return flow_create_split_inner(dev, flow, NULL, attr, items, + actions, flow_split_info, error); + actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, + &encap_idx); + if (qrss) { + /* Exclude hairpin flows from splitting. */ + if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { + const struct rte_flow_action_queue *queue; + + queue = qrss->conf; + if (mlx5_rxq_get_type(dev, queue->index) == + MLX5_RXQ_TYPE_HAIRPIN) + qrss = NULL; + } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { const struct rte_flow_action_rss *rss; rss = qrss->conf; @@ -3998,8 +5781,8 @@ flow_create_split_metadata(struct rte_eth_dev *dev, if (qrss) { /* Check if it is in meter suffix table. */ mtr_sfx = attr->group == (attr->transfer ? - (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : - MLX5_FLOW_TABLE_LEVEL_SUFFIX); + (MLX5_FLOW_TABLE_LEVEL_METER - 1) : + MLX5_FLOW_TABLE_LEVEL_METER); /* * Q/RSS action on NIC Rx should be split in order to pass by * the mreg copy table (RX_CP_TBL) and then it jumps to the @@ -4008,23 +5791,13 @@ flow_create_split_metadata(struct rte_eth_dev *dev, act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + sizeof(struct rte_flow_action_set_tag) + sizeof(struct rte_flow_action_jump); - ext_actions = rte_zmalloc(__func__, act_size, 0); + ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, + SOCKET_ID_ANY); if (!ext_actions) return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "no memory to split " "metadata flow"); - /* - * If we are the suffix flow of meter, tag already exist. - * Set the tag action to void. - */ - if (mtr_sfx) - ext_actions[qrss - actions].type = - RTE_FLOW_ACTION_TYPE_VOID; - else - ext_actions[qrss - actions].type = - (enum rte_flow_action_type) - MLX5_RTE_FLOW_ACTION_TYPE_TAG; /* * Create the new actions list with removed Q/RSS action * and appended set tag and jump to register copy table @@ -4032,7 +5805,8 @@ flow_create_split_metadata(struct rte_eth_dev *dev, * in advance, because it is needed for set tag action. */ qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, - qrss, actions_n, error); + qrss, actions_n, + mtr_sfx, error); if (!mtr_sfx && !qrss_id) { ret = -rte_errno; goto exit; @@ -4044,7 +5818,8 @@ flow_create_split_metadata(struct rte_eth_dev *dev, */ act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + sizeof(struct mlx5_flow_action_copy_mreg); - ext_actions = rte_zmalloc(__func__, act_size, 0); + ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, + SOCKET_ID_ANY); if (!ext_actions) return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_ACTION, @@ -4057,9 +5832,9 @@ flow_create_split_metadata(struct rte_eth_dev *dev, goto exit; } /* Add the unmodified original or prefix subflow. */ - ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers, attr, + ret = flow_create_split_inner(dev, flow, &dev_flow, attr, items, ext_actions ? ext_actions : - actions, external, flow_idx, error); + actions, flow_split_info, error); if (ret < 0) goto exit; MLX5_ASSERT(dev_flow); @@ -4071,7 +5846,7 @@ flow_create_split_metadata(struct rte_eth_dev *dev, /* Internal PMD action to set register. */ struct mlx5_rte_flow_item_tag q_tag_spec = { .data = qrss_id, - .id = 0, + .id = REG_NON, }; struct rte_flow_item q_items[] = { { @@ -4120,10 +5895,13 @@ flow_create_split_metadata(struct rte_eth_dev *dev, } dev_flow = NULL; /* Add suffix subflow to execute Q/RSS. */ - ret = flow_create_split_inner(dev, flow, &dev_flow, layers, + flow_split_info->prefix_layers = layers; + flow_split_info->prefix_mark = 0; + flow_split_info->table_id = 0; + ret = flow_create_split_inner(dev, flow, &dev_flow, &q_attr, mtr_sfx ? items : q_items, q_actions, - external, flow_idx, error); + flow_split_info, error); if (ret < 0) goto exit; /* qrss ID should be freed if failed. */ @@ -4137,11 +5915,63 @@ exit: * These ones are included into parent flow list and will be destroyed * by flow_drv_destroy. */ - flow_qrss_free_id(dev, qrss_id); - rte_free(ext_actions); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], + qrss_id); + mlx5_free(ext_actions); return ret; } +/** + * Create meter internal drop flow with the original pattern. + * + * @param dev + * Pointer to Ethernet device. + * @param[in] flow + * Parent flow structure pointer. + * @param[in] attr + * Flow rule attributes. + * @param[in] items + * Pattern specification (list terminated by the END pattern item). + * @param[in] flow_split_info + * Pointer to flow split info structure. + * @param[in] fm + * Pointer to flow meter structure. + * @param[out] error + * Perform verbose error reporting if not NULL. + * @return + * 0 on success, negative value otherwise + */ +static uint32_t +flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, + struct rte_flow *flow, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + struct mlx5_flow_split_info *flow_split_info, + struct mlx5_flow_meter_info *fm, + struct rte_flow_error *error) +{ + struct mlx5_flow *dev_flow = NULL; + struct rte_flow_attr drop_attr = *attr; + struct rte_flow_action drop_actions[3]; + struct mlx5_flow_split_info drop_split_info = *flow_split_info; + + MLX5_ASSERT(fm->drop_cnt); + drop_actions[0].type = + (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; + drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; + drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; + drop_actions[1].conf = NULL; + drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; + drop_actions[2].conf = NULL; + drop_split_info.external = false; + drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; + drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; + drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; + return flow_create_split_inner(dev, flow, &dev_flow, + &drop_attr, items, drop_actions, + &drop_split_info, error); +} + /** * The splitting for meter feature. * @@ -4162,10 +5992,8 @@ exit: * Pattern specification (list terminated by the END pattern item). * @param[in] actions * Associated actions (list terminated by the END action). - * @param[in] external - * This flow rule is created by request external to PMD. - * @param[in] flow_idx - * This memory pool index to the flow. + * @param[in] flow_split_info + * Pointer to flow split info structure. * @param[out] error * Perform verbose error reporting if not NULL. * @return @@ -4173,37 +6001,95 @@ exit: */ static int flow_create_split_meter(struct rte_eth_dev *dev, - struct rte_flow *flow, - const struct rte_flow_attr *attr, - const struct rte_flow_item items[], - const struct rte_flow_action actions[], - bool external, uint32_t flow_idx, - struct rte_flow_error *error) + struct rte_flow *flow, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct mlx5_flow_split_info *flow_split_info, + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); struct rte_flow_action *sfx_actions = NULL; struct rte_flow_action *pre_actions = NULL; struct rte_flow_item *sfx_items = NULL; struct mlx5_flow *dev_flow = NULL; struct rte_flow_attr sfx_attr = *attr; - uint32_t mtr = 0; - uint32_t mtr_tag_id = 0; + struct mlx5_flow_meter_info *fm = NULL; + uint8_t skip_scale_restore; + bool has_mtr = false; + bool has_modify = false; + bool set_mtr_reg = true; + bool is_mtr_hierarchy = false; + uint32_t meter_id = 0; + uint32_t mtr_idx = 0; + uint32_t mtr_flow_id = 0; size_t act_size; size_t item_size; int actions_n = 0; - int ret; + int ret = 0; if (priv->mtr_en) - actions_n = flow_check_meter_action(actions, &mtr); - if (mtr) { - /* The five prefix actions: meter, decap, encap, tag, end. */ - act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + + actions_n = flow_check_meter_action(dev, actions, &has_mtr, + &has_modify, &meter_id); + if (has_mtr) { + if (flow->meter) { + fm = flow_dv_meter_find_by_idx(priv, flow->meter); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + } else { + fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + ret = mlx5_flow_meter_attach(priv, fm, + &sfx_attr, error); + if (ret) + return -rte_errno; + flow->meter = mtr_idx; + } + MLX5_ASSERT(wks); + wks->fm = fm; + if (!fm->def_policy) { + wks->policy = mlx5_flow_meter_policy_find(dev, + fm->policy_id, + NULL); + MLX5_ASSERT(wks->policy); + if (wks->policy->is_hierarchy) { + wks->final_policy = + mlx5_flow_meter_hierarchy_get_final_policy(dev, + wks->policy); + if (!wks->final_policy) + return rte_flow_error_set(error, + EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Failed to find terminal policy of hierarchy."); + is_mtr_hierarchy = true; + } + } + /* + * If it isn't default-policy Meter, and + * 1. There's no action in flow to change + * packet (modify/encap/decap etc.), OR + * 2. No drop count needed for this meter. + * 3. It's not meter hierarchy. + * Then no need to use regC to save meter id anymore. + */ + if (!fm->def_policy && !is_mtr_hierarchy && + (!has_modify || !fm->drop_cnt)) + set_mtr_reg = false; + /* Prefix actions: meter, decap, encap, tag, jump, end. */ + act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + sizeof(struct mlx5_rte_flow_action_set_tag); - /* tag, vlan, port id, end. */ + /* Suffix items: tag, vlan, port id, end. */ #define METER_SUFFIX_ITEM 4 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + sizeof(struct mlx5_rte_flow_item_tag) * 2; - sfx_actions = rte_zmalloc(__func__, (act_size + item_size), 0); + sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), + 0, SOCKET_ID_ANY); if (!sfx_actions) return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_ACTION, @@ -4211,58 +6097,82 @@ flow_create_split_meter(struct rte_eth_dev *dev, "meter flow"); sfx_items = (struct rte_flow_item *)((char *)sfx_actions + act_size); - pre_actions = sfx_actions + actions_n; - mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, - actions, sfx_actions, - pre_actions); - if (!mtr_tag_id) { + /* There's no suffix flow for meter of non-default policy. */ + if (!fm->def_policy) + pre_actions = sfx_actions + 1; + else + pre_actions = sfx_actions + actions_n; + ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, + items, sfx_items, actions, + sfx_actions, pre_actions, + (set_mtr_reg ? &mtr_flow_id : NULL), + error); + if (ret) { ret = -rte_errno; goto exit; } /* Add the prefix subflow. */ - ret = flow_create_split_inner(dev, flow, &dev_flow, 0, attr, - items, pre_actions, external, - flow_idx, error); + skip_scale_restore = flow_split_info->skip_scale; + flow_split_info->skip_scale |= + 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; + ret = flow_create_split_inner(dev, flow, &dev_flow, + attr, items, pre_actions, + flow_split_info, error); + flow_split_info->skip_scale = skip_scale_restore; if (ret) { + if (mtr_flow_id) + mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); ret = -rte_errno; goto exit; } - dev_flow->handle->split_flow_id = mtr_tag_id; + if (mtr_flow_id) { + dev_flow->handle->split_flow_id = mtr_flow_id; + dev_flow->handle->is_meter_flow_id = 1; + } + if (!fm->def_policy) { + if (!set_mtr_reg && fm->drop_cnt) + ret = + flow_meter_create_drop_flow_with_org_pattern(dev, flow, + &sfx_attr, items, + flow_split_info, + fm, error); + goto exit; + } /* Setting the sfx group atrr. */ sfx_attr.group = sfx_attr.transfer ? - (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : - MLX5_FLOW_TABLE_LEVEL_SUFFIX; + (MLX5_FLOW_TABLE_LEVEL_METER - 1) : + MLX5_FLOW_TABLE_LEVEL_METER; + flow_split_info->prefix_layers = + flow_get_prefix_layer_flags(dev_flow); + flow_split_info->prefix_mark |= wks->mark; + flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; } /* Add the prefix subflow. */ - ret = flow_create_split_metadata(dev, flow, dev_flow ? - flow_get_prefix_layer_flags(dev_flow) : - 0, &sfx_attr, - sfx_items ? sfx_items : items, + ret = flow_create_split_metadata(dev, flow, + &sfx_attr, sfx_items ? + sfx_items : items, sfx_actions ? sfx_actions : actions, - external, flow_idx, error); + flow_split_info, error); exit: if (sfx_actions) - rte_free(sfx_actions); + mlx5_free(sfx_actions); return ret; } /** - * Split the flow to subflow set. The splitters might be linked - * in the chain, like this: - * flow_create_split_outer() calls: - * flow_create_split_meter() calls: - * flow_create_split_metadata(meter_subflow_0) calls: - * flow_create_split_inner(metadata_subflow_0) - * flow_create_split_inner(metadata_subflow_1) - * flow_create_split_inner(metadata_subflow_2) - * flow_create_split_metadata(meter_subflow_1) calls: - * flow_create_split_inner(metadata_subflow_0) - * flow_create_split_inner(metadata_subflow_1) - * flow_create_split_inner(metadata_subflow_2) + * The splitting for sample feature. * - * This provide flexible way to add new levels of flow splitting. - * The all of successfully created subflows are included to the - * parent flow dev_flow list. + * Once Sample action is detected in the action list, the flow actions should + * be split into prefix sub flow and suffix sub flow. + * + * The original items remain in the prefix sub flow, all actions preceding the + * sample action and the sample action itself will be copied to the prefix + * sub flow, the actions following the sample action will be copied to the + * suffix sub flow, Queue action always be located in the suffix sub flow. + * + * In order to make the packet from prefix sub flow matches with suffix sub + * flow, an extra tag action be added into prefix sub flow, and the suffix sub + * flow uses tag item with the unique flow id. * * @param dev * Pointer to Ethernet device. @@ -4274,31 +6184,233 @@ exit: * Pattern specification (list terminated by the END pattern item). * @param[in] actions * Associated actions (list terminated by the END action). - * @param[in] external - * This flow rule is created by request external to PMD. - * @param[in] flow_idx - * This memory pool index to the flow. + * @param[in] flow_split_info + * Pointer to flow split info structure. * @param[out] error * Perform verbose error reporting if not NULL. * @return * 0 on success, negative value otherwise */ static int -flow_create_split_outer(struct rte_eth_dev *dev, - struct rte_flow *flow, - const struct rte_flow_attr *attr, - const struct rte_flow_item items[], - const struct rte_flow_action actions[], - bool external, uint32_t flow_idx, - struct rte_flow_error *error) +flow_create_split_sample(struct rte_eth_dev *dev, + struct rte_flow *flow, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct mlx5_flow_split_info *flow_split_info, + struct rte_flow_error *error) { - int ret; + struct mlx5_priv *priv = dev->data->dev_private; + struct rte_flow_action *sfx_actions = NULL; + struct rte_flow_action *pre_actions = NULL; + struct rte_flow_item *sfx_items = NULL; + struct mlx5_flow *dev_flow = NULL; + struct rte_flow_attr sfx_attr = *attr; +#ifdef HAVE_IBV_FLOW_DV_SUPPORT + struct mlx5_flow_dv_sample_resource *sample_res; + struct mlx5_flow_tbl_data_entry *sfx_tbl_data; + struct mlx5_flow_tbl_resource *sfx_tbl; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); +#endif + size_t act_size; + size_t item_size; + uint32_t fdb_tx = 0; + int32_t tag_id = 0; + int actions_n = 0; + int sample_action_pos; + int qrss_action_pos; + int add_tag = 0; + int modify_after_mirror = 0; + uint16_t jump_table = 0; + const uint32_t next_ft_step = 1; + int ret = 0; - ret = flow_create_split_meter(dev, flow, attr, items, - actions, external, flow_idx, error); - MLX5_ASSERT(ret <= 0); - return ret; -} + if (priv->sampler_en) + actions_n = flow_check_match_action(actions, attr, + RTE_FLOW_ACTION_TYPE_SAMPLE, + &sample_action_pos, &qrss_action_pos, + &modify_after_mirror); + if (actions_n) { + /* The prefix actions must includes sample, tag, end. */ + act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) + + sizeof(struct mlx5_rte_flow_action_set_tag); + item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + + sizeof(struct mlx5_rte_flow_item_tag) * 2; + sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + + item_size), 0, SOCKET_ID_ANY); + if (!sfx_actions) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "no memory to split " + "sample flow"); + /* The representor_id is UINT16_MAX for uplink. */ + fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); + /* + * When reg_c_preserve is set, metadata registers Cx preserve + * their value even through packet duplication. + */ + add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve); + if (add_tag) + sfx_items = (struct rte_flow_item *)((char *)sfx_actions + + act_size); + if (modify_after_mirror) + jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + + next_ft_step; + pre_actions = sfx_actions + actions_n; + tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, + actions, sfx_actions, + pre_actions, actions_n, + sample_action_pos, + qrss_action_pos, jump_table, + error); + if (tag_id < 0 || (add_tag && !tag_id)) { + ret = -rte_errno; + goto exit; + } + if (modify_after_mirror) + flow_split_info->skip_scale = + 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; + /* Add the prefix subflow. */ + ret = flow_create_split_inner(dev, flow, &dev_flow, attr, + items, pre_actions, + flow_split_info, error); + if (ret) { + ret = -rte_errno; + goto exit; + } + dev_flow->handle->split_flow_id = tag_id; +#ifdef HAVE_IBV_FLOW_DV_SUPPORT + if (!modify_after_mirror) { + /* Set the sfx group attr. */ + sample_res = (struct mlx5_flow_dv_sample_resource *) + dev_flow->dv.sample_res; + sfx_tbl = (struct mlx5_flow_tbl_resource *) + sample_res->normal_path_tbl; + sfx_tbl_data = container_of(sfx_tbl, + struct mlx5_flow_tbl_data_entry, + tbl); + sfx_attr.group = sfx_attr.transfer ? + (sfx_tbl_data->level - 1) : sfx_tbl_data->level; + } else { + MLX5_ASSERT(attr->transfer); + sfx_attr.group = jump_table; + } + flow_split_info->prefix_layers = + flow_get_prefix_layer_flags(dev_flow); + MLX5_ASSERT(wks); + flow_split_info->prefix_mark |= wks->mark; + /* Suffix group level already be scaled with factor, set + * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale + * again in translation. + */ + flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; +#endif + } + /* Add the suffix subflow. */ + ret = flow_create_split_meter(dev, flow, &sfx_attr, + sfx_items ? sfx_items : items, + sfx_actions ? sfx_actions : actions, + flow_split_info, error); +exit: + if (sfx_actions) + mlx5_free(sfx_actions); + return ret; +} + +/** + * Split the flow to subflow set. The splitters might be linked + * in the chain, like this: + * flow_create_split_outer() calls: + * flow_create_split_meter() calls: + * flow_create_split_metadata(meter_subflow_0) calls: + * flow_create_split_inner(metadata_subflow_0) + * flow_create_split_inner(metadata_subflow_1) + * flow_create_split_inner(metadata_subflow_2) + * flow_create_split_metadata(meter_subflow_1) calls: + * flow_create_split_inner(metadata_subflow_0) + * flow_create_split_inner(metadata_subflow_1) + * flow_create_split_inner(metadata_subflow_2) + * + * This provide flexible way to add new levels of flow splitting. + * The all of successfully created subflows are included to the + * parent flow dev_flow list. + * + * @param dev + * Pointer to Ethernet device. + * @param[in] flow + * Parent flow structure pointer. + * @param[in] attr + * Flow rule attributes. + * @param[in] items + * Pattern specification (list terminated by the END pattern item). + * @param[in] actions + * Associated actions (list terminated by the END action). + * @param[in] flow_split_info + * Pointer to flow split info structure. + * @param[out] error + * Perform verbose error reporting if not NULL. + * @return + * 0 on success, negative value otherwise + */ +static int +flow_create_split_outer(struct rte_eth_dev *dev, + struct rte_flow *flow, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct mlx5_flow_split_info *flow_split_info, + struct rte_flow_error *error) +{ + int ret; + + ret = flow_create_split_sample(dev, flow, attr, items, + actions, flow_split_info, error); + MLX5_ASSERT(ret <= 0); + return ret; +} + +static inline struct mlx5_flow_tunnel * +flow_tunnel_from_rule(const struct mlx5_flow *flow) +{ + struct mlx5_flow_tunnel *tunnel; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" + tunnel = (typeof(tunnel))flow->tunnel; +#pragma GCC diagnostic pop + + return tunnel; +} + +/** + * Adjust flow RSS workspace if needed. + * + * @param wks + * Pointer to thread flow work space. + * @param rss_desc + * Pointer to RSS descriptor. + * @param[in] nrssq_num + * New RSS queue number. + * + * @return + * 0 on success, -1 otherwise and rte_errno is set. + */ +static int +flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, + struct mlx5_flow_rss_desc *rss_desc, + uint32_t nrssq_num) +{ + if (likely(nrssq_num <= wks->rssq_num)) + return 0; + rss_desc->queue = realloc(rss_desc->queue, + sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); + if (!rss_desc->queue) { + rte_errno = ENOMEM; + return -1; + } + wks->rssq_num = RTE_ALIGN(nrssq_num, 2); + return 0; +} /** * Create a flow and add it to @p list. @@ -4325,19 +6437,22 @@ flow_create_split_outer(struct rte_eth_dev *dev, * A flow index on success, 0 otherwise and rte_errno is set. */ static uint32_t -flow_list_create(struct rte_eth_dev *dev, uint32_t *list, +flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, const struct rte_flow_attr *attr, const struct rte_flow_item items[], - const struct rte_flow_action actions[], + const struct rte_flow_action original_actions[], bool external, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow *flow = NULL; struct mlx5_flow *dev_flow; - const struct rte_flow_action_rss *rss; + const struct rte_flow_action_rss *rss = NULL; + struct mlx5_translated_action_handle + indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; + int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; union { - struct rte_flow_expand_rss buf; - uint8_t buffer[2048]; + struct mlx5_flow_expand_rss buf; + uint8_t buffer[4096]; } expand_buffer; union { struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; @@ -4351,78 +6466,104 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list, struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; uint8_t buffer[2048]; } items_tx; - struct rte_flow_expand_rss *buf = &expand_buffer.buf; - struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *) - priv->rss_desc)[!!priv->flow_idx]; - const struct rte_flow_action *p_actions_rx = actions; + struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; + struct mlx5_flow_rss_desc *rss_desc; + const struct rte_flow_action *p_actions_rx; uint32_t i; uint32_t idx = 0; int hairpin_flow; - uint32_t hairpin_id = 0; struct rte_flow_attr attr_tx = { .priority = 0 }; + const struct rte_flow_action *actions; + struct rte_flow_action *translated_actions = NULL; + struct mlx5_flow_tunnel *tunnel; + struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; + struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); + struct mlx5_flow_split_info flow_split_info = { + .external = !!external, + .skip_scale = 0, + .flow_idx = 0, + .prefix_mark = 0, + .prefix_layers = 0, + .table_id = 0 + }; int ret; + MLX5_ASSERT(wks); + rss_desc = &wks->rss_desc; + ret = flow_action_handles_translate(dev, original_actions, + indir_actions, + &indir_actions_n, + &translated_actions, error); + if (ret < 0) { + MLX5_ASSERT(translated_actions == NULL); + return 0; + } + actions = translated_actions ? translated_actions : original_actions; + p_actions_rx = actions; hairpin_flow = flow_check_hairpin_split(dev, attr, actions); ret = flow_drv_validate(dev, attr, items, p_actions_rx, external, hairpin_flow, error); if (ret < 0) - return 0; + goto error_before_hairpin_split; + flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); + if (!flow) { + rte_errno = ENOMEM; + goto error_before_hairpin_split; + } if (hairpin_flow > 0) { if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { rte_errno = EINVAL; - return 0; + goto error_before_hairpin_split; } flow_hairpin_split(dev, actions, actions_rx.actions, actions_hairpin_tx.actions, items_tx.items, - &hairpin_id); + idx); p_actions_rx = actions_rx.actions; } - flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); - if (!flow) { - rte_errno = ENOMEM; - goto error_before_flow; - } + flow_split_info.flow_idx = idx; flow->drv_type = flow_get_drv_type(dev, attr); - if (hairpin_id != 0) - flow->hairpin_flow_id = hairpin_id; MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && flow->drv_type < MLX5_FLOW_TYPE_MAX); - memset(rss_desc, 0, sizeof(*rss_desc)); - rss = flow_get_rss_action(p_actions_rx); + memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); + /* RSS Action only works on NIC RX domain */ + if (attr->ingress && !attr->transfer) + rss = flow_get_rss_action(dev, p_actions_rx); if (rss) { + if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) + return 0; /* * The following information is required by * mlx5_flow_hashfields_adjust() in advance. */ rss_desc->level = rss->level; - /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ - rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; + /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ + rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; } flow->dev_handles = 0; if (rss && rss->types) { unsigned int graph_root; - graph_root = find_graph_root(items, rss->level); - ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer), - items, rss->types, - mlx5_support_expansion, - graph_root); + graph_root = find_graph_root(rss->level); + ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), + items, rss->types, + mlx5_support_expansion, graph_root); MLX5_ASSERT(ret > 0 && (unsigned int)ret < sizeof(expand_buffer.buffer)); + if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { + for (i = 0; i < buf->entries; ++i) + mlx5_dbg__print_pattern(buf->entry[i].pattern); + } } else { buf->entries = 1; buf->entry[0].pattern = (void *)(uintptr_t)items; } - /* - * Record the start index when there is a nested call. All sub-flows - * need to be translated before another calling. - * No need to use ping-pong buffer to save memory here. - */ - if (priv->flow_idx) { - MLX5_ASSERT(!priv->flow_nested_idx); - priv->flow_nested_idx = priv->flow_idx; - } + rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, + indir_actions_n); for (i = 0; i < buf->entries; ++i) { + /* Initialize flow split data. */ + flow_split_info.prefix_layers = 0; + flow_split_info.prefix_mark = 0; + flow_split_info.skip_scale = 0; /* * The splitter may create multiple dev_flows, * depending on configuration. In the simplest @@ -4430,10 +6571,22 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list, */ ret = flow_create_split_outer(dev, flow, attr, buf->entry[i].pattern, - p_actions_rx, external, idx, + p_actions_rx, &flow_split_info, error); if (ret < 0) goto error; + if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { + ret = flow_tunnel_add_default_miss(dev, flow, attr, + p_actions_rx, + idx, + wks->flows[0].tunnel, + &default_miss_ctx, + error); + if (ret < 0) { + mlx5_free(default_miss_ctx.queue); + goto error; + } + } } /* Create the tx flow. */ if (hairpin_flow) { @@ -4472,39 +6625,45 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list, goto error; } /* - * If the flow is external (from application) OR device is started, then - * the flow will be applied immediately. + * If the flow is external (from application) OR device is started, + * OR mreg discover, then apply immediately. */ - if (external || dev->data->dev_started) { + if (external || dev->data->dev_started || + (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && + attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { ret = flow_drv_apply(dev, flow, error); if (ret < 0) goto error; } - if (list) - ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, - flow, next); + flow->type = type; flow_rxq_flags_set(dev, flow); - /* Nested flow creation index recovery. */ - priv->flow_idx = priv->flow_nested_idx; - if (priv->flow_nested_idx) - priv->flow_nested_idx = 0; + rte_free(translated_actions); + tunnel = flow_tunnel_from_rule(wks->flows); + if (tunnel) { + flow->tunnel = 1; + flow->tunnel_id = tunnel->tunnel_id; + __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); + mlx5_free(default_miss_ctx.queue); + } + mlx5_flow_pop_thread_workspace(); return idx; error: MLX5_ASSERT(flow); ret = rte_errno; /* Save rte_errno before cleanup. */ flow_mreg_del_copy_action(dev, flow); flow_drv_destroy(dev, flow); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); + if (rss_desc->shared_rss) + __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) + mlx5_ipool_get + (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], + rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); + mlx5_ipool_free(priv->flows[type], idx); rte_errno = ret; /* Restore rte_errno. */ -error_before_flow: ret = rte_errno; - if (hairpin_id) - mlx5_flow_id_release(priv->sh->flow_id_pool, - hairpin_id); rte_errno = ret; - priv->flow_idx = priv->flow_nested_idx; - if (priv->flow_nested_idx) - priv->flow_nested_idx = 0; + mlx5_flow_pop_thread_workspace(); +error_before_hairpin_split: + rte_free(translated_actions); return 0; } @@ -4550,14 +6709,87 @@ mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) .type = RTE_FLOW_ACTION_TYPE_END, }, }; - struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow_error error; - return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, + return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, &pattern, actions, false, &error); } +/** + * Create a dedicated flow rule on e-switch table 1, matches ESW manager + * and sq number, directs all packets to peer vport. + * + * @param dev + * Pointer to Ethernet device. + * @param txq + * Txq index. + * + * @return + * Flow ID on success, 0 otherwise and rte_errno is set. + */ +uint32_t +mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq) +{ + struct rte_flow_attr attr = { + .group = 0, + .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, + .ingress = 1, + .egress = 0, + .transfer = 1, + }; + struct rte_flow_item_port_id port_spec = { + .id = MLX5_PORT_ESW_MGR, + }; + struct mlx5_rte_flow_item_tx_queue txq_spec = { + .queue = txq, + }; + struct rte_flow_item pattern[] = { + { + .type = RTE_FLOW_ITEM_TYPE_PORT_ID, + .spec = &port_spec, + }, + { + .type = (enum rte_flow_item_type) + MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, + .spec = &txq_spec, + }, + { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + }; + struct rte_flow_action_jump jump = { + .group = 1, + }; + struct rte_flow_action_port_id port = { + .id = dev->data->port_id, + }; + struct rte_flow_action actions[] = { + { + .type = RTE_FLOW_ACTION_TYPE_JUMP, + .conf = &jump, + }, + { + .type = RTE_FLOW_ACTION_TYPE_END, + }, + }; + struct rte_flow_error error; + + /* + * Creates group 0, highest priority jump flow. + * Matches txq to bypass kernel packets. + */ + if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, + false, &error) == 0) + return 0; + /* Create group 1, lowest priority redirect flow for txq. */ + attr.group = 1; + actions[0].conf = &port; + actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; + return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, + actions, false, &error); +} + /** * Validate a flow supported by the NIC. * @@ -4568,14 +6800,28 @@ int mlx5_flow_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, const struct rte_flow_item items[], - const struct rte_flow_action actions[], + const struct rte_flow_action original_actions[], struct rte_flow_error *error) { int hairpin_flow; + struct mlx5_translated_action_handle + indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; + int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; + const struct rte_flow_action *actions; + struct rte_flow_action *translated_actions = NULL; + int ret = flow_action_handles_translate(dev, original_actions, + indir_actions, + &indir_actions_n, + &translated_actions, error); + if (ret) + return ret; + actions = translated_actions ? translated_actions : original_actions; hairpin_flow = flow_check_hairpin_split(dev, attr, actions); - return flow_drv_validate(dev, attr, items, actions, + ret = flow_drv_validate(dev, attr, items, actions, true, hairpin_flow, error); + rte_free(translated_actions); + return ret; } /** @@ -4591,8 +6837,6 @@ mlx5_flow_create(struct rte_eth_dev *dev, const struct rte_flow_action actions[], struct rte_flow_error *error) { - struct mlx5_priv *priv = dev->data->dev_private; - /* * If the device is not started yet, it is not allowed to created a * flow from application. PMD default flows and traffic control flows @@ -4607,8 +6851,10 @@ mlx5_flow_create(struct rte_eth_dev *dev, "port not started"); return NULL; } - return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, - attr, items, actions, true, error); + + return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, + attr, items, actions, + true, error); } /** @@ -4616,51 +6862,36 @@ mlx5_flow_create(struct rte_eth_dev *dev, * * @param dev * Pointer to Ethernet device. - * @param list - * Pointer to the Indexed flow list. If this parameter NULL, - * there is no flow removal from the list. Be noted that as - * flow is add to the indexed list, memory of the indexed - * list points to maybe changed as flow destroyed. * @param[in] flow_idx * Index of flow to destroy. */ static void -flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, +flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, uint32_t flow_idx) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_fdir_flow *priv_fdir_flow = NULL; - struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool - [MLX5_IPOOL_RTE_FLOW], flow_idx); + struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); if (!flow) return; + MLX5_ASSERT(flow->type == type); /* * Update RX queue flags only if port is started, otherwise it is * already clean. */ if (dev->data->dev_started) flow_rxq_flags_trim(dev, flow); - if (flow->hairpin_flow_id) - mlx5_flow_id_release(priv->sh->flow_id_pool, - flow->hairpin_flow_id); flow_drv_destroy(dev, flow); - if (list) - ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, - flow_idx, flow, next); - flow_mreg_del_copy_action(dev, flow); - if (flow->fdir) { - LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { - if (priv_fdir_flow->rix_flow == flow_idx) - break; - } - if (priv_fdir_flow) { - LIST_REMOVE(priv_fdir_flow, next); - rte_free(priv_fdir_flow->fdir); - rte_free(priv_fdir_flow); - } + if (flow->tunnel) { + struct mlx5_flow_tunnel *tunnel; + + tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); + RTE_VERIFY(tunnel); + if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) + mlx5_flow_tunnel_free(dev, tunnel); } - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); + flow_mreg_del_copy_action(dev, flow); + mlx5_ipool_free(priv->flows[type], flow_idx); } /** @@ -4668,18 +6899,21 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, * * @param dev * Pointer to Ethernet device. - * @param list - * Pointer to the Indexed flow list. + * @param type + * Flow type to be flushed. * @param active - * If flushing is called avtively. + * If flushing is called actively. */ void -mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) +mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, + bool active) { - uint32_t num_flushed = 0; + struct mlx5_priv *priv = dev->data->dev_private; + uint32_t num_flushed = 0, fidx = 1; + struct rte_flow *flow; - while (*list) { - flow_list_destroy(dev, list, *list); + MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { + flow_list_destroy(dev, type, fidx); num_flushed++; } if (active) { @@ -4689,144 +6923,157 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) } /** - * Remove all flows. + * Stop all default actions for flows. * * @param dev * Pointer to Ethernet device. - * @param list - * Pointer to the Indexed flow list. */ void -mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list) +mlx5_flow_stop_default(struct rte_eth_dev *dev) { - struct mlx5_priv *priv = dev->data->dev_private; - struct rte_flow *flow = NULL; - uint32_t idx; - - ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, - flow, next) { - flow_drv_remove(dev, flow); - flow_mreg_stop_copy_action(dev, flow); - } flow_mreg_del_default_copy_action(dev); flow_rxq_flags_clear(dev); } /** - * Add all flows. + * Start all default actions for flows. * * @param dev * Pointer to Ethernet device. - * @param list - * Pointer to the Indexed flow list. - * * @return * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list) +mlx5_flow_start_default(struct rte_eth_dev *dev) { - struct mlx5_priv *priv = dev->data->dev_private; - struct rte_flow *flow = NULL; struct rte_flow_error error; - uint32_t idx; - int ret = 0; /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ - ret = flow_mreg_add_default_copy_action(dev, &error); - if (ret < 0) - return -rte_errno; - /* Apply Flows created by application. */ - ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, - flow, next) { - ret = flow_mreg_start_copy_action(dev, flow); - if (ret < 0) - goto error; - ret = flow_drv_apply(dev, flow, &error); - if (ret < 0) - goto error; - flow_rxq_flags_set(dev, flow); - } - return 0; -error: - ret = rte_errno; /* Save rte_errno before cleanup. */ - mlx5_flow_stop(dev, list); - rte_errno = ret; /* Restore rte_errno. */ - return -rte_errno; + return flow_mreg_add_default_copy_action(dev, &error); } /** - * Stop all default actions for flows. - * - * @param dev - * Pointer to Ethernet device. + * Release key of thread specific flow workspace data. */ void -mlx5_flow_stop_default(struct rte_eth_dev *dev) +flow_release_workspace(void *data) { - flow_mreg_del_default_copy_action(dev); - flow_rxq_flags_clear(dev); + struct mlx5_flow_workspace *wks = data; + struct mlx5_flow_workspace *next; + + while (wks) { + next = wks->next; + free(wks->rss_desc.queue); + free(wks); + wks = next; + } } /** - * Start all default actions for flows. + * Get thread specific current flow workspace. * - * @param dev - * Pointer to Ethernet device. - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. + * @return pointer to thread specific flow workspace data, NULL on error. */ -int -mlx5_flow_start_default(struct rte_eth_dev *dev) +struct mlx5_flow_workspace* +mlx5_flow_get_thread_workspace(void) { - struct rte_flow_error error; + struct mlx5_flow_workspace *data; - /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ - return flow_mreg_add_default_copy_action(dev, &error); + data = mlx5_flow_os_get_specific_workspace(); + MLX5_ASSERT(data && data->inuse); + if (!data || !data->inuse) + DRV_LOG(ERR, "flow workspace not initialized."); + return data; } /** - * Allocate intermediate resources for flow creation. + * Allocate and init new flow workspace. * - * @param dev - * Pointer to Ethernet device. + * @return pointer to flow workspace data, NULL on error. */ -void -mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev) +static struct mlx5_flow_workspace* +flow_alloc_thread_workspace(void) { - struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); - if (!priv->inter_flows) { - priv->inter_flows = rte_calloc(__func__, 1, - MLX5_NUM_MAX_DEV_FLOWS * - sizeof(struct mlx5_flow) + - (sizeof(struct mlx5_flow_rss_desc) + - sizeof(uint16_t) * UINT16_MAX) * 2, 0); - if (!priv->inter_flows) { - DRV_LOG(ERR, "can't allocate intermediate memory."); - return; - } + if (!data) { + DRV_LOG(ERR, "Failed to allocate flow workspace " + "memory."); + return NULL; } - priv->rss_desc = &((struct mlx5_flow *)priv->inter_flows) - [MLX5_NUM_MAX_DEV_FLOWS]; - /* Reset the index. */ - priv->flow_idx = 0; - priv->flow_nested_idx = 0; + data->rss_desc.queue = calloc(1, + sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); + if (!data->rss_desc.queue) + goto err; + data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; + return data; +err: + if (data->rss_desc.queue) + free(data->rss_desc.queue); + free(data); + return NULL; } /** - * Free intermediate resources for flows. + * Get new thread specific flow workspace. * - * @param dev - * Pointer to Ethernet device. + * If current workspace inuse, create new one and set as current. + * + * @return pointer to thread specific flow workspace data, NULL on error. */ -void -mlx5_flow_free_intermediate(struct rte_eth_dev *dev) +static struct mlx5_flow_workspace* +mlx5_flow_push_thread_workspace(void) { - struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *curr; + struct mlx5_flow_workspace *data; + + curr = mlx5_flow_os_get_specific_workspace(); + if (!curr) { + data = flow_alloc_thread_workspace(); + if (!data) + return NULL; + } else if (!curr->inuse) { + data = curr; + } else if (curr->next) { + data = curr->next; + } else { + data = flow_alloc_thread_workspace(); + if (!data) + return NULL; + curr->next = data; + data->prev = curr; + } + data->inuse = 1; + data->flow_idx = 0; + /* Set as current workspace */ + if (mlx5_flow_os_set_specific_workspace(data)) + DRV_LOG(ERR, "Failed to set flow workspace to thread."); + return data; +} + +/** + * Close current thread specific flow workspace. + * + * If previous workspace available, set it as current. + * + * @return pointer to thread specific flow workspace data, NULL on error. + */ +static void +mlx5_flow_pop_thread_workspace(void) +{ + struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); - rte_free(priv->inter_flows); - priv->inter_flows = NULL; + if (!data) + return; + if (!data->inuse) { + DRV_LOG(ERR, "Failed to close unused flow workspace."); + return; + } + data->inuse = 0; + if (!data->prev) + return; + if (mlx5_flow_os_set_specific_workspace(data->prev)) + DRV_LOG(ERR, "Failed to set flow workspace to thread."); } /** @@ -4838,18 +7085,19 @@ mlx5_flow_free_intermediate(struct rte_eth_dev *dev) * @return the number of flows not released. */ int -mlx5_flow_verify(struct rte_eth_dev *dev) +mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) { struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow *flow; - uint32_t idx; - int ret = 0; + uint32_t idx = 0; + int ret = 0, i; - ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, - flow, next) { - DRV_LOG(DEBUG, "port %u flow %p still referenced", - dev->data->port_id, (void *)flow); - ++ret; + for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { + MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { + DRV_LOG(DEBUG, "port %u flow %p still referenced", + dev->data->port_id, (void *)flow); + ret++; + } } return ret; } @@ -4869,7 +7117,6 @@ int mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, uint32_t queue) { - struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_attr attr = { .egress = 1, .priority = 0, @@ -4902,8 +7149,8 @@ mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; actions[0].conf = &jump; actions[1].type = RTE_FLOW_ACTION_TYPE_END; - flow_idx = flow_list_create(dev, &priv->ctrl_flows, - &attr, items, actions, false, &error); + flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, + &attr, items, actions, false, &error); if (!flow_idx) { DRV_LOG(DEBUG, "Failed to create ctrl flow: rte_errno(%d)," @@ -4942,7 +7189,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_attr attr = { .ingress = 1, - .priority = MLX5_FLOW_PRIO_RSVD, + .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, }; struct rte_flow_item items[] = { { @@ -4988,12 +7235,12 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, if (!priv->reta_idx_n || !priv->rxqs_n) { return 0; } - if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) + if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) action_rss.types = 0; for (i = 0; i != priv->reta_idx_n; ++i) queue[i] = (*priv->reta_idx)[i]; - flow_idx = flow_list_create(dev, &priv->ctrl_flows, - &attr, items, actions, false, &error); + flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, + &attr, items, actions, false, &error); if (!flow_idx) return -rte_errno; return 0; @@ -5034,7 +7281,6 @@ mlx5_ctrl_flow(struct rte_eth_dev *dev, int mlx5_flow_lacp_miss(struct rte_eth_dev *dev) { - struct mlx5_priv *priv = dev->data->dev_private; /* * The LACP matching is done by only using ether type since using * a multicast dst mac causes kernel to give low priority to this flow. @@ -5068,8 +7314,9 @@ mlx5_flow_lacp_miss(struct rte_eth_dev *dev) }, }; struct rte_flow_error error; - uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows, - &attr, items, actions, false, &error); + uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, + &attr, items, actions, + false, &error); if (!flow_idx) return -rte_errno; @@ -5087,9 +7334,8 @@ mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow, struct rte_flow_error *error __rte_unused) { - struct mlx5_priv *priv = dev->data->dev_private; - - flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); + flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, + (uintptr_t)(void *)flow); return 0; } @@ -5103,9 +7349,7 @@ int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error __rte_unused) { - struct mlx5_priv *priv = dev->data->dev_private; - - mlx5_flow_list_flush(dev, &priv->flows, false); + mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); return 0; } @@ -5131,9 +7375,13 @@ mlx5_flow_isolate(struct rte_eth_dev *dev, } priv->isolated = !!enable; if (enable) - dev->dev_ops = &mlx5_os_dev_ops_isolate; + dev->dev_ops = &mlx5_dev_ops_isolate; else - dev->dev_ops = &mlx5_os_dev_ops; + dev->dev_ops = &mlx5_dev_ops; + + dev->rx_descriptor_status = mlx5_rx_descriptor_status; + dev->tx_descriptor_status = mlx5_tx_descriptor_status; + return 0; } @@ -5152,8 +7400,7 @@ flow_drv_query(struct rte_eth_dev *dev, { struct mlx5_priv *priv = dev->data->dev_private; const struct mlx5_flow_driver_ops *fops; - struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool - [MLX5_IPOOL_RTE_FLOW], + struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], flow_idx); enum mlx5_flow_drv_type ftype; @@ -5193,605 +7440,294 @@ mlx5_flow_query(struct rte_eth_dev *dev, } /** - * Convert a flow director filter to a generic flow. + * Get rte_flow callbacks. * * @param dev - * Pointer to Ethernet device. - * @param fdir_filter - * Flow director filter to add. - * @param attributes - * Generic flow parameters structure. + * Pointer to Ethernet device structure. + * @param ops + * Pointer to operation-specific structure. * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. + * @return 0 */ -static int -flow_fdir_filter_convert(struct rte_eth_dev *dev, - const struct rte_eth_fdir_filter *fdir_filter, - struct mlx5_fdir *attributes) +int +mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, + const struct rte_flow_ops **ops) { - struct mlx5_priv *priv = dev->data->dev_private; - const struct rte_eth_fdir_input *input = &fdir_filter->input; - const struct rte_eth_fdir_masks *mask = - &dev->data->dev_conf.fdir_conf.mask; - - /* Validate queue number. */ - if (fdir_filter->action.rx_queue >= priv->rxqs_n) { - DRV_LOG(ERR, "port %u invalid queue number %d", - dev->data->port_id, fdir_filter->action.rx_queue); - rte_errno = EINVAL; - return -rte_errno; - } - attributes->attr.ingress = 1; - attributes->items[0] = (struct rte_flow_item) { - .type = RTE_FLOW_ITEM_TYPE_ETH, - .spec = &attributes->l2, - .mask = &attributes->l2_mask, - }; - switch (fdir_filter->action.behavior) { - case RTE_ETH_FDIR_ACCEPT: - attributes->actions[0] = (struct rte_flow_action){ - .type = RTE_FLOW_ACTION_TYPE_QUEUE, - .conf = &attributes->queue, - }; - break; - case RTE_ETH_FDIR_REJECT: - attributes->actions[0] = (struct rte_flow_action){ - .type = RTE_FLOW_ACTION_TYPE_DROP, - }; - break; - default: - DRV_LOG(ERR, "port %u invalid behavior %d", - dev->data->port_id, - fdir_filter->action.behavior); - rte_errno = ENOTSUP; - return -rte_errno; - } - attributes->queue.index = fdir_filter->action.rx_queue; - /* Handle L3. */ - switch (fdir_filter->input.flow_type) { - case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: - case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: - case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: - attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ - .src_addr = input->flow.ip4_flow.src_ip, - .dst_addr = input->flow.ip4_flow.dst_ip, - .time_to_live = input->flow.ip4_flow.ttl, - .type_of_service = input->flow.ip4_flow.tos, - }; - attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ - .src_addr = mask->ipv4_mask.src_ip, - .dst_addr = mask->ipv4_mask.dst_ip, - .time_to_live = mask->ipv4_mask.ttl, - .type_of_service = mask->ipv4_mask.tos, - .next_proto_id = mask->ipv4_mask.proto, - }; - attributes->items[1] = (struct rte_flow_item){ - .type = RTE_FLOW_ITEM_TYPE_IPV4, - .spec = &attributes->l3, - .mask = &attributes->l3_mask, - }; - break; - case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: - case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: - case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: - attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ - .hop_limits = input->flow.ipv6_flow.hop_limits, - .proto = input->flow.ipv6_flow.proto, - }; - - memcpy(attributes->l3.ipv6.hdr.src_addr, - input->flow.ipv6_flow.src_ip, - RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); - memcpy(attributes->l3.ipv6.hdr.dst_addr, - input->flow.ipv6_flow.dst_ip, - RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); - memcpy(attributes->l3_mask.ipv6.hdr.src_addr, - mask->ipv6_mask.src_ip, - RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); - memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, - mask->ipv6_mask.dst_ip, - RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); - attributes->items[1] = (struct rte_flow_item){ - .type = RTE_FLOW_ITEM_TYPE_IPV6, - .spec = &attributes->l3, - .mask = &attributes->l3_mask, - }; - break; - default: - DRV_LOG(ERR, "port %u invalid flow type%d", - dev->data->port_id, fdir_filter->input.flow_type); - rte_errno = ENOTSUP; - return -rte_errno; - } - /* Handle L4. */ - switch (fdir_filter->input.flow_type) { - case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: - attributes->l4.udp.hdr = (struct rte_udp_hdr){ - .src_port = input->flow.udp4_flow.src_port, - .dst_port = input->flow.udp4_flow.dst_port, - }; - attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ - .src_port = mask->src_port_mask, - .dst_port = mask->dst_port_mask, - }; - attributes->items[2] = (struct rte_flow_item){ - .type = RTE_FLOW_ITEM_TYPE_UDP, - .spec = &attributes->l4, - .mask = &attributes->l4_mask, - }; - break; - case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: - attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ - .src_port = input->flow.tcp4_flow.src_port, - .dst_port = input->flow.tcp4_flow.dst_port, - }; - attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ - .src_port = mask->src_port_mask, - .dst_port = mask->dst_port_mask, - }; - attributes->items[2] = (struct rte_flow_item){ - .type = RTE_FLOW_ITEM_TYPE_TCP, - .spec = &attributes->l4, - .mask = &attributes->l4_mask, - }; - break; - case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: - attributes->l4.udp.hdr = (struct rte_udp_hdr){ - .src_port = input->flow.udp6_flow.src_port, - .dst_port = input->flow.udp6_flow.dst_port, - }; - attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ - .src_port = mask->src_port_mask, - .dst_port = mask->dst_port_mask, - }; - attributes->items[2] = (struct rte_flow_item){ - .type = RTE_FLOW_ITEM_TYPE_UDP, - .spec = &attributes->l4, - .mask = &attributes->l4_mask, - }; - break; - case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: - attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ - .src_port = input->flow.tcp6_flow.src_port, - .dst_port = input->flow.tcp6_flow.dst_port, - }; - attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ - .src_port = mask->src_port_mask, - .dst_port = mask->dst_port_mask, - }; - attributes->items[2] = (struct rte_flow_item){ - .type = RTE_FLOW_ITEM_TYPE_TCP, - .spec = &attributes->l4, - .mask = &attributes->l4_mask, - }; - break; - case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: - case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: - break; - default: - DRV_LOG(ERR, "port %u invalid flow type%d", - dev->data->port_id, fdir_filter->input.flow_type); - rte_errno = ENOTSUP; - return -rte_errno; - } + *ops = &mlx5_flow_ops; return 0; } -#define FLOW_FDIR_CMP(f1, f2, fld) \ - memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) - /** - * Compare two FDIR flows. If items and actions are identical, the two flows are - * regarded as same. + * Validate meter policy actions. + * Dispatcher for action type specific validation. * - * @param dev - * Pointer to Ethernet device. - * @param f1 - * FDIR flow to compare. - * @param f2 - * FDIR flow to compare. + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] action + * The meter policy action object to validate. + * @param[in] attr + * Attributes of flow to determine steering domain. + * @param[out] is_rss + * Is RSS or not. + * @param[out] domain_bitmap + * Domain bitmap. + * @param[out] is_def_policy + * Is default policy or not. + * @param[out] error + * Perform verbose error reporting if not NULL. Initialized in case of + * error only. * * @return - * Zero on match, 1 otherwise. + * 0 on success, otherwise negative errno value. */ -static int -flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) -{ - if (FLOW_FDIR_CMP(f1, f2, attr) || - FLOW_FDIR_CMP(f1, f2, l2) || - FLOW_FDIR_CMP(f1, f2, l2_mask) || - FLOW_FDIR_CMP(f1, f2, l3) || - FLOW_FDIR_CMP(f1, f2, l3_mask) || - FLOW_FDIR_CMP(f1, f2, l4) || - FLOW_FDIR_CMP(f1, f2, l4_mask) || - FLOW_FDIR_CMP(f1, f2, actions[0].type)) - return 1; - if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && - FLOW_FDIR_CMP(f1, f2, queue)) - return 1; - return 0; +int +mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, + const struct rte_flow_action *actions[RTE_COLORS], + struct rte_flow_attr *attr, + bool *is_rss, + uint8_t *domain_bitmap, + uint8_t *policy_mode, + struct rte_mtr_error *error) +{ + const struct mlx5_flow_driver_ops *fops; + + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + return fops->validate_mtr_acts(dev, actions, attr, is_rss, + domain_bitmap, policy_mode, error); } /** - * Search device flow list to find out a matched FDIR flow. + * Destroy the meter table set. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. - * @param fdir_flow - * FDIR flow to lookup. - * - * @return - * Index of flow if found, 0 otherwise. + * @param[in] mtr_policy + * Meter policy struct. */ -static uint32_t -flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) +void +mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy) { - struct mlx5_priv *priv = dev->data->dev_private; - uint32_t flow_idx = 0; - struct mlx5_fdir_flow *priv_fdir_flow = NULL; - - MLX5_ASSERT(fdir_flow); - LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { - if (!flow_fdir_cmp(priv_fdir_flow->fdir, fdir_flow)) { - DRV_LOG(DEBUG, "port %u found FDIR flow %u", - dev->data->port_id, flow_idx); - flow_idx = priv_fdir_flow->rix_flow; - break; - } - } - return flow_idx; + const struct mlx5_flow_driver_ops *fops; + + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + fops->destroy_mtr_acts(dev, mtr_policy); } /** - * Add new flow director filter and store it in list. + * Create policy action, lock free, + * (mutex should be acquired by caller). + * Dispatcher for action type specific call. * - * @param dev - * Pointer to Ethernet device. - * @param fdir_filter - * Flow director filter to add. + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] mtr_policy + * Meter policy struct. + * @param[in] action + * Action specification used to create meter actions. + * @param[out] error + * Perform verbose error reporting if not NULL. Initialized in case of + * error only. * * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. + * 0 on success, otherwise negative errno value. */ -static int -flow_fdir_filter_add(struct rte_eth_dev *dev, - const struct rte_eth_fdir_filter *fdir_filter) +int +mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy, + const struct rte_flow_action *actions[RTE_COLORS], + struct rte_mtr_error *error) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_fdir *fdir_flow; - struct rte_flow *flow; - struct mlx5_fdir_flow *priv_fdir_flow = NULL; - uint32_t flow_idx; - int ret; + const struct mlx5_flow_driver_ops *fops; - fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0); - if (!fdir_flow) { - rte_errno = ENOMEM; - return -rte_errno; - } - ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); - if (ret) - goto error; - flow_idx = flow_fdir_filter_lookup(dev, fdir_flow); - if (flow_idx) { - rte_errno = EEXIST; - goto error; - } - priv_fdir_flow = rte_zmalloc(__func__, sizeof(struct mlx5_fdir_flow), - 0); - if (!priv_fdir_flow) { - rte_errno = ENOMEM; - goto error; - } - flow_idx = flow_list_create(dev, &priv->flows, &fdir_flow->attr, - fdir_flow->items, fdir_flow->actions, true, - NULL); - flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); - if (!flow) - goto error; - flow->fdir = 1; - priv_fdir_flow->fdir = fdir_flow; - priv_fdir_flow->rix_flow = flow_idx; - LIST_INSERT_HEAD(&priv->fdir_flows, priv_fdir_flow, next); - DRV_LOG(DEBUG, "port %u created FDIR flow %p", - dev->data->port_id, (void *)flow); - return 0; -error: - rte_free(priv_fdir_flow); - rte_free(fdir_flow); - return -rte_errno; + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + return fops->create_mtr_acts(dev, mtr_policy, actions, error); } /** - * Delete specific filter. + * Create policy rules, lock free, + * (mutex should be acquired by caller). + * Dispatcher for action type specific call. * - * @param dev - * Pointer to Ethernet device. - * @param fdir_filter - * Filter to be deleted. + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] mtr_policy + * Meter policy struct. * * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. + * 0 on success, -1 otherwise. */ -static int -flow_fdir_filter_delete(struct rte_eth_dev *dev, - const struct rte_eth_fdir_filter *fdir_filter) +int +mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy) { - struct mlx5_priv *priv = dev->data->dev_private; - uint32_t flow_idx; - struct mlx5_fdir fdir_flow = { - .attr.group = 0, - }; - struct mlx5_fdir_flow *priv_fdir_flow = NULL; - int ret; + const struct mlx5_flow_driver_ops *fops; - ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); - if (ret) - return -rte_errno; - LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { - /* Find the fdir in priv list */ - if (!flow_fdir_cmp(priv_fdir_flow->fdir, &fdir_flow)) - break; - } - if (!priv_fdir_flow) - return 0; - LIST_REMOVE(priv_fdir_flow, next); - flow_idx = priv_fdir_flow->rix_flow; - flow_list_destroy(dev, &priv->flows, flow_idx); - rte_free(priv_fdir_flow->fdir); - rte_free(priv_fdir_flow); - DRV_LOG(DEBUG, "port %u deleted FDIR flow %u", - dev->data->port_id, flow_idx); - return 0; + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + return fops->create_policy_rules(dev, mtr_policy); } /** - * Update queue for specific filter. + * Destroy policy rules, lock free, + * (mutex should be acquired by caller). + * Dispatcher for action type specific call. * - * @param dev - * Pointer to Ethernet device. - * @param fdir_filter - * Filter to be updated. - * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] mtr_policy + * Meter policy struct. */ -static int -flow_fdir_filter_update(struct rte_eth_dev *dev, - const struct rte_eth_fdir_filter *fdir_filter) +void +mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy) { - int ret; + const struct mlx5_flow_driver_ops *fops; - ret = flow_fdir_filter_delete(dev, fdir_filter); - if (ret) - return ret; - return flow_fdir_filter_add(dev, fdir_filter); + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + fops->destroy_policy_rules(dev, mtr_policy); } /** - * Flush all filters. + * Destroy the default policy table set. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. */ -static void -flow_fdir_filter_flush(struct rte_eth_dev *dev) +void +mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_fdir_flow *priv_fdir_flow = NULL; + const struct mlx5_flow_driver_ops *fops; - while (!LIST_EMPTY(&priv->fdir_flows)) { - priv_fdir_flow = LIST_FIRST(&priv->fdir_flows); - LIST_REMOVE(priv_fdir_flow, next); - flow_list_destroy(dev, &priv->flows, priv_fdir_flow->rix_flow); - rte_free(priv_fdir_flow->fdir); - rte_free(priv_fdir_flow); - } + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + fops->destroy_def_policy(dev); } /** - * Get flow director information. + * Destroy the default policy table set. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. - * @param[out] fdir_info - * Resulting flow director information. + * + * @return + * 0 on success, -1 otherwise. */ -static void -flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) +int +mlx5_flow_create_def_policy(struct rte_eth_dev *dev) { - struct rte_eth_fdir_masks *mask = - &dev->data->dev_conf.fdir_conf.mask; + const struct mlx5_flow_driver_ops *fops; - fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; - fdir_info->guarant_spc = 0; - rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); - fdir_info->max_flexpayload = 0; - fdir_info->flow_types_mask[0] = 0; - fdir_info->flex_payload_unit = 0; - fdir_info->max_flex_payload_segment_num = 0; - fdir_info->flex_payload_limit = 0; - memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + return fops->create_def_policy(dev); } /** - * Deal with flow director operations. + * Create the needed meter and suffix tables. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. - * @param filter_op - * Operation to perform. - * @param arg - * Pointer to operation-specific structure. * * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. + * 0 on success, -1 otherwise. */ -static int -flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, - void *arg) +int +mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, + struct mlx5_flow_meter_info *fm, + uint32_t mtr_idx, + uint8_t domain_bitmap) { - enum rte_fdir_mode fdir_mode = - dev->data->dev_conf.fdir_conf.mode; + const struct mlx5_flow_driver_ops *fops; - if (filter_op == RTE_ETH_FILTER_NOP) - return 0; - if (fdir_mode != RTE_FDIR_MODE_PERFECT && - fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { - DRV_LOG(ERR, "port %u flow director mode %d not supported", - dev->data->port_id, fdir_mode); - rte_errno = EINVAL; - return -rte_errno; - } - switch (filter_op) { - case RTE_ETH_FILTER_ADD: - return flow_fdir_filter_add(dev, arg); - case RTE_ETH_FILTER_UPDATE: - return flow_fdir_filter_update(dev, arg); - case RTE_ETH_FILTER_DELETE: - return flow_fdir_filter_delete(dev, arg); - case RTE_ETH_FILTER_FLUSH: - flow_fdir_filter_flush(dev); - break; - case RTE_ETH_FILTER_INFO: - flow_fdir_info_get(dev, arg); - break; - default: - DRV_LOG(DEBUG, "port %u unknown operation %u", - dev->data->port_id, filter_op); - rte_errno = EINVAL; - return -rte_errno; - } - return 0; + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); } /** - * Manage filter operations. - * - * @param dev - * Pointer to Ethernet device structure. - * @param filter_type - * Filter type. - * @param filter_op - * Operation to perform. - * @param arg - * Pointer to operation-specific structure. + * Destroy the meter table set. * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] tbl + * Pointer to the meter table set. */ -int -mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, - enum rte_filter_type filter_type, - enum rte_filter_op filter_op, - void *arg) -{ - switch (filter_type) { - case RTE_ETH_FILTER_GENERIC: - if (filter_op != RTE_ETH_FILTER_GET) { - rte_errno = EINVAL; - return -rte_errno; - } - *(const void **)arg = &mlx5_flow_ops; - return 0; - case RTE_ETH_FILTER_FDIR: - return flow_fdir_ctrl_func(dev, filter_op, arg); - default: - DRV_LOG(ERR, "port %u filter type (%d) not supported", - dev->data->port_id, filter_type); - rte_errno = ENOTSUP; - return -rte_errno; - } - return 0; +void +mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, + struct mlx5_flow_meter_info *fm) +{ + const struct mlx5_flow_driver_ops *fops; + + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + fops->destroy_mtr_tbls(dev, fm); } /** - * Create the needed meter and suffix tables. + * Destroy the global meter drop table. * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. - * - * @return - * Pointer to table set on success, NULL otherwise. */ -struct mlx5_meter_domains_infos * -mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +void +mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_mtr_tbls(dev, fm); + fops->destroy_mtr_drop_tbls(dev); } /** - * Destroy the meter table set. + * Destroy the sub policy table with RX queue. * * @param[in] dev * Pointer to Ethernet device. - * @param[in] tbl - * Pointer to the meter table set. - * - * @return - * 0 on success. + * @param[in] mtr_policy + * Pointer to meter policy table. */ -int -mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, - struct mlx5_meter_domains_infos *tbls) +void +mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->destroy_mtr_tbls(dev, tbls); + fops->destroy_sub_policy_with_rxq(dev, mtr_policy); } /** - * Create policer rules. + * Allocate the needed aso flow meter id. * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to flow meter structure. - * @param[in] attr - * Pointer to flow attributes. * * @return - * 0 on success, -1 otherwise. + * Index to aso flow meter on success, NULL otherwise. */ -int -mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) +uint32_t +mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_policer_rules(dev, fm, attr); + return fops->create_meter(dev); } /** - * Destroy policer rules. + * Free the aso flow meter id. * - * @param[in] fm - * Pointer to flow meter structure. - * @param[in] attr - * Pointer to flow attributes. + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] mtr_idx + * Index to aso flow meter to be free. * * @return - * 0 on success, -1 otherwise. + * 0 on success. */ -int -mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) +void +mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->destroy_policer_rules(dev, fm, attr); + fops->free_meter(dev, mtr_idx); } /** @@ -5877,33 +7813,103 @@ mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, return -ENOTSUP; } -#define MLX5_POOL_QUERY_FREQ_US 1000000 - /** - * Get number of all validate pools. + * Allocate a new memory for the counter values wrapped by all the needed + * management. * * @param[in] sh * Pointer to mlx5_dev_ctx_shared object. * * @return - * The number of all validate pools. + * 0 on success, a negative errno value otherwise. */ -static uint32_t -mlx5_get_all_valid_pool_count(struct mlx5_dev_ctx_shared *sh) +static int +mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) { + struct mlx5_counter_stats_mem_mng *mem_mng; + volatile struct flow_counter_stats *raw_data; + int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; + int size = (sizeof(struct flow_counter_stats) * + MLX5_COUNTERS_PER_POOL + + sizeof(struct mlx5_counter_stats_raw)) * raws_n + + sizeof(struct mlx5_counter_stats_mem_mng); + size_t pgsize = rte_mem_page_size(); + uint8_t *mem; + int ret; int i; - uint32_t pools_n = 0; - - for (i = 0; i < MLX5_CCONT_TYPE_MAX; ++i) - pools_n += rte_atomic16_read(&sh->cmng.ccont[i].n_valid); - return pools_n; -} -/** - * Set the periodic procedure for triggering asynchronous batch queries for all - * the counter pools. - * - * @param[in] sh + if (pgsize == (size_t)-1) { + DRV_LOG(ERR, "Failed to get mem page size"); + rte_errno = ENOMEM; + return -ENOMEM; + } + mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); + if (!mem) { + rte_errno = ENOMEM; + return -ENOMEM; + } + mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; + size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; + ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, + sh->cdev->pdn, mem, size, + &mem_mng->wm); + if (ret) { + rte_errno = errno; + mlx5_free(mem); + return -rte_errno; + } + mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); + raw_data = (volatile struct flow_counter_stats *)mem; + for (i = 0; i < raws_n; ++i) { + mem_mng->raws[i].mem_mng = mem_mng; + mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; + } + for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) + LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, + mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, + next); + LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); + sh->cmng.mem_mng = mem_mng; + return 0; +} + +/** + * Set the statistic memory to the new counter pool. + * + * @param[in] sh + * Pointer to mlx5_dev_ctx_shared object. + * @param[in] pool + * Pointer to the pool to set the statistic memory. + * + * @return + * 0 on success, a negative errno value otherwise. + */ +static int +mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, + struct mlx5_flow_counter_pool *pool) +{ + struct mlx5_flow_counter_mng *cmng = &sh->cmng; + /* Resize statistic memory once used out. */ + if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && + mlx5_flow_create_counter_stat_mem_mng(sh)) { + DRV_LOG(ERR, "Cannot resize counter stat mem."); + return -1; + } + rte_spinlock_lock(&pool->sl); + pool->raw = cmng->mem_mng->raws + pool->index % + MLX5_CNT_CONTAINER_RESIZE; + rte_spinlock_unlock(&pool->sl); + pool->raw_hw = NULL; + return 0; +} + +#define MLX5_POOL_QUERY_FREQ_US 1000000 + +/** + * Set the periodic procedure for triggering asynchronous batch queries for all + * the counter pools. + * + * @param[in] sh * Pointer to mlx5_dev_ctx_shared object. */ void @@ -5911,7 +7917,7 @@ mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) { uint32_t pools_n, us; - pools_n = mlx5_get_all_valid_pool_count(sh); + pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); us = MLX5_POOL_QUERY_FREQ_US / pools_n; DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { @@ -5933,37 +7939,21 @@ void mlx5_flow_query_alarm(void *arg) { struct mlx5_dev_ctx_shared *sh = arg; - struct mlx5_devx_obj *dcs; - uint16_t offset; int ret; - uint8_t batch = sh->cmng.batch; - uint8_t age = sh->cmng.age; uint16_t pool_index = sh->cmng.pool_index; - struct mlx5_pools_container *cont; + struct mlx5_flow_counter_mng *cmng = &sh->cmng; struct mlx5_flow_counter_pool *pool; - int cont_loop = MLX5_CCONT_TYPE_MAX; + uint16_t n_valid; if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) goto set_alarm; -next_container: - cont = MLX5_CNT_CONTAINER(sh, batch, age); - rte_spinlock_lock(&cont->resize_sl); - if (!cont->pools) { - rte_spinlock_unlock(&cont->resize_sl); - /* Check if all the containers are empty. */ - if (unlikely(--cont_loop == 0)) - goto set_alarm; - batch ^= 0x1; - pool_index = 0; - if (batch == 0 && pool_index == 0) { - age ^= 0x1; - sh->cmng.batch = batch; - sh->cmng.age = age; - } - goto next_container; - } - pool = cont->pools[pool_index]; - rte_spinlock_unlock(&cont->resize_sl); + rte_spinlock_lock(&cmng->pool_update_sl); + pool = cmng->pools[pool_index]; + n_valid = cmng->n_valid; + rte_spinlock_unlock(&cmng->pool_update_sl); + /* Set the statistic memory to the new created pool. */ + if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) + goto set_alarm; if (pool->raw_hw) /* There is a pool query in progress. */ goto set_alarm; @@ -5972,21 +7962,19 @@ next_container: if (!pool->raw_hw) /* No free counter statistics raw memory. */ goto set_alarm; - dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read - (&pool->a64_dcs); - offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL; /* * Identify the counters released between query trigger and query - * handle more effiecntly. The counter released in this gap period + * handle more efficiently. The counter released in this gap period * should wait for a new round of query as the new arrived packets * will not be taken into account. */ pool->query_gen++; - ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL - - offset, NULL, NULL, - pool->raw_hw->mem_mng->dm->id, + ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, + MLX5_COUNTERS_PER_POOL, + NULL, NULL, + pool->raw_hw->mem_mng->wm.lkey, (void *)(uintptr_t) - (pool->raw_hw->data + offset), + pool->raw_hw->data, sh->devx_comp, (uint64_t)(uintptr_t)pool); if (ret) { @@ -5995,20 +7983,13 @@ next_container: pool->raw_hw = NULL; goto set_alarm; } - pool->raw_hw->min_dcs_id = dcs->id; LIST_REMOVE(pool->raw_hw, next); sh->cmng.pending_queries++; pool_index++; - if (pool_index >= rte_atomic16_read(&cont->n_valid)) { - batch ^= 0x1; + if (pool_index >= n_valid) pool_index = 0; - if (batch == 0 && pool_index == 0) - age ^= 0x1; - } set_alarm: - sh->cmng.batch = batch; sh->cmng.pool_index = pool_index; - sh->cmng.age = age; mlx5_set_query_alarm(sh); } @@ -6030,19 +8011,26 @@ mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, struct mlx5_age_param *age_param; struct mlx5_counter_stats_raw *cur = pool->raw_hw; struct mlx5_counter_stats_raw *prev = pool->raw; - uint16_t curr = rte_rdtsc() / (rte_get_tsc_hz() / 10); + const uint64_t curr_time = MLX5_CURR_TIME_SEC; + const uint32_t time_delta = curr_time - pool->time_of_last_age_check; + uint16_t expected = AGE_CANDIDATE; uint32_t i; + pool->time_of_last_age_check = curr_time; for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { cnt = MLX5_POOL_GET_CNT(pool, i); age_param = MLX5_CNT_TO_AGE(cnt); - if (rte_atomic16_read(&age_param->state) != AGE_CANDIDATE) + if (__atomic_load_n(&age_param->state, + __ATOMIC_RELAXED) != AGE_CANDIDATE) continue; if (cur->data[i].hits != prev->data[i].hits) { - age_param->expire = curr + age_param->timeout; + __atomic_store_n(&age_param->sec_since_last_hit, 0, + __ATOMIC_RELAXED); continue; } - if ((uint16_t)(curr - age_param->expire) >= (UINT16_MAX / 2)) + if (__atomic_add_fetch(&age_param->sec_since_last_hit, + time_delta, + __ATOMIC_RELAXED) <= age_param->timeout) continue; /** * Hold the lock first, or if between the @@ -6053,27 +8041,16 @@ mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, priv = rte_eth_devices[age_param->port_id].data->dev_private; age_info = GET_PORT_AGE_INFO(priv); rte_spinlock_lock(&age_info->aged_sl); - /* If the cpmset fails, release happens. */ - if (rte_atomic16_cmpset((volatile uint16_t *) - &age_param->state, - AGE_CANDIDATE, - AGE_TMOUT) == - AGE_CANDIDATE) { + if (__atomic_compare_exchange_n(&age_param->state, &expected, + AGE_TMOUT, false, + __ATOMIC_RELAXED, + __ATOMIC_RELAXED)) { TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); } rte_spinlock_unlock(&age_info->aged_sl); } - for (i = 0; i < sh->max_port; i++) { - age_info = &sh->port[i].age_info; - if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) - continue; - if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) - _rte_eth_dev_callback_process - (&rte_eth_devices[sh->port[i].devx_ih_port_id], - RTE_ETH_EVENT_FLOW_AGED, NULL); - age_info->flags = 0; - } + mlx5_age_event_prepare(sh); } /** @@ -6094,27 +8071,28 @@ mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, struct mlx5_flow_counter_pool *pool = (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; struct mlx5_counter_stats_raw *raw_to_free; - uint8_t age = !!IS_AGE_POOL(pool); uint8_t query_gen = pool->query_gen ^ 1; - struct mlx5_pools_container *cont = - MLX5_CNT_CONTAINER(sh, !IS_EXT_POOL(pool), age); + struct mlx5_flow_counter_mng *cmng = &sh->cmng; + enum mlx5_counter_type cnt_type = + pool->is_aged ? MLX5_COUNTER_TYPE_AGE : + MLX5_COUNTER_TYPE_ORIGIN; if (unlikely(status)) { raw_to_free = pool->raw_hw; } else { raw_to_free = pool->raw; - if (IS_AGE_POOL(pool)) + if (pool->is_aged) mlx5_flow_aging_check(sh, pool); rte_spinlock_lock(&pool->sl); pool->raw = pool->raw_hw; rte_spinlock_unlock(&pool->sl); /* Be sure the new raw counters data is updated in memory. */ - rte_cio_wmb(); + rte_io_wmb(); if (!TAILQ_EMPTY(&pool->counters[query_gen])) { - rte_spinlock_lock(&cont->csl); - TAILQ_CONCAT(&cont->counters, + rte_spinlock_lock(&cmng->csl[cnt_type]); + TAILQ_CONCAT(&cmng->counters[cnt_type], &pool->counters[query_gen], next); - rte_spinlock_unlock(&cont->csl); + rte_spinlock_unlock(&cmng->csl[cnt_type]); } } LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); @@ -6122,19 +8100,59 @@ mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, sh->cmng.pending_queries--; } +static int +flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, + const struct flow_grp_info *grp_info, + struct rte_flow_error *error) +{ + if (grp_info->transfer && grp_info->external && + grp_info->fdb_def_rule) { + if (group == UINT32_MAX) + return rte_flow_error_set + (error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_GROUP, + NULL, + "group index not supported"); + *table = group + 1; + } else { + *table = group; + } + DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); + return 0; +} + /** * Translate the rte_flow group index to HW table value. * - * @param[in] attributes - * Pointer to flow attributes - * @param[in] external - * Value is part of flow rule created by request external to PMD. + * If tunnel offload is disabled, all group ids converted to flow table + * id using the standard method. + * If tunnel offload is enabled, group id can be converted using the + * standard or tunnel conversion method. Group conversion method + * selection depends on flags in `grp_info` parameter: + * - Internal (grp_info.external == 0) groups conversion uses the + * standard method. + * - Group ids in JUMP action converted with the tunnel conversion. + * - Group id in rule attribute conversion depends on a rule type and + * group id value: + * ** non zero group attributes converted with the tunnel method + * ** zero group attribute in non-tunnel rule is converted using the + * standard method - there's only one root table + * ** zero group attribute in steer tunnel rule is converted with the + * standard method - single root table + * ** zero group attribute in match tunnel rule is a special OvS + * case: that value is used for portability reasons. That group + * id is converted with the tunnel conversion method. + * + * @param[in] dev + * Port device + * @param[in] tunnel + * PMD tunnel offload object * @param[in] group * rte_flow group index value. - * @param[out] fdb_def_rule - * Whether fdb jump to table 1 is configured. * @param[out] table * HW table value. + * @param[in] grp_info + * flags used for conversion * @param[out] error * Pointer to error structure. * @@ -6142,22 +8160,37 @@ mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external, - uint32_t group, bool fdb_def_rule, uint32_t *table, +mlx5_flow_group_to_table(struct rte_eth_dev *dev, + const struct mlx5_flow_tunnel *tunnel, + uint32_t group, uint32_t *table, + const struct flow_grp_info *grp_info, struct rte_flow_error *error) { - if (attributes->transfer && external && fdb_def_rule) { - if (group == UINT32_MAX) - return rte_flow_error_set - (error, EINVAL, - RTE_FLOW_ERROR_TYPE_ATTR_GROUP, - NULL, - "group index not supported"); - *table = group + 1; + int ret; + bool standard_translation; + + if (!grp_info->skip_scale && grp_info->external && + group < MLX5_MAX_TABLES_EXTERNAL) + group *= MLX5_FLOW_TABLE_FACTOR; + if (is_tunnel_offload_active(dev)) { + standard_translation = !grp_info->external || + grp_info->std_tbl_fix; } else { - *table = group; + standard_translation = true; } - return 0; + DRV_LOG(DEBUG, + "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", + dev->data->port_id, group, grp_info->transfer, + grp_info->external, grp_info->fdb_def_rule, + standard_translation ? "STANDARD" : "TUNNEL"); + if (standard_translation) + ret = flow_group_to_table(dev->data->port_id, group, table, + grp_info, error); + else + ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, + table, error); + + return ret; } /** @@ -6175,18 +8208,17 @@ int mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_config *config = &priv->config; enum modify_reg idx; int n = 0; /* reg_c[0] and reg_c[1] are reserved. */ - config->flow_mreg_c[n++] = REG_C_0; - config->flow_mreg_c[n++] = REG_C_1; + priv->sh->flow_mreg_c[n++] = REG_C_0; + priv->sh->flow_mreg_c[n++] = REG_C_1; /* Discover availability of other reg_c's. */ for (idx = REG_C_2; idx <= REG_C_7; ++idx) { struct rte_flow_attr attr = { .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, - .priority = MLX5_FLOW_PRIO_RSVD, + .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, .ingress = 1, }; struct rte_flow_item items[] = { @@ -6217,23 +8249,294 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) struct rte_flow *flow; struct rte_flow_error error; - if (!config->dv_flow_en) + if (!priv->config.dv_flow_en) break; /* Create internal flow, validation skips copy action. */ - flow_idx = flow_list_create(dev, NULL, &attr, items, - actions, false, &error); - flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], + flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, + items, actions, false, &error); + flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], flow_idx); if (!flow) continue; - if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL)) - config->flow_mreg_c[n++] = idx; - flow_list_destroy(dev, NULL, flow_idx); + priv->sh->flow_mreg_c[n++] = idx; + flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); } for (; n < MLX5_MREG_C_NUM; ++n) - config->flow_mreg_c[n] = REG_NONE; + priv->sh->flow_mreg_c[n] = REG_NON; + priv->sh->metadata_regc_check_flag = 1; + return 0; +} + +int +save_dump_file(const uint8_t *data, uint32_t size, + uint32_t type, uint64_t id, void *arg, FILE *file) +{ + char line[BUF_SIZE]; + uint32_t out = 0; + uint32_t k; + uint32_t actions_num; + struct rte_flow_query_count *count; + + memset(line, 0, BUF_SIZE); + switch (type) { + case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: + actions_num = *(uint32_t *)(arg); + out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", + type, id, actions_num); + break; + case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: + out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", + type, id); + break; + case DR_DUMP_REC_TYPE_PMD_COUNTER: + count = (struct rte_flow_query_count *)arg; + fprintf(file, + "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", + type, id, count->hits, count->bytes); + return 0; + default: + return -1; + } + + for (k = 0; k < size; k++) { + /* Make sure we do not overrun the line buffer length. */ + if (out >= BUF_SIZE - 4) { + line[out] = '\0'; + break; + } + out += snprintf(line + out, BUF_SIZE - out, "%02x", + (data[k]) & 0xff); + } + fprintf(file, "%s\n", line); + return 0; +} + +int +mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, + struct rte_flow_query_count *count, struct rte_flow_error *error) +{ + struct rte_flow_action action[2]; + enum mlx5_flow_drv_type ftype; + const struct mlx5_flow_driver_ops *fops; + + if (!flow) { + return rte_flow_error_set(error, ENOENT, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "invalid flow handle"); + } + action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; + action[1].type = RTE_FLOW_ACTION_TYPE_END; + if (flow->counter) { + memset(count, 0, sizeof(struct rte_flow_query_count)); + ftype = (enum mlx5_flow_drv_type)(flow->drv_type); + MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && + ftype < MLX5_FLOW_TYPE_MAX); + fops = flow_get_drv_ops(ftype); + return fops->query(dev, flow, action, count, error); + } + return -1; +} + +#ifdef HAVE_IBV_FLOW_DV_SUPPORT +/** + * Dump flow ipool data to file + * + * @param[in] dev + * The pointer to Ethernet device. + * @param[in] file + * A pointer to a file for output. + * @param[out] error + * Perform verbose error reporting if not NULL. PMDs initialize this + * structure in case of error only. + * @return + * 0 on success, a negative value otherwise. + */ +int +mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, + struct rte_flow *flow, FILE *file, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; + struct mlx5_flow_dv_encap_decap_resource *encap_decap; + uint32_t handle_idx; + struct mlx5_flow_handle *dh; + struct rte_flow_query_count count; + uint32_t actions_num; + const uint8_t *data; + size_t size; + uint64_t id; + uint32_t type; + void *action = NULL; + + if (!flow) { + return rte_flow_error_set(error, ENOENT, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "invalid flow handle"); + } + handle_idx = flow->dev_handles; + while (handle_idx) { + dh = mlx5_ipool_get(priv->sh->ipool + [MLX5_IPOOL_MLX5_FLOW], handle_idx); + if (!dh) + continue; + handle_idx = dh->next.next; + + /* query counter */ + type = DR_DUMP_REC_TYPE_PMD_COUNTER; + flow_dv_query_count_ptr(dev, flow->counter, + &action, error); + if (action) { + id = (uint64_t)(uintptr_t)action; + if (!mlx5_flow_query_counter(dev, flow, &count, error)) + save_dump_file(NULL, 0, type, + id, (void *)&count, file); + } + /* Get modify_hdr and encap_decap buf from ipools. */ + encap_decap = NULL; + modify_hdr = dh->dvh.modify_hdr; + + if (dh->dvh.rix_encap_decap) { + encap_decap = mlx5_ipool_get(priv->sh->ipool + [MLX5_IPOOL_DECAP_ENCAP], + dh->dvh.rix_encap_decap); + } + if (modify_hdr) { + data = (const uint8_t *)modify_hdr->actions; + size = (size_t)(modify_hdr->actions_num) * 8; + id = (uint64_t)(uintptr_t)modify_hdr->action; + actions_num = modify_hdr->actions_num; + type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; + save_dump_file(data, size, type, id, + (void *)(&actions_num), file); + } + if (encap_decap) { + data = encap_decap->buf; + size = encap_decap->size; + id = (uint64_t)(uintptr_t)encap_decap->action; + type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; + save_dump_file(data, size, type, + id, NULL, file); + } + } + return 0; +} + +/** + * Dump all flow's encap_decap/modify_hdr/counter data to file + * + * @param[in] dev + * The pointer to Ethernet device. + * @param[in] file + * A pointer to a file for output. + * @param[out] error + * Perform verbose error reporting if not NULL. PMDs initialize this + * structure in case of error only. + * @return + * 0 on success, a negative value otherwise. + */ +static int +mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, + FILE *file, struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_ctx_shared *sh = priv->sh; + struct mlx5_hlist *h; + struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; + struct mlx5_flow_dv_encap_decap_resource *encap_decap; + struct rte_flow_query_count count; + uint32_t actions_num; + const uint8_t *data; + size_t size; + uint64_t id; + uint32_t type; + uint32_t i; + uint32_t j; + struct mlx5_list_inconst *l_inconst; + struct mlx5_list_entry *e; + int lcore_index; + struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; + uint32_t max; + void *action; + + /* encap_decap hlist is lcore_share, get global core cache. */ + i = MLX5_LIST_GLOBAL; + h = sh->encaps_decaps; + if (h) { + for (j = 0; j <= h->mask; j++) { + l_inconst = &h->buckets[j].l; + if (!l_inconst || !l_inconst->cache[i]) + continue; + + e = LIST_FIRST(&l_inconst->cache[i]->h); + while (e) { + encap_decap = + (struct mlx5_flow_dv_encap_decap_resource *)e; + data = encap_decap->buf; + size = encap_decap->size; + id = (uint64_t)(uintptr_t)encap_decap->action; + type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; + save_dump_file(data, size, type, + id, NULL, file); + e = LIST_NEXT(e, next); + } + } + } + + /* get modify_hdr */ + h = sh->modify_cmds; + if (h) { + lcore_index = rte_lcore_index(rte_lcore_id()); + if (unlikely(lcore_index == -1)) { + lcore_index = MLX5_LIST_NLCORE; + rte_spinlock_lock(&h->l_const.lcore_lock); + } + i = lcore_index; + + for (j = 0; j <= h->mask; j++) { + l_inconst = &h->buckets[j].l; + if (!l_inconst || !l_inconst->cache[i]) + continue; + + e = LIST_FIRST(&l_inconst->cache[i]->h); + while (e) { + modify_hdr = + (struct mlx5_flow_dv_modify_hdr_resource *)e; + data = (const uint8_t *)modify_hdr->actions; + size = (size_t)(modify_hdr->actions_num) * 8; + actions_num = modify_hdr->actions_num; + id = (uint64_t)(uintptr_t)modify_hdr->action; + type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; + save_dump_file(data, size, type, id, + (void *)(&actions_num), file); + e = LIST_NEXT(e, next); + } + } + + if (unlikely(lcore_index == MLX5_LIST_NLCORE)) + rte_spinlock_unlock(&h->l_const.lcore_lock); + } + + /* get counter */ + MLX5_ASSERT(cmng->n_valid <= cmng->n); + max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; + for (j = 1; j <= max; j++) { + action = NULL; + flow_dv_query_count_ptr(dev, j, &action, error); + if (action) { + if (!flow_dv_query_count(dev, j, &count, error)) { + type = DR_DUMP_REC_TYPE_PMD_COUNTER; + id = (uint64_t)(uintptr_t)action; + save_dump_file(NULL, 0, type, + id, (void *)&count, file); + } + } + } return 0; } +#endif /** * Dump flow raw hw data to file @@ -6246,18 +8549,60 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) * Perform verbose error reporting if not NULL. PMDs initialize this * structure in case of error only. * @return - * 0 on success, a nagative value otherwise. + * 0 on success, a negative value otherwise. */ int -mlx5_flow_dev_dump(struct rte_eth_dev *dev, +mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, FILE *file, struct rte_flow_error *error __rte_unused) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_dev_ctx_shared *sh = priv->sh; + uint32_t handle_idx; + int ret; + struct mlx5_flow_handle *dh; + struct rte_flow *flow; + + if (!priv->config.dv_flow_en) { + if (fputs("device dv flow disabled\n", file) <= 0) + return -errno; + return -ENOTSUP; + } + + /* dump all */ + if (!flow_idx) { +#ifdef HAVE_IBV_FLOW_DV_SUPPORT + if (mlx5_flow_dev_dump_sh_all(dev, file, error)) + return -EINVAL; +#endif + return mlx5_devx_cmd_flow_dump(sh->fdb_domain, + sh->rx_domain, + sh->tx_domain, file); + } + /* dump one */ + flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], + (uintptr_t)(void *)flow_idx); + if (!flow) + return -EINVAL; - return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain, - sh->tx_domain, file); +#ifdef HAVE_IBV_FLOW_DV_SUPPORT + mlx5_flow_dev_dump_ipool(dev, flow, file, error); +#endif + handle_idx = flow->dev_handles; + while (handle_idx) { + dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], + handle_idx); + if (!dh) + return -ENOENT; + if (dh->drv_flow) { + ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, + file); + if (ret) + return -ENOENT; + } + handle_idx = dh->next.next; + } + return 0; } /** @@ -6296,3 +8641,1392 @@ mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, dev->data->port_id); return -ENOTSUP; } + +/* Wrapper for driver action_validate op callback */ +static int +flow_drv_action_validate(struct rte_eth_dev *dev, + const struct rte_flow_indir_action_conf *conf, + const struct rte_flow_action *action, + const struct mlx5_flow_driver_ops *fops, + struct rte_flow_error *error) +{ + static const char err_msg[] = "indirect action validation unsupported"; + + if (!fops->action_validate) { + DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, + NULL, err_msg); + return -rte_errno; + } + return fops->action_validate(dev, conf, action, error); +} + +/** + * Destroys the shared action by handle. + * + * @param dev + * Pointer to Ethernet device structure. + * @param[in] handle + * Handle for the indirect action object to be destroyed. + * @param[out] error + * Perform verbose error reporting if not NULL. PMDs initialize this + * structure in case of error only. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + * + * @note: wrapper for driver action_create op callback. + */ +static int +mlx5_action_handle_destroy(struct rte_eth_dev *dev, + struct rte_flow_action_handle *handle, + struct rte_flow_error *error) +{ + static const char err_msg[] = "indirect action destruction unsupported"; + struct rte_flow_attr attr = { .transfer = 0 }; + const struct mlx5_flow_driver_ops *fops = + flow_get_drv_ops(flow_get_drv_type(dev, &attr)); + + if (!fops->action_destroy) { + DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, + NULL, err_msg); + return -rte_errno; + } + return fops->action_destroy(dev, handle, error); +} + +/* Wrapper for driver action_destroy op callback */ +static int +flow_drv_action_update(struct rte_eth_dev *dev, + struct rte_flow_action_handle *handle, + const void *update, + const struct mlx5_flow_driver_ops *fops, + struct rte_flow_error *error) +{ + static const char err_msg[] = "indirect action update unsupported"; + + if (!fops->action_update) { + DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, + NULL, err_msg); + return -rte_errno; + } + return fops->action_update(dev, handle, update, error); +} + +/* Wrapper for driver action_destroy op callback */ +static int +flow_drv_action_query(struct rte_eth_dev *dev, + const struct rte_flow_action_handle *handle, + void *data, + const struct mlx5_flow_driver_ops *fops, + struct rte_flow_error *error) +{ + static const char err_msg[] = "indirect action query unsupported"; + + if (!fops->action_query) { + DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, + NULL, err_msg); + return -rte_errno; + } + return fops->action_query(dev, handle, data, error); +} + +/** + * Create indirect action for reuse in multiple flow rules. + * + * @param dev + * Pointer to Ethernet device structure. + * @param conf + * Pointer to indirect action object configuration. + * @param[in] action + * Action configuration for indirect action object creation. + * @param[out] error + * Perform verbose error reporting if not NULL. PMDs initialize this + * structure in case of error only. + * @return + * A valid handle in case of success, NULL otherwise and rte_errno is set. + */ +static struct rte_flow_action_handle * +mlx5_action_handle_create(struct rte_eth_dev *dev, + const struct rte_flow_indir_action_conf *conf, + const struct rte_flow_action *action, + struct rte_flow_error *error) +{ + static const char err_msg[] = "indirect action creation unsupported"; + struct rte_flow_attr attr = { .transfer = 0 }; + const struct mlx5_flow_driver_ops *fops = + flow_get_drv_ops(flow_get_drv_type(dev, &attr)); + + if (flow_drv_action_validate(dev, conf, action, fops, error)) + return NULL; + if (!fops->action_create) { + DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, + NULL, err_msg); + return NULL; + } + return fops->action_create(dev, conf, action, error); +} + +/** + * Updates inplace the indirect action configuration pointed by *handle* + * with the configuration provided as *update* argument. + * The update of the indirect action configuration effects all flow rules + * reusing the action via handle. + * + * @param dev + * Pointer to Ethernet device structure. + * @param[in] handle + * Handle for the indirect action to be updated. + * @param[in] update + * Action specification used to modify the action pointed by handle. + * *update* could be of same type with the action pointed by the *handle* + * handle argument, or some other structures like a wrapper, depending on + * the indirect action type. + * @param[out] error + * Perform verbose error reporting if not NULL. PMDs initialize this + * structure in case of error only. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx5_action_handle_update(struct rte_eth_dev *dev, + struct rte_flow_action_handle *handle, + const void *update, + struct rte_flow_error *error) +{ + struct rte_flow_attr attr = { .transfer = 0 }; + const struct mlx5_flow_driver_ops *fops = + flow_get_drv_ops(flow_get_drv_type(dev, &attr)); + int ret; + + ret = flow_drv_action_validate(dev, NULL, + (const struct rte_flow_action *)update, fops, error); + if (ret) + return ret; + return flow_drv_action_update(dev, handle, update, fops, + error); +} + +/** + * Query the indirect action by handle. + * + * This function allows retrieving action-specific data such as counters. + * Data is gathered by special action which may be present/referenced in + * more than one flow rule definition. + * + * see @RTE_FLOW_ACTION_TYPE_COUNT + * + * @param dev + * Pointer to Ethernet device structure. + * @param[in] handle + * Handle for the indirect action to query. + * @param[in, out] data + * Pointer to storage for the associated query data type. + * @param[out] error + * Perform verbose error reporting if not NULL. PMDs initialize this + * structure in case of error only. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx5_action_handle_query(struct rte_eth_dev *dev, + const struct rte_flow_action_handle *handle, + void *data, + struct rte_flow_error *error) +{ + struct rte_flow_attr attr = { .transfer = 0 }; + const struct mlx5_flow_driver_ops *fops = + flow_get_drv_ops(flow_get_drv_type(dev, &attr)); + + return flow_drv_action_query(dev, handle, data, fops, error); +} + +/** + * Destroy all indirect actions (shared RSS). + * + * @param dev + * Pointer to Ethernet device. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_action_handle_flush(struct rte_eth_dev *dev) +{ + struct rte_flow_error error; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_shared_action_rss *shared_rss; + int ret = 0; + uint32_t idx; + + ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], + priv->rss_shared_actions, idx, shared_rss, next) { + ret |= mlx5_action_handle_destroy(dev, + (struct rte_flow_action_handle *)(uintptr_t)idx, &error); + } + return ret; +} + +/** + * Validate existing indirect actions against current device configuration + * and attach them to device resources. + * + * @param dev + * Pointer to Ethernet device. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_action_handle_attach(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_indexed_pool *ipool = + priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS]; + struct mlx5_shared_action_rss *shared_rss, *shared_rss_last; + int ret = 0; + uint32_t idx; + + ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { + struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; + const char *message; + uint32_t queue_idx; + + ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, + ind_tbl->queues_n, + &message, &queue_idx); + if (ret != 0) { + DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", + dev->data->port_id, ind_tbl->queues[queue_idx], + message); + break; + } + } + if (ret != 0) + return ret; + ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { + struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; + + ret = mlx5_ind_table_obj_attach(dev, ind_tbl); + if (ret != 0) { + DRV_LOG(ERR, "Port %u could not attach " + "indirection table obj %p", + dev->data->port_id, (void *)ind_tbl); + goto error; + } + } + return 0; +error: + shared_rss_last = shared_rss; + ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { + struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; + + if (shared_rss == shared_rss_last) + break; + if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) + DRV_LOG(CRIT, "Port %u could not detach " + "indirection table obj %p on rollback", + dev->data->port_id, (void *)ind_tbl); + } + return ret; +} + +/** + * Detach indirect actions of the device from its resources. + * + * @param dev + * Pointer to Ethernet device. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_action_handle_detach(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_indexed_pool *ipool = + priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS]; + struct mlx5_shared_action_rss *shared_rss, *shared_rss_last; + int ret = 0; + uint32_t idx; + + ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { + struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; + + ret = mlx5_ind_table_obj_detach(dev, ind_tbl); + if (ret != 0) { + DRV_LOG(ERR, "Port %u could not detach " + "indirection table obj %p", + dev->data->port_id, (void *)ind_tbl); + goto error; + } + } + return 0; +error: + shared_rss_last = shared_rss; + ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { + struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; + + if (shared_rss == shared_rss_last) + break; + if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) + DRV_LOG(CRIT, "Port %u could not attach " + "indirection table obj %p on rollback", + dev->data->port_id, (void *)ind_tbl); + } + return ret; +} + +#ifndef HAVE_MLX5DV_DR +#define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) +#else +#define MLX5_DOMAIN_SYNC_FLOW \ + (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) +#endif + +int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + const struct mlx5_flow_driver_ops *fops; + int ret; + struct rte_flow_attr attr = { .transfer = 0 }; + + fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); + ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); + if (ret > 0) + ret = -ret; + return ret; +} + +const struct mlx5_flow_tunnel * +mlx5_get_tof(const struct rte_flow_item *item, + const struct rte_flow_action *action, + enum mlx5_tof_rule_type *rule_type) +{ + for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + if (item->type == (typeof(item->type)) + MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { + *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; + return flow_items_to_tunnel(item); + } + } + for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { + if (action->type == (typeof(action->type)) + MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { + *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; + return flow_actions_to_tunnel(action); + } + } + return NULL; +} + +/** + * tunnel offload functionality is defined for DV environment only + */ +#ifdef HAVE_IBV_FLOW_DV_SUPPORT +__extension__ +union tunnel_offload_mark { + uint32_t val; + struct { + uint32_t app_reserve:8; + uint32_t table_id:15; + uint32_t transfer:1; + uint32_t _unused_:8; + }; +}; + +static bool +mlx5_access_tunnel_offload_db + (struct rte_eth_dev *dev, + bool (*match)(struct rte_eth_dev *, + struct mlx5_flow_tunnel *, const void *), + void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), + void (*miss)(struct rte_eth_dev *, void *), + void *ctx, bool lock_op); + +static int +flow_tunnel_add_default_miss(struct rte_eth_dev *dev, + struct rte_flow *flow, + const struct rte_flow_attr *attr, + const struct rte_flow_action *app_actions, + uint32_t flow_idx, + const struct mlx5_flow_tunnel *tunnel, + struct tunnel_default_miss_ctx *ctx, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow *dev_flow; + struct rte_flow_attr miss_attr = *attr; + const struct rte_flow_item miss_items[2] = { + { + .type = RTE_FLOW_ITEM_TYPE_ETH, + .spec = NULL, + .last = NULL, + .mask = NULL + }, + { + .type = RTE_FLOW_ITEM_TYPE_END, + .spec = NULL, + .last = NULL, + .mask = NULL + } + }; + union tunnel_offload_mark mark_id; + struct rte_flow_action_mark miss_mark; + struct rte_flow_action miss_actions[3] = { + [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, + [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } + }; + const struct rte_flow_action_jump *jump_data; + uint32_t i, flow_table = 0; /* prevent compilation warning */ + struct flow_grp_info grp_info = { + .external = 1, + .transfer = attr->transfer, + .fdb_def_rule = !!priv->fdb_def_rule, + .std_tbl_fix = 0, + }; + int ret; + + if (!attr->transfer) { + uint32_t q_size; + + miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; + q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); + ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, + 0, SOCKET_ID_ANY); + if (!ctx->queue) + return rte_flow_error_set + (error, ENOMEM, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, + NULL, "invalid default miss RSS"); + ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, + ctx->action_rss.level = 0, + ctx->action_rss.types = priv->rss_conf.rss_hf, + ctx->action_rss.key_len = priv->rss_conf.rss_key_len, + ctx->action_rss.queue_num = priv->reta_idx_n, + ctx->action_rss.key = priv->rss_conf.rss_key, + ctx->action_rss.queue = ctx->queue; + if (!priv->reta_idx_n || !priv->rxqs_n) + return rte_flow_error_set + (error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, + NULL, "invalid port configuration"); + if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) + ctx->action_rss.types = 0; + for (i = 0; i != priv->reta_idx_n; ++i) + ctx->queue[i] = (*priv->reta_idx)[i]; + } else { + miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; + ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; + } + miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; + for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); + jump_data = app_actions->conf; + miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; + miss_attr.group = jump_data->group; + ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, + &flow_table, &grp_info, error); + if (ret) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, + NULL, "invalid tunnel id"); + mark_id.app_reserve = 0; + mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); + mark_id.transfer = !!attr->transfer; + mark_id._unused_ = 0; + miss_mark.id = mark_id.val; + dev_flow = flow_drv_prepare(dev, flow, &miss_attr, + miss_items, miss_actions, flow_idx, error); + if (!dev_flow) + return -rte_errno; + dev_flow->flow = flow; + dev_flow->external = true; + dev_flow->tunnel = tunnel; + dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; + /* Subflow object was created, we must include one in the list. */ + SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, + dev_flow->handle, next); + DRV_LOG(DEBUG, + "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", + dev->data->port_id, tunnel->app_tunnel.type, + tunnel->tunnel_id, miss_attr.priority, miss_attr.group); + ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, + miss_actions, error); + if (!ret) + ret = flow_mreg_update_copy_table(dev, flow, miss_actions, + error); + + return ret; +} + +static const struct mlx5_flow_tbl_data_entry * +tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_ctx_shared *sh = priv->sh; + struct mlx5_list_entry *he; + union tunnel_offload_mark mbits = { .val = mark }; + union mlx5_flow_tbl_key table_key = { + { + .level = tunnel_id_to_flow_tbl(mbits.table_id), + .id = 0, + .reserved = 0, + .dummy = 0, + .is_fdb = !!mbits.transfer, + .is_egress = 0, + } + }; + struct mlx5_flow_cb_ctx ctx = { + .data = &table_key.v64, + }; + + he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); + return he ? + container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; +} + +static void +mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, + struct mlx5_list_entry *entry) +{ + struct mlx5_dev_ctx_shared *sh = tool_ctx; + struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); + + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], + tunnel_flow_tbl_to_id(tte->flow_table)); + mlx5_free(tte); +} + +static int +mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry, void *cb_ctx) +{ + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + union tunnel_tbl_key tbl = { + .val = *(uint64_t *)(ctx->data), + }; + struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); + + return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; +} + +static struct mlx5_list_entry * +mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) +{ + struct mlx5_dev_ctx_shared *sh = tool_ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct tunnel_tbl_entry *tte; + union tunnel_tbl_key tbl = { + .val = *(uint64_t *)(ctx->data), + }; + + tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, + sizeof(*tte), 0, + SOCKET_ID_ANY); + if (!tte) + goto err; + mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], + &tte->flow_table); + if (tte->flow_table >= MLX5_MAX_TABLES) { + DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", + tte->flow_table); + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], + tte->flow_table); + goto err; + } else if (!tte->flow_table) { + goto err; + } + tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); + tte->tunnel_id = tbl.tunnel_id; + tte->group = tbl.group; + return &tte->hash; +err: + if (tte) + mlx5_free(tte); + return NULL; +} + +static struct mlx5_list_entry * +mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *oentry, + void *cb_ctx __rte_unused) +{ + struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), + 0, SOCKET_ID_ANY); + + if (!tte) + return NULL; + memcpy(tte, oentry, sizeof(*tte)); + return &tte->hash; +} + +static void +mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry) +{ + struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); + + mlx5_free(tte); +} + +static uint32_t +tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, + const struct mlx5_flow_tunnel *tunnel, + uint32_t group, uint32_t *table, + struct rte_flow_error *error) +{ + struct mlx5_list_entry *he; + struct tunnel_tbl_entry *tte; + union tunnel_tbl_key key = { + .tunnel_id = tunnel ? tunnel->tunnel_id : 0, + .group = group + }; + struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); + struct mlx5_hlist *group_hash; + struct mlx5_flow_cb_ctx ctx = { + .data = &key.val, + }; + + group_hash = tunnel ? tunnel->groups : thub->groups; + he = mlx5_hlist_register(group_hash, key.val, &ctx); + if (!he) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_GROUP, + NULL, + "tunnel group index not supported"); + tte = container_of(he, typeof(*tte), hash); + *table = tte->flow_table; + DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", + dev->data->port_id, key.tunnel_id, group, *table); + return 0; +} + +static void +mlx5_flow_tunnel_free(struct rte_eth_dev *dev, + struct mlx5_flow_tunnel *tunnel) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_indexed_pool *ipool; + + DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", + dev->data->port_id, tunnel->tunnel_id); + LIST_REMOVE(tunnel, chain); + mlx5_hlist_destroy(tunnel->groups); + ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; + mlx5_ipool_free(ipool, tunnel->tunnel_id); +} + +static bool +mlx5_access_tunnel_offload_db + (struct rte_eth_dev *dev, + bool (*match)(struct rte_eth_dev *, + struct mlx5_flow_tunnel *, const void *), + void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), + void (*miss)(struct rte_eth_dev *, void *), + void *ctx, bool lock_op) +{ + bool verdict = false; + struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); + struct mlx5_flow_tunnel *tunnel; + + rte_spinlock_lock(&thub->sl); + LIST_FOREACH(tunnel, &thub->tunnels, chain) { + verdict = match(dev, tunnel, (const void *)ctx); + if (verdict) + break; + } + if (!lock_op) + rte_spinlock_unlock(&thub->sl); + if (verdict && hit) + hit(dev, tunnel, ctx); + if (!verdict && miss) + miss(dev, ctx); + if (lock_op) + rte_spinlock_unlock(&thub->sl); + + return verdict; +} + +struct tunnel_db_find_tunnel_id_ctx { + uint32_t tunnel_id; + struct mlx5_flow_tunnel *tunnel; +}; + +static bool +find_tunnel_id_match(struct rte_eth_dev *dev, + struct mlx5_flow_tunnel *tunnel, const void *x) +{ + const struct tunnel_db_find_tunnel_id_ctx *ctx = x; + + RTE_SET_USED(dev); + return tunnel->tunnel_id == ctx->tunnel_id; +} + +static void +find_tunnel_id_hit(struct rte_eth_dev *dev, + struct mlx5_flow_tunnel *tunnel, void *x) +{ + struct tunnel_db_find_tunnel_id_ctx *ctx = x; + RTE_SET_USED(dev); + ctx->tunnel = tunnel; +} + +static struct mlx5_flow_tunnel * +mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) +{ + struct tunnel_db_find_tunnel_id_ctx ctx = { + .tunnel_id = id, + }; + + mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, + find_tunnel_id_hit, NULL, &ctx, true); + + return ctx.tunnel; +} + +static struct mlx5_flow_tunnel * +mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, + const struct rte_flow_tunnel *app_tunnel) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_indexed_pool *ipool; + struct mlx5_flow_tunnel *tunnel; + uint32_t id; + + ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; + tunnel = mlx5_ipool_zmalloc(ipool, &id); + if (!tunnel) + return NULL; + if (id >= MLX5_MAX_TUNNELS) { + mlx5_ipool_free(ipool, id); + DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); + return NULL; + } + tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, + priv->sh, + mlx5_flow_tunnel_grp2tbl_create_cb, + mlx5_flow_tunnel_grp2tbl_match_cb, + mlx5_flow_tunnel_grp2tbl_remove_cb, + mlx5_flow_tunnel_grp2tbl_clone_cb, + mlx5_flow_tunnel_grp2tbl_clone_free_cb); + if (!tunnel->groups) { + mlx5_ipool_free(ipool, id); + return NULL; + } + /* initiate new PMD tunnel */ + memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); + tunnel->tunnel_id = id; + tunnel->action.type = (typeof(tunnel->action.type)) + MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; + tunnel->action.conf = tunnel; + tunnel->item.type = (typeof(tunnel->item.type)) + MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; + tunnel->item.spec = tunnel; + tunnel->item.last = NULL; + tunnel->item.mask = NULL; + + DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", + dev->data->port_id, tunnel->tunnel_id); + + return tunnel; +} + +struct tunnel_db_get_tunnel_ctx { + const struct rte_flow_tunnel *app_tunnel; + struct mlx5_flow_tunnel *tunnel; +}; + +static bool get_tunnel_match(struct rte_eth_dev *dev, + struct mlx5_flow_tunnel *tunnel, const void *x) +{ + const struct tunnel_db_get_tunnel_ctx *ctx = x; + + RTE_SET_USED(dev); + return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, + sizeof(*ctx->app_tunnel)); +} + +static void get_tunnel_hit(struct rte_eth_dev *dev, + struct mlx5_flow_tunnel *tunnel, void *x) +{ + /* called under tunnel spinlock protection */ + struct tunnel_db_get_tunnel_ctx *ctx = x; + + RTE_SET_USED(dev); + tunnel->refctn++; + ctx->tunnel = tunnel; +} + +static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) +{ + /* called under tunnel spinlock protection */ + struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); + struct tunnel_db_get_tunnel_ctx *ctx = x; + + rte_spinlock_unlock(&thub->sl); + ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); + rte_spinlock_lock(&thub->sl); + if (ctx->tunnel) { + ctx->tunnel->refctn = 1; + LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); + } +} + + +static int +mlx5_get_flow_tunnel(struct rte_eth_dev *dev, + const struct rte_flow_tunnel *app_tunnel, + struct mlx5_flow_tunnel **tunnel) +{ + struct tunnel_db_get_tunnel_ctx ctx = { + .app_tunnel = app_tunnel, + }; + + mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, + get_tunnel_miss, &ctx, true); + *tunnel = ctx.tunnel; + return ctx.tunnel ? 0 : -ENOMEM; +} + +void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) +{ + struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; + + if (!thub) + return; + if (!LIST_EMPTY(&thub->tunnels)) + DRV_LOG(WARNING, "port %u tunnels present", port_id); + mlx5_hlist_destroy(thub->groups); + mlx5_free(thub); +} + +int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) +{ + int err; + struct mlx5_flow_tunnel_hub *thub; + + thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), + 0, SOCKET_ID_ANY); + if (!thub) + return -ENOMEM; + LIST_INIT(&thub->tunnels); + rte_spinlock_init(&thub->sl); + thub->groups = mlx5_hlist_create("flow groups", 64, + false, true, sh, + mlx5_flow_tunnel_grp2tbl_create_cb, + mlx5_flow_tunnel_grp2tbl_match_cb, + mlx5_flow_tunnel_grp2tbl_remove_cb, + mlx5_flow_tunnel_grp2tbl_clone_cb, + mlx5_flow_tunnel_grp2tbl_clone_free_cb); + if (!thub->groups) { + err = -rte_errno; + goto err; + } + sh->tunnel_hub = thub; + + return 0; + +err: + if (thub->groups) + mlx5_hlist_destroy(thub->groups); + if (thub) + mlx5_free(thub); + return err; +} + +static inline int +mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, + struct rte_flow_tunnel *tunnel, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + + if (!priv->config.dv_flow_en) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, + "flow DV interface is off"); + if (!is_tunnel_offload_active(dev)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, + "tunnel offload was not activated"); + if (!tunnel) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, + "no application tunnel"); + switch (tunnel->type) { + default: + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, + "unsupported tunnel type"); + case RTE_FLOW_ITEM_TYPE_VXLAN: + case RTE_FLOW_ITEM_TYPE_GRE: + case RTE_FLOW_ITEM_TYPE_NVGRE: + case RTE_FLOW_ITEM_TYPE_GENEVE: + break; + } + return 0; +} + +static int +mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, + struct rte_flow_tunnel *app_tunnel, + struct rte_flow_action **actions, + uint32_t *num_of_actions, + struct rte_flow_error *error) +{ + struct mlx5_flow_tunnel *tunnel; + int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); + + if (ret) + return ret; + ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); + if (ret < 0) { + return rte_flow_error_set(error, ret, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, + "failed to initialize pmd tunnel"); + } + *actions = &tunnel->action; + *num_of_actions = 1; + return 0; +} + +static int +mlx5_flow_tunnel_match(struct rte_eth_dev *dev, + struct rte_flow_tunnel *app_tunnel, + struct rte_flow_item **items, + uint32_t *num_of_items, + struct rte_flow_error *error) +{ + struct mlx5_flow_tunnel *tunnel; + int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); + + if (ret) + return ret; + ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); + if (ret < 0) { + return rte_flow_error_set(error, ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "failed to initialize pmd tunnel"); + } + *items = &tunnel->item; + *num_of_items = 1; + return 0; +} + +struct tunnel_db_element_release_ctx { + struct rte_flow_item *items; + struct rte_flow_action *actions; + uint32_t num_elements; + struct rte_flow_error *error; + int ret; +}; + +static bool +tunnel_element_release_match(struct rte_eth_dev *dev, + struct mlx5_flow_tunnel *tunnel, const void *x) +{ + const struct tunnel_db_element_release_ctx *ctx = x; + + RTE_SET_USED(dev); + if (ctx->num_elements != 1) + return false; + else if (ctx->items) + return ctx->items == &tunnel->item; + else if (ctx->actions) + return ctx->actions == &tunnel->action; + + return false; +} + +static void +tunnel_element_release_hit(struct rte_eth_dev *dev, + struct mlx5_flow_tunnel *tunnel, void *x) +{ + struct tunnel_db_element_release_ctx *ctx = x; + ctx->ret = 0; + if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) + mlx5_flow_tunnel_free(dev, tunnel); +} + +static void +tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) +{ + struct tunnel_db_element_release_ctx *ctx = x; + RTE_SET_USED(dev); + ctx->ret = rte_flow_error_set(ctx->error, EINVAL, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "invalid argument"); +} + +static int +mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, + struct rte_flow_item *pmd_items, + uint32_t num_items, struct rte_flow_error *err) +{ + struct tunnel_db_element_release_ctx ctx = { + .items = pmd_items, + .actions = NULL, + .num_elements = num_items, + .error = err, + }; + + mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, + tunnel_element_release_hit, + tunnel_element_release_miss, &ctx, false); + + return ctx.ret; +} + +static int +mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, + struct rte_flow_action *pmd_actions, + uint32_t num_actions, struct rte_flow_error *err) +{ + struct tunnel_db_element_release_ctx ctx = { + .items = NULL, + .actions = pmd_actions, + .num_elements = num_actions, + .error = err, + }; + + mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, + tunnel_element_release_hit, + tunnel_element_release_miss, &ctx, false); + + return ctx.ret; +} + +static int +mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, + struct rte_mbuf *m, + struct rte_flow_restore_info *info, + struct rte_flow_error *err) +{ + uint64_t ol_flags = m->ol_flags; + const struct mlx5_flow_tbl_data_entry *tble; + const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; + + if (!is_tunnel_offload_active(dev)) { + info->flags = 0; + return 0; + } + + if ((ol_flags & mask) != mask) + goto err; + tble = tunnel_mark_decode(dev, m->hash.fdir.hi); + if (!tble) { + DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", + dev->data->port_id, m->hash.fdir.hi); + goto err; + } + MLX5_ASSERT(tble->tunnel); + memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); + info->group_id = tble->group_id; + info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | + RTE_FLOW_RESTORE_INFO_GROUP_ID | + RTE_FLOW_RESTORE_INFO_ENCAPSULATED; + + return 0; + +err: + return rte_flow_error_set(err, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "failed to get restore info"); +} + +#else /* HAVE_IBV_FLOW_DV_SUPPORT */ +static int +mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, + __rte_unused struct rte_flow_tunnel *app_tunnel, + __rte_unused struct rte_flow_action **actions, + __rte_unused uint32_t *num_of_actions, + __rte_unused struct rte_flow_error *error) +{ + return -ENOTSUP; +} + +static int +mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, + __rte_unused struct rte_flow_tunnel *app_tunnel, + __rte_unused struct rte_flow_item **items, + __rte_unused uint32_t *num_of_items, + __rte_unused struct rte_flow_error *error) +{ + return -ENOTSUP; +} + +static int +mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, + __rte_unused struct rte_flow_item *pmd_items, + __rte_unused uint32_t num_items, + __rte_unused struct rte_flow_error *err) +{ + return -ENOTSUP; +} + +static int +mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, + __rte_unused struct rte_flow_action *pmd_action, + __rte_unused uint32_t num_actions, + __rte_unused struct rte_flow_error *err) +{ + return -ENOTSUP; +} + +static int +mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, + __rte_unused struct rte_mbuf *m, + __rte_unused struct rte_flow_restore_info *i, + __rte_unused struct rte_flow_error *err) +{ + return -ENOTSUP; +} + +static int +flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, + __rte_unused struct rte_flow *flow, + __rte_unused const struct rte_flow_attr *attr, + __rte_unused const struct rte_flow_action *actions, + __rte_unused uint32_t flow_idx, + __rte_unused const struct mlx5_flow_tunnel *tunnel, + __rte_unused struct tunnel_default_miss_ctx *ctx, + __rte_unused struct rte_flow_error *error) +{ + return -ENOTSUP; +} + +static struct mlx5_flow_tunnel * +mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, + __rte_unused uint32_t id) +{ + return NULL; +} + +static void +mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, + __rte_unused struct mlx5_flow_tunnel *tunnel) +{ +} + +static uint32_t +tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, + __rte_unused const struct mlx5_flow_tunnel *t, + __rte_unused uint32_t group, + __rte_unused uint32_t *table, + struct rte_flow_error *error) +{ + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "tunnel offload requires DV support"); +} + +void +mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, + __rte_unused uint16_t port_id) +{ +} +#endif /* HAVE_IBV_FLOW_DV_SUPPORT */ + +/* Flex flow item API */ +static struct rte_flow_item_flex_handle * +mlx5_flow_flex_item_create(struct rte_eth_dev *dev, + const struct rte_flow_item_flex_conf *conf, + struct rte_flow_error *error) +{ + static const char err_msg[] = "flex item creation unsupported"; + struct rte_flow_attr attr = { .transfer = 0 }; + const struct mlx5_flow_driver_ops *fops = + flow_get_drv_ops(flow_get_drv_type(dev, &attr)); + + if (!fops->item_create) { + DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, + NULL, err_msg); + return NULL; + } + return fops->item_create(dev, conf, error); +} + +static int +mlx5_flow_flex_item_release(struct rte_eth_dev *dev, + const struct rte_flow_item_flex_handle *handle, + struct rte_flow_error *error) +{ + static const char err_msg[] = "flex item release unsupported"; + struct rte_flow_attr attr = { .transfer = 0 }; + const struct mlx5_flow_driver_ops *fops = + flow_get_drv_ops(flow_get_drv_type(dev, &attr)); + + if (!fops->item_release) { + DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, + NULL, err_msg); + return -rte_errno; + } + return fops->item_release(dev, handle, error); +} + +static void +mlx5_dbg__print_pattern(const struct rte_flow_item *item) +{ + int ret; + struct rte_flow_error error; + + for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + char *item_name; + ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, + sizeof(item_name), + (void *)(uintptr_t)item->type, &error); + if (ret > 0) + printf("%s ", item_name); + else + printf("%d\n", (int)item->type); + } + printf("END\n"); +} + +static int +mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) +{ + const struct rte_flow_item_udp *spec = udp_item->spec; + const struct rte_flow_item_udp *mask = udp_item->mask; + uint16_t udp_dport = 0; + + if (spec != NULL) { + if (!mask) + mask = &rte_flow_item_udp_mask; + udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & + mask->hdr.dst_port); + } + return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); +} + +static const struct mlx5_flow_expand_node * +mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, + unsigned int item_idx, + const struct mlx5_flow_expand_node graph[], + const struct mlx5_flow_expand_node *node) +{ + const struct rte_flow_item *item = pattern + item_idx, *prev_item; + + if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && + node != NULL && + node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { + /* + * The expansion node is VXLAN and it is also the last + * expandable item in the pattern, so need to continue + * expansion of the inner tunnel. + */ + MLX5_ASSERT(item_idx > 0); + prev_item = pattern + item_idx - 1; + MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); + if (mlx5_flow_is_std_vxlan_port(prev_item)) + return &graph[MLX5_EXPANSION_STD_VXLAN]; + return &graph[MLX5_EXPANSION_L3_VXLAN]; + } + return node; +} + +/* Map of Verbs to Flow priority with 8 Verbs priorities. */ +static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { + { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, +}; + +/* Map of Verbs to Flow priority with 16 Verbs priorities. */ +static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { + { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, + { 9, 10, 11 }, { 12, 13, 14 }, +}; + +/** + * Discover the number of available flow priorities. + * + * @param dev + * Ethernet device. + * + * @return + * On success, number of available flow priorities. + * On failure, a negative errno-style code and rte_errno is set. + */ +int +mlx5_flow_discover_priorities(struct rte_eth_dev *dev) +{ + static const uint16_t vprio[] = {8, 16}; + const struct mlx5_priv *priv = dev->data->dev_private; + const struct mlx5_flow_driver_ops *fops; + enum mlx5_flow_drv_type type; + int ret; + + type = mlx5_flow_os_get_type(); + if (type == MLX5_FLOW_TYPE_MAX) { + type = MLX5_FLOW_TYPE_VERBS; + if (priv->sh->devx && priv->config.dv_flow_en) + type = MLX5_FLOW_TYPE_DV; + } + fops = flow_get_drv_ops(type); + if (fops->discover_priorities == NULL) { + DRV_LOG(ERR, "Priority discovery not supported"); + rte_errno = ENOTSUP; + return -rte_errno; + } + ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); + if (ret < 0) + return ret; + switch (ret) { + case 8: + ret = RTE_DIM(priority_map_3); + break; + case 16: + ret = RTE_DIM(priority_map_5); + break; + default: + rte_errno = ENOTSUP; + DRV_LOG(ERR, + "port %u maximum priority: %d expected 8/16", + dev->data->port_id, ret); + return -rte_errno; + } + DRV_LOG(INFO, "port %u supported flow priorities:" + " 0-%d for ingress or egress root table," + " 0-%d for non-root table or transfer root table.", + dev->data->port_id, ret - 2, + MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); + return ret; +} + +/** + * Adjust flow priority based on the highest layer and the request priority. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] priority + * The rule base priority. + * @param[in] subpriority + * The priority based on the items. + * + * @return + * The new priority. + */ +uint32_t +mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, + uint32_t subpriority) +{ + uint32_t res = 0; + struct mlx5_priv *priv = dev->data->dev_private; + + switch (priv->sh->flow_max_priority) { + case RTE_DIM(priority_map_3): + res = priority_map_3[priority][subpriority]; + break; + case RTE_DIM(priority_map_5): + res = priority_map_5[priority][subpriority]; + break; + } + return res; +}