1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2016 6WIND S.A.
3 * Copyright 2016 Mellanox Technologies, Ltd
10 #include <sys/queue.h>
12 #include <rte_common.h>
13 #include <rte_ether.h>
14 #include <ethdev_driver.h>
15 #include <rte_eal_paging.h>
17 #include <rte_cycles.h>
18 #include <rte_flow_driver.h>
19 #include <rte_malloc.h>
22 #include <mlx5_glue.h>
23 #include <mlx5_devx_cmds.h>
25 #include <mlx5_malloc.h>
27 #include "mlx5_defs.h"
29 #include "mlx5_flow.h"
30 #include "mlx5_flow_os.h"
33 #include "mlx5_common_os.h"
34 #include "rte_pmd_mlx5.h"
36 struct tunnel_default_miss_ctx {
40 struct rte_flow_action_rss action_rss;
41 struct rte_flow_action_queue miss_queue;
42 struct rte_flow_action_jump miss_jump;
48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
49 struct rte_flow *flow,
50 const struct rte_flow_attr *attr,
51 const struct rte_flow_action *app_actions,
53 const struct mlx5_flow_tunnel *tunnel,
54 struct tunnel_default_miss_ctx *ctx,
55 struct rte_flow_error *error);
56 static struct mlx5_flow_tunnel *
57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
62 const struct mlx5_flow_tunnel *tunnel,
63 uint32_t group, uint32_t *table,
64 struct rte_flow_error *error);
66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void);
67 static void mlx5_flow_pop_thread_workspace(void);
70 /** Device flow drivers. */
71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
79 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops,
81 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
82 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
85 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
86 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
91 /** Node object of input graph for mlx5_flow_expand_rss(). */
92 struct mlx5_flow_expand_node {
93 const int *const next;
95 * List of next node indexes. Index 0 is interpreted as a terminator.
97 const enum rte_flow_item_type type;
98 /**< Pattern item type of current node. */
101 * RSS types bit-field associated with this node
102 * (see RTE_ETH_RSS_* definitions).
106 * Bit-fields that define how the node is used in the expansion.
107 * (see MLX5_EXPANSION_NODE_* definitions).
111 /* Optional expand field. The expansion alg will not go deeper. */
112 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0)
114 /* The node is not added implicitly as expansion to the flow pattern.
115 * If the node type does not match the flow pattern item type, the
116 * expansion alg will go deeper to its next items.
117 * In the current implementation, the list of next nodes indexes can
118 * have up to one node with this flag set and it has to be the last
119 * node index (before the list terminator).
121 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1)
123 /** Object returned by mlx5_flow_expand_rss(). */
124 struct mlx5_flow_expand_rss {
126 /**< Number of entries @p patterns and @p priorities. */
128 struct rte_flow_item *pattern; /**< Expanded pattern array. */
129 uint32_t priority; /**< Priority offset for each expansion. */
134 mlx5_dbg__print_pattern(const struct rte_flow_item *item);
136 static const struct mlx5_flow_expand_node *
137 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern,
138 unsigned int item_idx,
139 const struct mlx5_flow_expand_node graph[],
140 const struct mlx5_flow_expand_node *node);
143 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item)
145 switch (item->type) {
146 case RTE_FLOW_ITEM_TYPE_ETH:
147 case RTE_FLOW_ITEM_TYPE_VLAN:
148 case RTE_FLOW_ITEM_TYPE_IPV4:
149 case RTE_FLOW_ITEM_TYPE_IPV6:
150 case RTE_FLOW_ITEM_TYPE_UDP:
151 case RTE_FLOW_ITEM_TYPE_TCP:
152 case RTE_FLOW_ITEM_TYPE_VXLAN:
153 case RTE_FLOW_ITEM_TYPE_NVGRE:
154 case RTE_FLOW_ITEM_TYPE_GRE:
155 case RTE_FLOW_ITEM_TYPE_GENEVE:
156 case RTE_FLOW_ITEM_TYPE_MPLS:
157 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
158 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
159 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
160 case RTE_FLOW_ITEM_TYPE_GTP:
169 * Network Service Header (NSH) and its next protocol values
170 * are described in RFC-8393.
172 static enum rte_flow_item_type
173 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask)
175 enum rte_flow_item_type type;
177 switch (proto_mask & proto_spec) {
179 type = RTE_FLOW_ITEM_TYPE_VOID;
181 case RTE_VXLAN_GPE_TYPE_IPV4:
182 type = RTE_FLOW_ITEM_TYPE_IPV4;
184 case RTE_VXLAN_GPE_TYPE_IPV6:
185 type = RTE_VXLAN_GPE_TYPE_IPV6;
187 case RTE_VXLAN_GPE_TYPE_ETH:
188 type = RTE_FLOW_ITEM_TYPE_ETH;
191 type = RTE_FLOW_ITEM_TYPE_END;
196 static enum rte_flow_item_type
197 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask)
199 enum rte_flow_item_type type;
201 switch (proto_mask & proto_spec) {
203 type = RTE_FLOW_ITEM_TYPE_VOID;
206 type = RTE_FLOW_ITEM_TYPE_UDP;
209 type = RTE_FLOW_ITEM_TYPE_TCP;
212 type = RTE_FLOW_ITEM_TYPE_IPV4;
215 type = RTE_FLOW_ITEM_TYPE_IPV6;
218 type = RTE_FLOW_ITEM_TYPE_END;
223 static enum rte_flow_item_type
224 mlx5_ethertype_to_item_type(rte_be16_t type_spec,
225 rte_be16_t type_mask, bool is_tunnel)
227 enum rte_flow_item_type type;
229 switch (rte_be_to_cpu_16(type_spec & type_mask)) {
231 type = RTE_FLOW_ITEM_TYPE_VOID;
233 case RTE_ETHER_TYPE_TEB:
235 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END;
237 case RTE_ETHER_TYPE_VLAN:
239 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END;
241 case RTE_ETHER_TYPE_IPV4:
242 type = RTE_FLOW_ITEM_TYPE_IPV4;
244 case RTE_ETHER_TYPE_IPV6:
245 type = RTE_FLOW_ITEM_TYPE_IPV6;
248 type = RTE_FLOW_ITEM_TYPE_END;
253 static enum rte_flow_item_type
254 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
256 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \
258 const void *m = item->mask; \
259 const void *s = item->spec; \
261 ((const struct rte_flow_item_##type *)m)->fld : \
262 rte_flow_item_##type##_mask.fld; \
263 spec = ((const struct rte_flow_item_##type *)s)->fld; \
266 enum rte_flow_item_type ret;
269 if (item == NULL || item->spec == NULL)
270 return RTE_FLOW_ITEM_TYPE_VOID;
271 switch (item->type) {
272 case RTE_FLOW_ITEM_TYPE_ETH:
273 MLX5_XSET_ITEM_MASK_SPEC(eth, type);
275 return RTE_FLOW_ITEM_TYPE_VOID;
276 ret = mlx5_ethertype_to_item_type(spec, mask, false);
278 case RTE_FLOW_ITEM_TYPE_VLAN:
279 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type);
281 return RTE_FLOW_ITEM_TYPE_VOID;
282 ret = mlx5_ethertype_to_item_type(spec, mask, false);
284 case RTE_FLOW_ITEM_TYPE_IPV4:
285 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id);
287 return RTE_FLOW_ITEM_TYPE_VOID;
288 ret = mlx5_inet_proto_to_item_type(spec, mask);
290 case RTE_FLOW_ITEM_TYPE_IPV6:
291 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto);
293 return RTE_FLOW_ITEM_TYPE_VOID;
294 ret = mlx5_inet_proto_to_item_type(spec, mask);
296 case RTE_FLOW_ITEM_TYPE_GENEVE:
297 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol);
298 ret = mlx5_ethertype_to_item_type(spec, mask, true);
300 case RTE_FLOW_ITEM_TYPE_GRE:
301 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol);
302 ret = mlx5_ethertype_to_item_type(spec, mask, true);
304 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
305 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol);
306 ret = mlx5_nsh_proto_to_item_type(spec, mask);
309 ret = RTE_FLOW_ITEM_TYPE_VOID;
313 #undef MLX5_XSET_ITEM_MASK_SPEC
317 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[],
318 const int *next_node)
320 const struct mlx5_flow_expand_node *node = NULL;
321 const int *next = next_node;
323 while (next && *next) {
325 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT
326 * flag set, because they were not found in the flow pattern.
328 node = &graph[*next];
329 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT))
336 #define MLX5_RSS_EXP_ELT_N 16
339 * Expand RSS flows into several possible flows according to the RSS hash
340 * fields requested and the driver capabilities.
343 * Buffer to store the result expansion.
345 * Buffer size in bytes. If 0, @p buf can be NULL.
349 * RSS types to expand (see RTE_ETH_RSS_* definitions).
351 * Input graph to expand @p pattern according to @p types.
352 * @param[in] graph_root_index
353 * Index of root node in @p graph, typically 0.
356 * A positive value representing the size of @p buf in bytes regardless of
357 * @p size on success, a negative errno value otherwise and rte_errno is
358 * set, the following errors are defined:
360 * -E2BIG: graph-depth @p graph is too deep.
361 * -EINVAL: @p size has not enough space for expanded pattern.
364 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
365 const struct rte_flow_item *pattern, uint64_t types,
366 const struct mlx5_flow_expand_node graph[],
367 int graph_root_index)
369 const struct rte_flow_item *item;
370 const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
371 const int *next_node;
372 const int *stack[MLX5_RSS_EXP_ELT_N];
374 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N];
375 unsigned int i, item_idx, last_expand_item_idx = 0;
377 size_t user_pattern_size = 0;
379 const struct mlx5_flow_expand_node *next = NULL;
380 struct rte_flow_item missed_item;
383 const struct rte_flow_item *last_expand_item = NULL;
385 memset(&missed_item, 0, sizeof(missed_item));
386 lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
387 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]);
390 buf->entry[0].priority = 0;
391 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N];
393 addr = buf->entry[0].pattern;
394 for (item = pattern, item_idx = 0;
395 item->type != RTE_FLOW_ITEM_TYPE_END;
396 item++, item_idx++) {
397 if (!mlx5_flow_is_rss_expandable_item(item)) {
398 user_pattern_size += sizeof(*item);
401 last_expand_item = item;
402 last_expand_item_idx = item_idx;
404 while (node->next && node->next[i]) {
405 next = &graph[node->next[i]];
406 if (next->type == item->type)
408 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) {
417 user_pattern_size += sizeof(*item);
419 user_pattern_size += sizeof(*item); /* Handle END item. */
420 lsize += user_pattern_size;
423 /* Copy the user pattern in the first entry of the buffer. */
424 rte_memcpy(addr, pattern, user_pattern_size);
425 addr = (void *)(((uintptr_t)addr) + user_pattern_size);
427 /* Start expanding. */
428 memset(flow_items, 0, sizeof(flow_items));
429 user_pattern_size -= sizeof(*item);
431 * Check if the last valid item has spec set, need complete pattern,
432 * and the pattern can be used for expansion.
434 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item);
435 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
436 /* Item type END indicates expansion is not required. */
439 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
443 while (node->next && node->next[i]) {
444 next = &graph[node->next[i]];
445 if (next->type == missed_item.type) {
446 flow_items[0].type = missed_item.type;
447 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
450 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) {
459 if (next && missed) {
460 elt = 2; /* missed item + item end. */
462 lsize += elt * sizeof(*item) + user_pattern_size;
465 if (node->rss_types & types) {
466 buf->entry[buf->entries].priority = 1;
467 buf->entry[buf->entries].pattern = addr;
469 rte_memcpy(addr, buf->entry[0].pattern,
471 addr = (void *)(((uintptr_t)addr) + user_pattern_size);
472 rte_memcpy(addr, flow_items, elt * sizeof(*item));
473 addr = (void *)(((uintptr_t)addr) +
474 elt * sizeof(*item));
476 } else if (last_expand_item != NULL) {
477 node = mlx5_flow_expand_rss_adjust_node(pattern,
478 last_expand_item_idx, graph, node);
480 memset(flow_items, 0, sizeof(flow_items));
481 next_node = mlx5_flow_expand_rss_skip_explicit(graph,
483 stack[stack_pos] = next_node;
484 node = next_node ? &graph[*next_node] : NULL;
486 flow_items[stack_pos].type = node->type;
487 if (node->rss_types & types) {
490 * compute the number of items to copy from the
491 * expansion and copy it.
492 * When the stack_pos is 0, there are 1 element in it,
493 * plus the addition END item.
496 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
497 lsize += elt * sizeof(*item) + user_pattern_size;
500 n = elt * sizeof(*item);
501 buf->entry[buf->entries].priority =
502 stack_pos + 1 + missed;
503 buf->entry[buf->entries].pattern = addr;
505 rte_memcpy(addr, buf->entry[0].pattern,
507 addr = (void *)(((uintptr_t)addr) +
509 rte_memcpy(addr, &missed_item,
510 missed * sizeof(*item));
511 addr = (void *)(((uintptr_t)addr) +
512 missed * sizeof(*item));
513 rte_memcpy(addr, flow_items, n);
514 addr = (void *)(((uintptr_t)addr) + n);
517 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) &&
519 next_node = mlx5_flow_expand_rss_skip_explicit(graph,
521 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) {
525 stack[stack_pos] = next_node;
526 } else if (*(next_node + 1)) {
527 /* Follow up with the next possibility. */
528 next_node = mlx5_flow_expand_rss_skip_explicit(graph,
530 } else if (!stack_pos) {
532 * Completing the traverse over the different paths.
533 * The next_node is advanced to the terminator.
537 /* Move to the next path. */
539 next_node = stack[--stack_pos];
544 next_node = mlx5_flow_expand_rss_skip_explicit(graph,
546 stack[stack_pos] = next_node;
548 node = next_node && *next_node ? &graph[*next_node] : NULL;
553 enum mlx5_expansion {
555 MLX5_EXPANSION_ROOT_OUTER,
556 MLX5_EXPANSION_OUTER_ETH,
557 MLX5_EXPANSION_OUTER_VLAN,
558 MLX5_EXPANSION_OUTER_IPV4,
559 MLX5_EXPANSION_OUTER_IPV4_UDP,
560 MLX5_EXPANSION_OUTER_IPV4_TCP,
561 MLX5_EXPANSION_OUTER_IPV6,
562 MLX5_EXPANSION_OUTER_IPV6_UDP,
563 MLX5_EXPANSION_OUTER_IPV6_TCP,
564 MLX5_EXPANSION_VXLAN,
565 MLX5_EXPANSION_STD_VXLAN,
566 MLX5_EXPANSION_L3_VXLAN,
567 MLX5_EXPANSION_VXLAN_GPE,
569 MLX5_EXPANSION_NVGRE,
570 MLX5_EXPANSION_GRE_KEY,
575 MLX5_EXPANSION_IPV4_UDP,
576 MLX5_EXPANSION_IPV4_TCP,
578 MLX5_EXPANSION_IPV6_UDP,
579 MLX5_EXPANSION_IPV6_TCP,
580 MLX5_EXPANSION_IPV6_FRAG_EXT,
582 MLX5_EXPANSION_GENEVE,
585 /** Supported expansion of items. */
586 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
587 [MLX5_EXPANSION_ROOT] = {
588 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
590 MLX5_EXPANSION_IPV6),
591 .type = RTE_FLOW_ITEM_TYPE_END,
593 [MLX5_EXPANSION_ROOT_OUTER] = {
594 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
595 MLX5_EXPANSION_OUTER_IPV4,
596 MLX5_EXPANSION_OUTER_IPV6),
597 .type = RTE_FLOW_ITEM_TYPE_END,
599 [MLX5_EXPANSION_OUTER_ETH] = {
600 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
601 .type = RTE_FLOW_ITEM_TYPE_ETH,
604 [MLX5_EXPANSION_OUTER_VLAN] = {
605 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
606 MLX5_EXPANSION_OUTER_IPV6),
607 .type = RTE_FLOW_ITEM_TYPE_VLAN,
608 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
610 [MLX5_EXPANSION_OUTER_IPV4] = {
611 .next = MLX5_FLOW_EXPAND_RSS_NEXT
612 (MLX5_EXPANSION_OUTER_IPV4_UDP,
613 MLX5_EXPANSION_OUTER_IPV4_TCP,
615 MLX5_EXPANSION_NVGRE,
617 MLX5_EXPANSION_IPV6),
618 .type = RTE_FLOW_ITEM_TYPE_IPV4,
619 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 |
620 RTE_ETH_RSS_NONFRAG_IPV4_OTHER,
622 [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
623 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
624 MLX5_EXPANSION_VXLAN_GPE,
626 MLX5_EXPANSION_GENEVE,
628 .type = RTE_FLOW_ITEM_TYPE_UDP,
629 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP,
631 [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
632 .type = RTE_FLOW_ITEM_TYPE_TCP,
633 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP,
635 [MLX5_EXPANSION_OUTER_IPV6] = {
636 .next = MLX5_FLOW_EXPAND_RSS_NEXT
637 (MLX5_EXPANSION_OUTER_IPV6_UDP,
638 MLX5_EXPANSION_OUTER_IPV6_TCP,
642 MLX5_EXPANSION_NVGRE),
643 .type = RTE_FLOW_ITEM_TYPE_IPV6,
644 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 |
645 RTE_ETH_RSS_NONFRAG_IPV6_OTHER,
647 [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
648 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
649 MLX5_EXPANSION_VXLAN_GPE,
651 MLX5_EXPANSION_GENEVE,
653 .type = RTE_FLOW_ITEM_TYPE_UDP,
654 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP,
656 [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
657 .type = RTE_FLOW_ITEM_TYPE_TCP,
658 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP,
660 [MLX5_EXPANSION_VXLAN] = {
661 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
663 MLX5_EXPANSION_IPV6),
664 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
666 [MLX5_EXPANSION_STD_VXLAN] = {
667 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
668 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
670 [MLX5_EXPANSION_L3_VXLAN] = {
671 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
672 MLX5_EXPANSION_IPV6),
673 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
675 [MLX5_EXPANSION_VXLAN_GPE] = {
676 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
678 MLX5_EXPANSION_IPV6),
679 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
681 [MLX5_EXPANSION_GRE] = {
682 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
685 MLX5_EXPANSION_GRE_KEY,
686 MLX5_EXPANSION_MPLS),
687 .type = RTE_FLOW_ITEM_TYPE_GRE,
689 [MLX5_EXPANSION_GRE_KEY] = {
690 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
692 MLX5_EXPANSION_MPLS),
693 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY,
694 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
696 [MLX5_EXPANSION_NVGRE] = {
697 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
698 .type = RTE_FLOW_ITEM_TYPE_NVGRE,
700 [MLX5_EXPANSION_MPLS] = {
701 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
704 .type = RTE_FLOW_ITEM_TYPE_MPLS,
705 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
707 [MLX5_EXPANSION_ETH] = {
708 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
709 .type = RTE_FLOW_ITEM_TYPE_ETH,
711 [MLX5_EXPANSION_VLAN] = {
712 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
713 MLX5_EXPANSION_IPV6),
714 .type = RTE_FLOW_ITEM_TYPE_VLAN,
715 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
717 [MLX5_EXPANSION_IPV4] = {
718 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
719 MLX5_EXPANSION_IPV4_TCP),
720 .type = RTE_FLOW_ITEM_TYPE_IPV4,
721 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 |
722 RTE_ETH_RSS_NONFRAG_IPV4_OTHER,
724 [MLX5_EXPANSION_IPV4_UDP] = {
725 .type = RTE_FLOW_ITEM_TYPE_UDP,
726 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP,
728 [MLX5_EXPANSION_IPV4_TCP] = {
729 .type = RTE_FLOW_ITEM_TYPE_TCP,
730 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP,
732 [MLX5_EXPANSION_IPV6] = {
733 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
734 MLX5_EXPANSION_IPV6_TCP,
735 MLX5_EXPANSION_IPV6_FRAG_EXT),
736 .type = RTE_FLOW_ITEM_TYPE_IPV6,
737 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 |
738 RTE_ETH_RSS_NONFRAG_IPV6_OTHER,
740 [MLX5_EXPANSION_IPV6_UDP] = {
741 .type = RTE_FLOW_ITEM_TYPE_UDP,
742 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP,
744 [MLX5_EXPANSION_IPV6_TCP] = {
745 .type = RTE_FLOW_ITEM_TYPE_TCP,
746 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP,
748 [MLX5_EXPANSION_IPV6_FRAG_EXT] = {
749 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT,
751 [MLX5_EXPANSION_GTP] = {
752 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
753 MLX5_EXPANSION_IPV6),
754 .type = RTE_FLOW_ITEM_TYPE_GTP,
756 [MLX5_EXPANSION_GENEVE] = {
757 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
759 MLX5_EXPANSION_IPV6),
760 .type = RTE_FLOW_ITEM_TYPE_GENEVE,
764 static struct rte_flow_action_handle *
765 mlx5_action_handle_create(struct rte_eth_dev *dev,
766 const struct rte_flow_indir_action_conf *conf,
767 const struct rte_flow_action *action,
768 struct rte_flow_error *error);
769 static int mlx5_action_handle_destroy
770 (struct rte_eth_dev *dev,
771 struct rte_flow_action_handle *handle,
772 struct rte_flow_error *error);
773 static int mlx5_action_handle_update
774 (struct rte_eth_dev *dev,
775 struct rte_flow_action_handle *handle,
777 struct rte_flow_error *error);
778 static int mlx5_action_handle_query
779 (struct rte_eth_dev *dev,
780 const struct rte_flow_action_handle *handle,
782 struct rte_flow_error *error);
784 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
785 struct rte_flow_tunnel *app_tunnel,
786 struct rte_flow_action **actions,
787 uint32_t *num_of_actions,
788 struct rte_flow_error *error);
790 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
791 struct rte_flow_tunnel *app_tunnel,
792 struct rte_flow_item **items,
793 uint32_t *num_of_items,
794 struct rte_flow_error *error);
796 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
797 struct rte_flow_item *pmd_items,
798 uint32_t num_items, struct rte_flow_error *err);
800 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
801 struct rte_flow_action *pmd_actions,
802 uint32_t num_actions,
803 struct rte_flow_error *err);
805 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
807 struct rte_flow_restore_info *info,
808 struct rte_flow_error *err);
809 static struct rte_flow_item_flex_handle *
810 mlx5_flow_flex_item_create(struct rte_eth_dev *dev,
811 const struct rte_flow_item_flex_conf *conf,
812 struct rte_flow_error *error);
814 mlx5_flow_flex_item_release(struct rte_eth_dev *dev,
815 const struct rte_flow_item_flex_handle *handle,
816 struct rte_flow_error *error);
818 mlx5_flow_info_get(struct rte_eth_dev *dev,
819 struct rte_flow_port_info *port_info,
820 struct rte_flow_queue_info *queue_info,
821 struct rte_flow_error *error);
823 mlx5_flow_port_configure(struct rte_eth_dev *dev,
824 const struct rte_flow_port_attr *port_attr,
826 const struct rte_flow_queue_attr *queue_attr[],
827 struct rte_flow_error *err);
829 static struct rte_flow_pattern_template *
830 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev,
831 const struct rte_flow_pattern_template_attr *attr,
832 const struct rte_flow_item items[],
833 struct rte_flow_error *error);
836 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev,
837 struct rte_flow_pattern_template *template,
838 struct rte_flow_error *error);
839 static struct rte_flow_actions_template *
840 mlx5_flow_actions_template_create(struct rte_eth_dev *dev,
841 const struct rte_flow_actions_template_attr *attr,
842 const struct rte_flow_action actions[],
843 const struct rte_flow_action masks[],
844 struct rte_flow_error *error);
846 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev,
847 struct rte_flow_actions_template *template,
848 struct rte_flow_error *error);
850 static struct rte_flow_template_table *
851 mlx5_flow_table_create(struct rte_eth_dev *dev,
852 const struct rte_flow_template_table_attr *attr,
853 struct rte_flow_pattern_template *item_templates[],
854 uint8_t nb_item_templates,
855 struct rte_flow_actions_template *action_templates[],
856 uint8_t nb_action_templates,
857 struct rte_flow_error *error);
859 mlx5_flow_table_destroy(struct rte_eth_dev *dev,
860 struct rte_flow_template_table *table,
861 struct rte_flow_error *error);
862 static struct rte_flow *
863 mlx5_flow_async_flow_create(struct rte_eth_dev *dev,
865 const struct rte_flow_op_attr *attr,
866 struct rte_flow_template_table *table,
867 const struct rte_flow_item items[],
868 uint8_t pattern_template_index,
869 const struct rte_flow_action actions[],
870 uint8_t action_template_index,
872 struct rte_flow_error *error);
874 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev,
876 const struct rte_flow_op_attr *attr,
877 struct rte_flow *flow,
879 struct rte_flow_error *error);
881 mlx5_flow_pull(struct rte_eth_dev *dev,
883 struct rte_flow_op_result res[],
885 struct rte_flow_error *error);
887 mlx5_flow_push(struct rte_eth_dev *dev,
889 struct rte_flow_error *error);
891 static struct rte_flow_action_handle *
892 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue,
893 const struct rte_flow_op_attr *attr,
894 const struct rte_flow_indir_action_conf *conf,
895 const struct rte_flow_action *action,
897 struct rte_flow_error *error);
900 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue,
901 const struct rte_flow_op_attr *attr,
902 struct rte_flow_action_handle *handle,
905 struct rte_flow_error *error);
908 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue,
909 const struct rte_flow_op_attr *attr,
910 struct rte_flow_action_handle *handle,
912 struct rte_flow_error *error);
914 static const struct rte_flow_ops mlx5_flow_ops = {
915 .validate = mlx5_flow_validate,
916 .create = mlx5_flow_create,
917 .destroy = mlx5_flow_destroy,
918 .flush = mlx5_flow_flush,
919 .isolate = mlx5_flow_isolate,
920 .query = mlx5_flow_query,
921 .dev_dump = mlx5_flow_dev_dump,
922 .get_aged_flows = mlx5_flow_get_aged_flows,
923 .action_handle_create = mlx5_action_handle_create,
924 .action_handle_destroy = mlx5_action_handle_destroy,
925 .action_handle_update = mlx5_action_handle_update,
926 .action_handle_query = mlx5_action_handle_query,
927 .tunnel_decap_set = mlx5_flow_tunnel_decap_set,
928 .tunnel_match = mlx5_flow_tunnel_match,
929 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release,
930 .tunnel_item_release = mlx5_flow_tunnel_item_release,
931 .get_restore_info = mlx5_flow_tunnel_get_restore_info,
932 .flex_item_create = mlx5_flow_flex_item_create,
933 .flex_item_release = mlx5_flow_flex_item_release,
934 .info_get = mlx5_flow_info_get,
935 .configure = mlx5_flow_port_configure,
936 .pattern_template_create = mlx5_flow_pattern_template_create,
937 .pattern_template_destroy = mlx5_flow_pattern_template_destroy,
938 .actions_template_create = mlx5_flow_actions_template_create,
939 .actions_template_destroy = mlx5_flow_actions_template_destroy,
940 .template_table_create = mlx5_flow_table_create,
941 .template_table_destroy = mlx5_flow_table_destroy,
942 .async_create = mlx5_flow_async_flow_create,
943 .async_destroy = mlx5_flow_async_flow_destroy,
944 .pull = mlx5_flow_pull,
945 .push = mlx5_flow_push,
946 .async_action_handle_create = mlx5_flow_async_action_handle_create,
947 .async_action_handle_update = mlx5_flow_async_action_handle_update,
948 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy,
951 /* Tunnel information. */
952 struct mlx5_flow_tunnel_info {
953 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
954 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
957 static struct mlx5_flow_tunnel_info tunnels_info[] = {
959 .tunnel = MLX5_FLOW_LAYER_VXLAN,
960 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
963 .tunnel = MLX5_FLOW_LAYER_GENEVE,
964 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
967 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
968 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
971 .tunnel = MLX5_FLOW_LAYER_GRE,
972 .ptype = RTE_PTYPE_TUNNEL_GRE,
975 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
976 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
979 .tunnel = MLX5_FLOW_LAYER_MPLS,
980 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
983 .tunnel = MLX5_FLOW_LAYER_NVGRE,
984 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
987 .tunnel = MLX5_FLOW_LAYER_IPIP,
988 .ptype = RTE_PTYPE_TUNNEL_IP,
991 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
992 .ptype = RTE_PTYPE_TUNNEL_IP,
995 .tunnel = MLX5_FLOW_LAYER_GTP,
996 .ptype = RTE_PTYPE_TUNNEL_GTPU,
1003 * Translate tag ID to register.
1006 * Pointer to the Ethernet device structure.
1007 * @param[in] feature
1008 * The feature that request the register.
1010 * The request register ID.
1012 * Error description in case of any.
1015 * The request register on success, a negative errno
1016 * value otherwise and rte_errno is set.
1019 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
1020 enum mlx5_feature_name feature,
1022 struct rte_flow_error *error)
1024 struct mlx5_priv *priv = dev->data->dev_private;
1025 struct mlx5_sh_config *config = &priv->sh->config;
1026 enum modify_reg start_reg;
1027 bool skip_mtr_reg = false;
1030 case MLX5_HAIRPIN_RX:
1032 case MLX5_HAIRPIN_TX:
1034 case MLX5_METADATA_RX:
1035 switch (config->dv_xmeta_en) {
1036 case MLX5_XMETA_MODE_LEGACY:
1038 case MLX5_XMETA_MODE_META16:
1040 case MLX5_XMETA_MODE_META32:
1044 case MLX5_METADATA_TX:
1046 case MLX5_METADATA_FDB:
1047 switch (config->dv_xmeta_en) {
1048 case MLX5_XMETA_MODE_LEGACY:
1050 case MLX5_XMETA_MODE_META16:
1052 case MLX5_XMETA_MODE_META32:
1056 case MLX5_FLOW_MARK:
1057 switch (config->dv_xmeta_en) {
1058 case MLX5_XMETA_MODE_LEGACY:
1060 case MLX5_XMETA_MODE_META16:
1062 case MLX5_XMETA_MODE_META32:
1068 * If meter color and meter id share one register, flow match
1069 * should use the meter color register for match.
1071 if (priv->mtr_reg_share)
1072 return priv->mtr_color_reg;
1074 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
1076 case MLX5_MTR_COLOR:
1077 case MLX5_ASO_FLOW_HIT:
1078 case MLX5_ASO_CONNTRACK:
1079 case MLX5_SAMPLE_ID:
1080 /* All features use the same REG_C. */
1081 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
1082 return priv->mtr_color_reg;
1083 case MLX5_COPY_MARK:
1085 * Metadata COPY_MARK register using is in meter suffix sub
1086 * flow while with meter. It's safe to share the same register.
1088 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
1091 * If meter is enable, it will engage the register for color
1092 * match and flow match. If meter color match is not using the
1093 * REG_C_2, need to skip the REG_C_x be used by meter color
1095 * If meter is disable, free to use all available registers.
1097 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
1098 (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
1099 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
1100 if (id > (uint32_t)(REG_C_7 - start_reg))
1101 return rte_flow_error_set(error, EINVAL,
1102 RTE_FLOW_ERROR_TYPE_ITEM,
1103 NULL, "invalid tag id");
1104 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
1105 return rte_flow_error_set(error, ENOTSUP,
1106 RTE_FLOW_ERROR_TYPE_ITEM,
1107 NULL, "unsupported tag id");
1109 * This case means meter is using the REG_C_x great than 2.
1110 * Take care not to conflict with meter color REG_C_x.
1111 * If the available index REG_C_y >= REG_C_x, skip the
1114 if (skip_mtr_reg && priv->sh->flow_mreg_c
1115 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
1116 if (id >= (uint32_t)(REG_C_7 - start_reg))
1117 return rte_flow_error_set(error, EINVAL,
1118 RTE_FLOW_ERROR_TYPE_ITEM,
1119 NULL, "invalid tag id");
1120 if (priv->sh->flow_mreg_c
1121 [id + 1 + start_reg - REG_C_0] != REG_NON)
1122 return priv->sh->flow_mreg_c
1123 [id + 1 + start_reg - REG_C_0];
1124 return rte_flow_error_set(error, ENOTSUP,
1125 RTE_FLOW_ERROR_TYPE_ITEM,
1126 NULL, "unsupported tag id");
1128 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0];
1131 return rte_flow_error_set(error, EINVAL,
1132 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1133 NULL, "invalid feature name");
1137 * Check extensive flow metadata register support.
1140 * Pointer to rte_eth_dev structure.
1143 * True if device supports extensive flow metadata register, otherwise false.
1146 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
1148 struct mlx5_priv *priv = dev->data->dev_private;
1151 * Having available reg_c can be regarded inclusively as supporting
1152 * extensive flow metadata register, which could mean,
1153 * - metadata register copy action by modify header.
1154 * - 16 modify header actions is supported.
1155 * - reg_c's are preserved across different domain (FDB and NIC) on
1156 * packet loopback by flow lookup miss.
1158 return priv->sh->flow_mreg_c[2] != REG_NON;
1162 * Get the lowest priority.
1165 * Pointer to the Ethernet device structure.
1166 * @param[in] attributes
1167 * Pointer to device flow rule attributes.
1170 * The value of lowest priority of flow.
1173 mlx5_get_lowest_priority(struct rte_eth_dev *dev,
1174 const struct rte_flow_attr *attr)
1176 struct mlx5_priv *priv = dev->data->dev_private;
1178 if (!attr->group && !attr->transfer)
1179 return priv->sh->flow_max_priority - 2;
1180 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1;
1184 * Calculate matcher priority of the flow.
1187 * Pointer to the Ethernet device structure.
1189 * Pointer to device flow rule attributes.
1190 * @param[in] subpriority
1191 * The priority based on the items.
1192 * @param[in] external
1193 * Flow is user flow.
1195 * The matcher priority of the flow.
1198 mlx5_get_matcher_priority(struct rte_eth_dev *dev,
1199 const struct rte_flow_attr *attr,
1200 uint32_t subpriority, bool external)
1202 uint16_t priority = (uint16_t)attr->priority;
1203 struct mlx5_priv *priv = dev->data->dev_private;
1205 if (!attr->group && !attr->transfer) {
1206 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
1207 priority = priv->sh->flow_max_priority - 1;
1208 return mlx5_os_flow_adjust_priority(dev, priority, subpriority);
1209 } else if (!external && attr->transfer && attr->group == 0 &&
1210 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) {
1211 return (priv->sh->flow_max_priority - 1) * 3;
1213 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
1214 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO;
1215 return priority * 3 + subpriority;
1219 * Verify the @p item specifications (spec, last, mask) are compatible with the
1223 * Item specification.
1225 * @p item->mask or flow default bit-masks.
1226 * @param[in] nic_mask
1227 * Bit-masks covering supported fields by the NIC to compare with user mask.
1229 * Bit-masks size in bytes.
1230 * @param[in] range_accepted
1231 * True if range of values is accepted for specific fields, false otherwise.
1233 * Pointer to error structure.
1236 * 0 on success, a negative errno value otherwise and rte_errno is set.
1239 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
1240 const uint8_t *mask,
1241 const uint8_t *nic_mask,
1243 bool range_accepted,
1244 struct rte_flow_error *error)
1248 MLX5_ASSERT(nic_mask);
1249 for (i = 0; i < size; ++i)
1250 if ((nic_mask[i] | mask[i]) != nic_mask[i])
1251 return rte_flow_error_set(error, ENOTSUP,
1252 RTE_FLOW_ERROR_TYPE_ITEM,
1254 "mask enables non supported"
1256 if (!item->spec && (item->mask || item->last))
1257 return rte_flow_error_set(error, EINVAL,
1258 RTE_FLOW_ERROR_TYPE_ITEM, item,
1259 "mask/last without a spec is not"
1261 if (item->spec && item->last && !range_accepted) {
1267 for (i = 0; i < size; ++i) {
1268 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
1269 last[i] = ((const uint8_t *)item->last)[i] & mask[i];
1271 ret = memcmp(spec, last, size);
1273 return rte_flow_error_set(error, EINVAL,
1274 RTE_FLOW_ERROR_TYPE_ITEM,
1276 "range is not valid");
1282 * Adjust the hash fields according to the @p flow information.
1284 * @param[in] dev_flow.
1285 * Pointer to the mlx5_flow.
1287 * 1 when the hash field is for a tunnel item.
1288 * @param[in] layer_types
1289 * RTE_ETH_RSS_* types.
1290 * @param[in] hash_fields
1294 * The hash fields that should be used.
1297 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
1298 int tunnel __rte_unused, uint64_t layer_types,
1299 uint64_t hash_fields)
1301 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1302 int rss_request_inner = rss_desc->level >= 2;
1304 /* Check RSS hash level for tunnel. */
1305 if (tunnel && rss_request_inner)
1306 hash_fields |= IBV_RX_HASH_INNER;
1307 else if (tunnel || rss_request_inner)
1310 /* Check if requested layer matches RSS hash fields. */
1311 if (!(rss_desc->types & layer_types))
1317 * Lookup and set the ptype in the data Rx part. A single Ptype can be used,
1318 * if several tunnel rules are used on this queue, the tunnel ptype will be
1322 * Rx queue to update.
1325 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
1328 uint32_t tunnel_ptype = 0;
1330 /* Look up for the ptype to use. */
1331 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
1332 if (!rxq_ctrl->flow_tunnels_n[i])
1334 if (!tunnel_ptype) {
1335 tunnel_ptype = tunnels_info[i].ptype;
1341 rxq_ctrl->rxq.tunnel = tunnel_ptype;
1345 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device
1349 * Pointer to the Ethernet device structure.
1350 * @param[in] dev_handle
1351 * Pointer to device flow handle structure.
1354 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
1355 struct mlx5_flow_handle *dev_handle)
1357 struct mlx5_priv *priv = dev->data->dev_private;
1358 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1359 struct mlx5_ind_table_obj *ind_tbl = NULL;
1362 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1363 struct mlx5_hrxq *hrxq;
1365 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1366 dev_handle->rix_hrxq);
1368 ind_tbl = hrxq->ind_table;
1369 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1370 struct mlx5_shared_action_rss *shared_rss;
1372 shared_rss = mlx5_ipool_get
1373 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1374 dev_handle->rix_srss);
1376 ind_tbl = shared_rss->ind_tbl;
1380 for (i = 0; i != ind_tbl->queues_n; ++i) {
1381 int idx = ind_tbl->queues[i];
1382 struct mlx5_rxq_ctrl *rxq_ctrl;
1384 if (mlx5_is_external_rxq(dev, idx))
1386 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx);
1387 MLX5_ASSERT(rxq_ctrl != NULL);
1388 if (rxq_ctrl == NULL)
1391 * To support metadata register copy on Tx loopback,
1392 * this must be always enabled (metadata may arive
1393 * from other port - not from local flows only.
1398 /* Increase the counter matching the flow. */
1399 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1400 if ((tunnels_info[j].tunnel &
1401 dev_handle->layers) ==
1402 tunnels_info[j].tunnel) {
1403 rxq_ctrl->flow_tunnels_n[j]++;
1407 flow_rxq_tunnel_ptype_update(rxq_ctrl);
1413 flow_rxq_mark_flag_set(struct rte_eth_dev *dev)
1415 struct mlx5_priv *priv = dev->data->dev_private;
1416 struct mlx5_rxq_ctrl *rxq_ctrl;
1418 if (priv->mark_enabled)
1420 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) {
1421 rxq_ctrl->rxq.mark = 1;
1423 priv->mark_enabled = 1;
1427 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1430 * Pointer to the Ethernet device structure.
1432 * Pointer to flow structure.
1435 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1437 struct mlx5_priv *priv = dev->data->dev_private;
1438 uint32_t handle_idx;
1439 struct mlx5_flow_handle *dev_handle;
1440 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
1444 flow_rxq_mark_flag_set(dev);
1445 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1446 handle_idx, dev_handle, next)
1447 flow_drv_rxq_flags_set(dev, dev_handle);
1451 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1452 * device flow if no other flow uses it with the same kind of request.
1455 * Pointer to Ethernet device.
1456 * @param[in] dev_handle
1457 * Pointer to the device flow handle structure.
1460 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1461 struct mlx5_flow_handle *dev_handle)
1463 struct mlx5_priv *priv = dev->data->dev_private;
1464 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1465 struct mlx5_ind_table_obj *ind_tbl = NULL;
1468 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1469 struct mlx5_hrxq *hrxq;
1471 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1472 dev_handle->rix_hrxq);
1474 ind_tbl = hrxq->ind_table;
1475 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1476 struct mlx5_shared_action_rss *shared_rss;
1478 shared_rss = mlx5_ipool_get
1479 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1480 dev_handle->rix_srss);
1482 ind_tbl = shared_rss->ind_tbl;
1486 MLX5_ASSERT(dev->data->dev_started);
1487 for (i = 0; i != ind_tbl->queues_n; ++i) {
1488 int idx = ind_tbl->queues[i];
1489 struct mlx5_rxq_ctrl *rxq_ctrl;
1491 if (mlx5_is_external_rxq(dev, idx))
1493 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx);
1494 MLX5_ASSERT(rxq_ctrl != NULL);
1495 if (rxq_ctrl == NULL)
1500 /* Decrease the counter matching the flow. */
1501 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1502 if ((tunnels_info[j].tunnel &
1503 dev_handle->layers) ==
1504 tunnels_info[j].tunnel) {
1505 rxq_ctrl->flow_tunnels_n[j]--;
1509 flow_rxq_tunnel_ptype_update(rxq_ctrl);
1515 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1516 * @p flow if no other flow uses it with the same kind of request.
1519 * Pointer to Ethernet device.
1521 * Pointer to the flow.
1524 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1526 struct mlx5_priv *priv = dev->data->dev_private;
1527 uint32_t handle_idx;
1528 struct mlx5_flow_handle *dev_handle;
1530 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1531 handle_idx, dev_handle, next)
1532 flow_drv_rxq_flags_trim(dev, dev_handle);
1536 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1539 * Pointer to Ethernet device.
1542 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1544 struct mlx5_priv *priv = dev->data->dev_private;
1547 for (i = 0; i != priv->rxqs_n; ++i) {
1548 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i);
1551 if (rxq == NULL || rxq->ctrl == NULL)
1553 rxq->ctrl->rxq.mark = 0;
1554 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1555 rxq->ctrl->flow_tunnels_n[j] = 0;
1556 rxq->ctrl->rxq.tunnel = 0;
1558 priv->mark_enabled = 0;
1562 * Set the Rx queue dynamic metadata (mask and offset) for a flow
1565 * Pointer to the Ethernet device structure.
1568 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1570 struct mlx5_priv *priv = dev->data->dev_private;
1573 for (i = 0; i != priv->rxqs_n; ++i) {
1574 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i);
1575 struct mlx5_rxq_data *data;
1577 if (rxq == NULL || rxq->ctrl == NULL)
1579 data = &rxq->ctrl->rxq;
1580 if (!rte_flow_dynf_metadata_avail()) {
1581 data->dynf_meta = 0;
1582 data->flow_meta_mask = 0;
1583 data->flow_meta_offset = -1;
1584 data->flow_meta_port_mask = 0;
1586 data->dynf_meta = 1;
1587 data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1588 data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1589 data->flow_meta_port_mask = priv->sh->dv_meta_mask;
1595 * return a pointer to the desired action in the list of actions.
1597 * @param[in] actions
1598 * The list of actions to search the action in.
1600 * The action to find.
1603 * Pointer to the action in the list, if found. NULL otherwise.
1605 const struct rte_flow_action *
1606 mlx5_flow_find_action(const struct rte_flow_action *actions,
1607 enum rte_flow_action_type action)
1609 if (actions == NULL)
1611 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1612 if (actions->type == action)
1618 * Validate the flag action.
1620 * @param[in] action_flags
1621 * Bit-fields that holds the actions detected until now.
1623 * Attributes of flow that includes this action.
1625 * Pointer to error structure.
1628 * 0 on success, a negative errno value otherwise and rte_errno is set.
1631 mlx5_flow_validate_action_flag(uint64_t action_flags,
1632 const struct rte_flow_attr *attr,
1633 struct rte_flow_error *error)
1635 if (action_flags & MLX5_FLOW_ACTION_MARK)
1636 return rte_flow_error_set(error, EINVAL,
1637 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1638 "can't mark and flag in same flow");
1639 if (action_flags & MLX5_FLOW_ACTION_FLAG)
1640 return rte_flow_error_set(error, EINVAL,
1641 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1643 " actions in same flow");
1645 return rte_flow_error_set(error, ENOTSUP,
1646 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1647 "flag action not supported for "
1653 * Validate the mark action.
1656 * Pointer to the queue action.
1657 * @param[in] action_flags
1658 * Bit-fields that holds the actions detected until now.
1660 * Attributes of flow that includes this action.
1662 * Pointer to error structure.
1665 * 0 on success, a negative errno value otherwise and rte_errno is set.
1668 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1669 uint64_t action_flags,
1670 const struct rte_flow_attr *attr,
1671 struct rte_flow_error *error)
1673 const struct rte_flow_action_mark *mark = action->conf;
1676 return rte_flow_error_set(error, EINVAL,
1677 RTE_FLOW_ERROR_TYPE_ACTION,
1679 "configuration cannot be null");
1680 if (mark->id >= MLX5_FLOW_MARK_MAX)
1681 return rte_flow_error_set(error, EINVAL,
1682 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1684 "mark id must in 0 <= id < "
1685 RTE_STR(MLX5_FLOW_MARK_MAX));
1686 if (action_flags & MLX5_FLOW_ACTION_FLAG)
1687 return rte_flow_error_set(error, EINVAL,
1688 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1689 "can't flag and mark in same flow");
1690 if (action_flags & MLX5_FLOW_ACTION_MARK)
1691 return rte_flow_error_set(error, EINVAL,
1692 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1693 "can't have 2 mark actions in same"
1696 return rte_flow_error_set(error, ENOTSUP,
1697 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1698 "mark action not supported for "
1704 * Validate the drop action.
1706 * @param[in] action_flags
1707 * Bit-fields that holds the actions detected until now.
1709 * Attributes of flow that includes this action.
1711 * Pointer to error structure.
1714 * 0 on success, a negative errno value otherwise and rte_errno is set.
1717 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1718 const struct rte_flow_attr *attr,
1719 struct rte_flow_error *error)
1722 return rte_flow_error_set(error, ENOTSUP,
1723 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1724 "drop action not supported for "
1730 * Validate the queue action.
1733 * Pointer to the queue action.
1734 * @param[in] action_flags
1735 * Bit-fields that holds the actions detected until now.
1737 * Pointer to the Ethernet device structure.
1739 * Attributes of flow that includes this action.
1741 * Pointer to error structure.
1744 * 0 on success, a negative errno value otherwise and rte_errno is set.
1747 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1748 uint64_t action_flags,
1749 struct rte_eth_dev *dev,
1750 const struct rte_flow_attr *attr,
1751 struct rte_flow_error *error)
1753 struct mlx5_priv *priv = dev->data->dev_private;
1754 const struct rte_flow_action_queue *queue = action->conf;
1756 if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1757 return rte_flow_error_set(error, EINVAL,
1758 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1759 "can't have 2 fate actions in"
1762 return rte_flow_error_set(error, ENOTSUP,
1763 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1764 "queue action not supported for egress.");
1765 if (mlx5_is_external_rxq(dev, queue->index))
1768 return rte_flow_error_set(error, EINVAL,
1769 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1770 NULL, "No Rx queues configured");
1771 if (queue->index >= priv->rxqs_n)
1772 return rte_flow_error_set(error, EINVAL,
1773 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1775 "queue index out of range");
1776 if (mlx5_rxq_get(dev, queue->index) == NULL)
1777 return rte_flow_error_set(error, EINVAL,
1778 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1780 "queue is not configured");
1785 * Validate queue numbers for device RSS.
1788 * Configured device.
1790 * Array of queue numbers.
1791 * @param[in] queues_n
1792 * Size of the @p queues array.
1794 * On error, filled with a textual error description.
1795 * @param[out] queue_idx
1796 * On error, filled with an offending queue index in @p queues array.
1799 * 0 on success, a negative errno code on error.
1802 mlx5_validate_rss_queues(struct rte_eth_dev *dev,
1803 const uint16_t *queues, uint32_t queues_n,
1804 const char **error, uint32_t *queue_idx)
1806 const struct mlx5_priv *priv = dev->data->dev_private;
1807 bool is_hairpin = false;
1808 bool is_ext_rss = false;
1811 for (i = 0; i != queues_n; ++i) {
1812 struct mlx5_rxq_ctrl *rxq_ctrl;
1814 if (mlx5_is_external_rxq(dev, queues[0])) {
1819 *error = "Combining external and regular RSS queues is not supported";
1823 if (queues[i] >= priv->rxqs_n) {
1824 *error = "queue index out of range";
1828 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]);
1829 if (rxq_ctrl == NULL) {
1830 *error = "queue is not configured";
1834 if (i == 0 && rxq_ctrl->is_hairpin)
1836 if (is_hairpin != rxq_ctrl->is_hairpin) {
1837 *error = "combining hairpin and regular RSS queues is not supported";
1846 * Validate the rss action.
1849 * Pointer to the Ethernet device structure.
1851 * Pointer to the queue action.
1853 * Pointer to error structure.
1856 * 0 on success, a negative errno value otherwise and rte_errno is set.
1859 mlx5_validate_action_rss(struct rte_eth_dev *dev,
1860 const struct rte_flow_action *action,
1861 struct rte_flow_error *error)
1863 struct mlx5_priv *priv = dev->data->dev_private;
1864 const struct rte_flow_action_rss *rss = action->conf;
1866 const char *message;
1869 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1870 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1871 return rte_flow_error_set(error, ENOTSUP,
1872 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1874 "RSS hash function not supported");
1875 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1880 return rte_flow_error_set(error, ENOTSUP,
1881 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1883 "tunnel RSS is not supported");
1884 /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1885 if (rss->key_len == 0 && rss->key != NULL)
1886 return rte_flow_error_set(error, ENOTSUP,
1887 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1889 "RSS hash key length 0");
1890 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1891 return rte_flow_error_set(error, ENOTSUP,
1892 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1894 "RSS hash key too small");
1895 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1896 return rte_flow_error_set(error, ENOTSUP,
1897 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1899 "RSS hash key too large");
1900 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size)
1901 return rte_flow_error_set(error, ENOTSUP,
1902 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1904 "number of queues too large");
1905 if (rss->types & MLX5_RSS_HF_MASK)
1906 return rte_flow_error_set(error, ENOTSUP,
1907 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1909 "some RSS protocols are not"
1911 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) &&
1912 !(rss->types & RTE_ETH_RSS_IP))
1913 return rte_flow_error_set(error, EINVAL,
1914 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1915 "L3 partial RSS requested but L3 RSS"
1916 " type not specified");
1917 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) &&
1918 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP)))
1919 return rte_flow_error_set(error, EINVAL,
1920 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1921 "L4 partial RSS requested but L4 RSS"
1922 " type not specified");
1923 if (!priv->rxqs_n && priv->ext_rxqs == NULL)
1924 return rte_flow_error_set(error, EINVAL,
1925 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1926 NULL, "No Rx queues configured");
1927 if (!rss->queue_num)
1928 return rte_flow_error_set(error, EINVAL,
1929 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1930 NULL, "No queues configured");
1931 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num,
1932 &message, &queue_idx);
1934 return rte_flow_error_set(error, -ret,
1935 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1936 &rss->queue[queue_idx], message);
1942 * Validate the rss action.
1945 * Pointer to the queue action.
1946 * @param[in] action_flags
1947 * Bit-fields that holds the actions detected until now.
1949 * Pointer to the Ethernet device structure.
1951 * Attributes of flow that includes this action.
1952 * @param[in] item_flags
1953 * Items that were detected.
1955 * Pointer to error structure.
1958 * 0 on success, a negative errno value otherwise and rte_errno is set.
1961 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1962 uint64_t action_flags,
1963 struct rte_eth_dev *dev,
1964 const struct rte_flow_attr *attr,
1965 uint64_t item_flags,
1966 struct rte_flow_error *error)
1968 const struct rte_flow_action_rss *rss = action->conf;
1969 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1972 if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1973 return rte_flow_error_set(error, EINVAL,
1974 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1975 "can't have 2 fate actions"
1977 ret = mlx5_validate_action_rss(dev, action, error);
1981 return rte_flow_error_set(error, ENOTSUP,
1982 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1983 "rss action not supported for "
1985 if (rss->level > 1 && !tunnel)
1986 return rte_flow_error_set(error, EINVAL,
1987 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1988 "inner RSS is not supported for "
1989 "non-tunnel flows");
1990 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
1991 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
1992 return rte_flow_error_set(error, EINVAL,
1993 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1994 "RSS on eCPRI is not supported now");
1996 if ((item_flags & MLX5_FLOW_LAYER_MPLS) &&
1998 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) &&
2000 return rte_flow_error_set(error, EINVAL,
2001 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2002 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern");
2007 * Validate the default miss action.
2009 * @param[in] action_flags
2010 * Bit-fields that holds the actions detected until now.
2012 * Pointer to error structure.
2015 * 0 on success, a negative errno value otherwise and rte_errno is set.
2018 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
2019 const struct rte_flow_attr *attr,
2020 struct rte_flow_error *error)
2022 if (action_flags & MLX5_FLOW_FATE_ACTIONS)
2023 return rte_flow_error_set(error, EINVAL,
2024 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2025 "can't have 2 fate actions in"
2028 return rte_flow_error_set(error, ENOTSUP,
2029 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
2030 "default miss action not supported "
2033 return rte_flow_error_set(error, ENOTSUP,
2034 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
2035 "only group 0 is supported");
2037 return rte_flow_error_set(error, ENOTSUP,
2038 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
2039 NULL, "transfer is not supported");
2044 * Validate the count action.
2047 * Pointer to the Ethernet device structure.
2049 * Attributes of flow that includes this action.
2051 * Pointer to error structure.
2054 * 0 on success, a negative errno value otherwise and rte_errno is set.
2057 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
2058 const struct rte_flow_attr *attr,
2059 struct rte_flow_error *error)
2062 return rte_flow_error_set(error, ENOTSUP,
2063 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
2064 "count action not supported for "
2070 * Validate the ASO CT action.
2073 * Pointer to the Ethernet device structure.
2074 * @param[in] conntrack
2075 * Pointer to the CT action profile.
2077 * Pointer to error structure.
2080 * 0 on success, a negative errno value otherwise and rte_errno is set.
2083 mlx5_validate_action_ct(struct rte_eth_dev *dev,
2084 const struct rte_flow_action_conntrack *conntrack,
2085 struct rte_flow_error *error)
2089 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT)
2090 return rte_flow_error_set(error, EINVAL,
2091 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2092 "Invalid CT state");
2093 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST)
2094 return rte_flow_error_set(error, EINVAL,
2095 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2096 "Invalid last TCP packet flag");
2101 * Verify the @p attributes will be correctly understood by the NIC and store
2102 * them in the @p flow if everything is correct.
2105 * Pointer to the Ethernet device structure.
2106 * @param[in] attributes
2107 * Pointer to flow attributes
2109 * Pointer to error structure.
2112 * 0 on success, a negative errno value otherwise and rte_errno is set.
2115 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
2116 const struct rte_flow_attr *attributes,
2117 struct rte_flow_error *error)
2119 struct mlx5_priv *priv = dev->data->dev_private;
2120 uint32_t priority_max = priv->sh->flow_max_priority - 1;
2122 if (attributes->group)
2123 return rte_flow_error_set(error, ENOTSUP,
2124 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
2125 NULL, "groups is not supported");
2126 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
2127 attributes->priority >= priority_max)
2128 return rte_flow_error_set(error, ENOTSUP,
2129 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
2130 NULL, "priority out of range");
2131 if (attributes->egress)
2132 return rte_flow_error_set(error, ENOTSUP,
2133 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
2134 "egress is not supported");
2135 if (attributes->transfer && !priv->sh->config.dv_esw_en)
2136 return rte_flow_error_set(error, ENOTSUP,
2137 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
2138 NULL, "transfer is not supported");
2139 if (!attributes->ingress)
2140 return rte_flow_error_set(error, EINVAL,
2141 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
2143 "ingress attribute is mandatory");
2148 * Validate ICMP6 item.
2151 * Item specification.
2152 * @param[in] item_flags
2153 * Bit-fields that holds the items detected until now.
2154 * @param[in] ext_vlan_sup
2155 * Whether extended VLAN features are supported or not.
2157 * Pointer to error structure.
2160 * 0 on success, a negative errno value otherwise and rte_errno is set.
2163 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
2164 uint64_t item_flags,
2165 uint8_t target_protocol,
2166 struct rte_flow_error *error)
2168 const struct rte_flow_item_icmp6 *mask = item->mask;
2169 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2170 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
2171 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
2172 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2173 MLX5_FLOW_LAYER_OUTER_L4;
2176 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
2177 return rte_flow_error_set(error, EINVAL,
2178 RTE_FLOW_ERROR_TYPE_ITEM, item,
2179 "protocol filtering not compatible"
2180 " with ICMP6 layer");
2181 if (!(item_flags & l3m))
2182 return rte_flow_error_set(error, EINVAL,
2183 RTE_FLOW_ERROR_TYPE_ITEM, item,
2184 "IPv6 is mandatory to filter on"
2186 if (item_flags & l4m)
2187 return rte_flow_error_set(error, EINVAL,
2188 RTE_FLOW_ERROR_TYPE_ITEM, item,
2189 "multiple L4 layers not supported");
2191 mask = &rte_flow_item_icmp6_mask;
2192 ret = mlx5_flow_item_acceptable
2193 (item, (const uint8_t *)mask,
2194 (const uint8_t *)&rte_flow_item_icmp6_mask,
2195 sizeof(struct rte_flow_item_icmp6),
2196 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2203 * Validate ICMP item.
2206 * Item specification.
2207 * @param[in] item_flags
2208 * Bit-fields that holds the items detected until now.
2210 * Pointer to error structure.
2213 * 0 on success, a negative errno value otherwise and rte_errno is set.
2216 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
2217 uint64_t item_flags,
2218 uint8_t target_protocol,
2219 struct rte_flow_error *error)
2221 const struct rte_flow_item_icmp *mask = item->mask;
2222 const struct rte_flow_item_icmp nic_mask = {
2223 .hdr.icmp_type = 0xff,
2224 .hdr.icmp_code = 0xff,
2225 .hdr.icmp_ident = RTE_BE16(0xffff),
2226 .hdr.icmp_seq_nb = RTE_BE16(0xffff),
2228 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2229 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
2230 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
2231 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2232 MLX5_FLOW_LAYER_OUTER_L4;
2235 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
2236 return rte_flow_error_set(error, EINVAL,
2237 RTE_FLOW_ERROR_TYPE_ITEM, item,
2238 "protocol filtering not compatible"
2239 " with ICMP layer");
2240 if (!(item_flags & l3m))
2241 return rte_flow_error_set(error, EINVAL,
2242 RTE_FLOW_ERROR_TYPE_ITEM, item,
2243 "IPv4 is mandatory to filter"
2245 if (item_flags & l4m)
2246 return rte_flow_error_set(error, EINVAL,
2247 RTE_FLOW_ERROR_TYPE_ITEM, item,
2248 "multiple L4 layers not supported");
2251 ret = mlx5_flow_item_acceptable
2252 (item, (const uint8_t *)mask,
2253 (const uint8_t *)&nic_mask,
2254 sizeof(struct rte_flow_item_icmp),
2255 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2262 * Validate Ethernet item.
2265 * Item specification.
2266 * @param[in] item_flags
2267 * Bit-fields that holds the items detected until now.
2269 * Pointer to error structure.
2272 * 0 on success, a negative errno value otherwise and rte_errno is set.
2275 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
2276 uint64_t item_flags, bool ext_vlan_sup,
2277 struct rte_flow_error *error)
2279 const struct rte_flow_item_eth *mask = item->mask;
2280 const struct rte_flow_item_eth nic_mask = {
2281 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2282 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2283 .type = RTE_BE16(0xffff),
2284 .has_vlan = ext_vlan_sup ? 1 : 0,
2287 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2288 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
2289 MLX5_FLOW_LAYER_OUTER_L2;
2291 if (item_flags & ethm)
2292 return rte_flow_error_set(error, ENOTSUP,
2293 RTE_FLOW_ERROR_TYPE_ITEM, item,
2294 "multiple L2 layers not supported");
2295 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
2296 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
2297 return rte_flow_error_set(error, EINVAL,
2298 RTE_FLOW_ERROR_TYPE_ITEM, item,
2299 "L2 layer should not follow "
2301 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
2302 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
2303 return rte_flow_error_set(error, EINVAL,
2304 RTE_FLOW_ERROR_TYPE_ITEM, item,
2305 "L2 layer should not follow VLAN");
2306 if (item_flags & MLX5_FLOW_LAYER_GTP)
2307 return rte_flow_error_set(error, EINVAL,
2308 RTE_FLOW_ERROR_TYPE_ITEM, item,
2309 "L2 layer should not follow GTP");
2311 mask = &rte_flow_item_eth_mask;
2312 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2313 (const uint8_t *)&nic_mask,
2314 sizeof(struct rte_flow_item_eth),
2315 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2320 * Validate VLAN item.
2323 * Item specification.
2324 * @param[in] item_flags
2325 * Bit-fields that holds the items detected until now.
2327 * Ethernet device flow is being created on.
2329 * Pointer to error structure.
2332 * 0 on success, a negative errno value otherwise and rte_errno is set.
2335 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
2336 uint64_t item_flags,
2337 struct rte_eth_dev *dev,
2338 struct rte_flow_error *error)
2340 const struct rte_flow_item_vlan *spec = item->spec;
2341 const struct rte_flow_item_vlan *mask = item->mask;
2342 const struct rte_flow_item_vlan nic_mask = {
2343 .tci = RTE_BE16(UINT16_MAX),
2344 .inner_type = RTE_BE16(UINT16_MAX),
2346 uint16_t vlan_tag = 0;
2347 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2349 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2350 MLX5_FLOW_LAYER_INNER_L4) :
2351 (MLX5_FLOW_LAYER_OUTER_L3 |
2352 MLX5_FLOW_LAYER_OUTER_L4);
2353 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2354 MLX5_FLOW_LAYER_OUTER_VLAN;
2356 if (item_flags & vlanm)
2357 return rte_flow_error_set(error, EINVAL,
2358 RTE_FLOW_ERROR_TYPE_ITEM, item,
2359 "multiple VLAN layers not supported");
2360 else if ((item_flags & l34m) != 0)
2361 return rte_flow_error_set(error, EINVAL,
2362 RTE_FLOW_ERROR_TYPE_ITEM, item,
2363 "VLAN cannot follow L3/L4 layer");
2365 mask = &rte_flow_item_vlan_mask;
2366 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2367 (const uint8_t *)&nic_mask,
2368 sizeof(struct rte_flow_item_vlan),
2369 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2372 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2373 struct mlx5_priv *priv = dev->data->dev_private;
2375 if (priv->vmwa_context) {
2377 * Non-NULL context means we have a virtual machine
2378 * and SR-IOV enabled, we have to create VLAN interface
2379 * to make hypervisor to setup E-Switch vport
2380 * context correctly. We avoid creating the multiple
2381 * VLAN interfaces, so we cannot support VLAN tag mask.
2383 return rte_flow_error_set(error, EINVAL,
2384 RTE_FLOW_ERROR_TYPE_ITEM,
2386 "VLAN tag mask is not"
2387 " supported in virtual"
2392 vlan_tag = spec->tci;
2393 vlan_tag &= mask->tci;
2396 * From verbs perspective an empty VLAN is equivalent
2397 * to a packet without VLAN layer.
2400 return rte_flow_error_set(error, EINVAL,
2401 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2403 "VLAN cannot be empty");
2408 * Validate IPV4 item.
2411 * Item specification.
2412 * @param[in] item_flags
2413 * Bit-fields that holds the items detected until now.
2414 * @param[in] last_item
2415 * Previous validated item in the pattern items.
2416 * @param[in] ether_type
2417 * Type in the ethernet layer header (including dot1q).
2418 * @param[in] acc_mask
2419 * Acceptable mask, if NULL default internal default mask
2420 * will be used to check whether item fields are supported.
2421 * @param[in] range_accepted
2422 * True if range of values is accepted for specific fields, false otherwise.
2424 * Pointer to error structure.
2427 * 0 on success, a negative errno value otherwise and rte_errno is set.
2430 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
2431 uint64_t item_flags,
2433 uint16_t ether_type,
2434 const struct rte_flow_item_ipv4 *acc_mask,
2435 bool range_accepted,
2436 struct rte_flow_error *error)
2438 const struct rte_flow_item_ipv4 *mask = item->mask;
2439 const struct rte_flow_item_ipv4 *spec = item->spec;
2440 const struct rte_flow_item_ipv4 nic_mask = {
2442 .src_addr = RTE_BE32(0xffffffff),
2443 .dst_addr = RTE_BE32(0xffffffff),
2444 .type_of_service = 0xff,
2445 .next_proto_id = 0xff,
2448 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2449 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2450 MLX5_FLOW_LAYER_OUTER_L3;
2451 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2452 MLX5_FLOW_LAYER_OUTER_L4;
2454 uint8_t next_proto = 0xFF;
2455 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2456 MLX5_FLOW_LAYER_OUTER_VLAN |
2457 MLX5_FLOW_LAYER_INNER_VLAN);
2459 if ((last_item & l2_vlan) && ether_type &&
2460 ether_type != RTE_ETHER_TYPE_IPV4)
2461 return rte_flow_error_set(error, EINVAL,
2462 RTE_FLOW_ERROR_TYPE_ITEM, item,
2463 "IPv4 cannot follow L2/VLAN layer "
2464 "which ether type is not IPv4");
2465 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2467 next_proto = mask->hdr.next_proto_id &
2468 spec->hdr.next_proto_id;
2469 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2470 return rte_flow_error_set(error, EINVAL,
2471 RTE_FLOW_ERROR_TYPE_ITEM,
2476 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
2477 return rte_flow_error_set(error, EINVAL,
2478 RTE_FLOW_ERROR_TYPE_ITEM, item,
2479 "wrong tunnel type - IPv6 specified "
2480 "but IPv4 item provided");
2481 if (item_flags & l3m)
2482 return rte_flow_error_set(error, ENOTSUP,
2483 RTE_FLOW_ERROR_TYPE_ITEM, item,
2484 "multiple L3 layers not supported");
2485 else if (item_flags & l4m)
2486 return rte_flow_error_set(error, EINVAL,
2487 RTE_FLOW_ERROR_TYPE_ITEM, item,
2488 "L3 cannot follow an L4 layer.");
2489 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2490 !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2491 return rte_flow_error_set(error, EINVAL,
2492 RTE_FLOW_ERROR_TYPE_ITEM, item,
2493 "L3 cannot follow an NVGRE layer.");
2495 mask = &rte_flow_item_ipv4_mask;
2496 else if (mask->hdr.next_proto_id != 0 &&
2497 mask->hdr.next_proto_id != 0xff)
2498 return rte_flow_error_set(error, EINVAL,
2499 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2500 "partial mask is not supported"
2502 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2503 acc_mask ? (const uint8_t *)acc_mask
2504 : (const uint8_t *)&nic_mask,
2505 sizeof(struct rte_flow_item_ipv4),
2506 range_accepted, error);
2513 * Validate IPV6 item.
2516 * Item specification.
2517 * @param[in] item_flags
2518 * Bit-fields that holds the items detected until now.
2519 * @param[in] last_item
2520 * Previous validated item in the pattern items.
2521 * @param[in] ether_type
2522 * Type in the ethernet layer header (including dot1q).
2523 * @param[in] acc_mask
2524 * Acceptable mask, if NULL default internal default mask
2525 * will be used to check whether item fields are supported.
2527 * Pointer to error structure.
2530 * 0 on success, a negative errno value otherwise and rte_errno is set.
2533 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2534 uint64_t item_flags,
2536 uint16_t ether_type,
2537 const struct rte_flow_item_ipv6 *acc_mask,
2538 struct rte_flow_error *error)
2540 const struct rte_flow_item_ipv6 *mask = item->mask;
2541 const struct rte_flow_item_ipv6 *spec = item->spec;
2542 const struct rte_flow_item_ipv6 nic_mask = {
2545 "\xff\xff\xff\xff\xff\xff\xff\xff"
2546 "\xff\xff\xff\xff\xff\xff\xff\xff",
2548 "\xff\xff\xff\xff\xff\xff\xff\xff"
2549 "\xff\xff\xff\xff\xff\xff\xff\xff",
2550 .vtc_flow = RTE_BE32(0xffffffff),
2554 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2555 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2556 MLX5_FLOW_LAYER_OUTER_L3;
2557 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2558 MLX5_FLOW_LAYER_OUTER_L4;
2560 uint8_t next_proto = 0xFF;
2561 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2562 MLX5_FLOW_LAYER_OUTER_VLAN |
2563 MLX5_FLOW_LAYER_INNER_VLAN);
2565 if ((last_item & l2_vlan) && ether_type &&
2566 ether_type != RTE_ETHER_TYPE_IPV6)
2567 return rte_flow_error_set(error, EINVAL,
2568 RTE_FLOW_ERROR_TYPE_ITEM, item,
2569 "IPv6 cannot follow L2/VLAN layer "
2570 "which ether type is not IPv6");
2571 if (mask && mask->hdr.proto == UINT8_MAX && spec)
2572 next_proto = spec->hdr.proto;
2573 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2574 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2575 return rte_flow_error_set(error, EINVAL,
2576 RTE_FLOW_ERROR_TYPE_ITEM,
2581 if (next_proto == IPPROTO_HOPOPTS ||
2582 next_proto == IPPROTO_ROUTING ||
2583 next_proto == IPPROTO_FRAGMENT ||
2584 next_proto == IPPROTO_ESP ||
2585 next_proto == IPPROTO_AH ||
2586 next_proto == IPPROTO_DSTOPTS)
2587 return rte_flow_error_set(error, EINVAL,
2588 RTE_FLOW_ERROR_TYPE_ITEM, item,
2589 "IPv6 proto (next header) should "
2590 "not be set as extension header");
2591 if (item_flags & MLX5_FLOW_LAYER_IPIP)
2592 return rte_flow_error_set(error, EINVAL,
2593 RTE_FLOW_ERROR_TYPE_ITEM, item,
2594 "wrong tunnel type - IPv4 specified "
2595 "but IPv6 item provided");
2596 if (item_flags & l3m)
2597 return rte_flow_error_set(error, ENOTSUP,
2598 RTE_FLOW_ERROR_TYPE_ITEM, item,
2599 "multiple L3 layers not supported");
2600 else if (item_flags & l4m)
2601 return rte_flow_error_set(error, EINVAL,
2602 RTE_FLOW_ERROR_TYPE_ITEM, item,
2603 "L3 cannot follow an L4 layer.");
2604 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2605 !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2606 return rte_flow_error_set(error, EINVAL,
2607 RTE_FLOW_ERROR_TYPE_ITEM, item,
2608 "L3 cannot follow an NVGRE layer.");
2610 mask = &rte_flow_item_ipv6_mask;
2611 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2612 acc_mask ? (const uint8_t *)acc_mask
2613 : (const uint8_t *)&nic_mask,
2614 sizeof(struct rte_flow_item_ipv6),
2615 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2622 * Validate UDP item.
2625 * Item specification.
2626 * @param[in] item_flags
2627 * Bit-fields that holds the items detected until now.
2628 * @param[in] target_protocol
2629 * The next protocol in the previous item.
2630 * @param[in] flow_mask
2631 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2633 * Pointer to error structure.
2636 * 0 on success, a negative errno value otherwise and rte_errno is set.
2639 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2640 uint64_t item_flags,
2641 uint8_t target_protocol,
2642 struct rte_flow_error *error)
2644 const struct rte_flow_item_udp *mask = item->mask;
2645 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2646 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2647 MLX5_FLOW_LAYER_OUTER_L3;
2648 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2649 MLX5_FLOW_LAYER_OUTER_L4;
2652 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2653 return rte_flow_error_set(error, EINVAL,
2654 RTE_FLOW_ERROR_TYPE_ITEM, item,
2655 "protocol filtering not compatible"
2657 if (!(item_flags & l3m))
2658 return rte_flow_error_set(error, EINVAL,
2659 RTE_FLOW_ERROR_TYPE_ITEM, item,
2660 "L3 is mandatory to filter on L4");
2661 if (item_flags & l4m)
2662 return rte_flow_error_set(error, EINVAL,
2663 RTE_FLOW_ERROR_TYPE_ITEM, item,
2664 "multiple L4 layers not supported");
2666 mask = &rte_flow_item_udp_mask;
2667 ret = mlx5_flow_item_acceptable
2668 (item, (const uint8_t *)mask,
2669 (const uint8_t *)&rte_flow_item_udp_mask,
2670 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2678 * Validate TCP item.
2681 * Item specification.
2682 * @param[in] item_flags
2683 * Bit-fields that holds the items detected until now.
2684 * @param[in] target_protocol
2685 * The next protocol in the previous item.
2687 * Pointer to error structure.
2690 * 0 on success, a negative errno value otherwise and rte_errno is set.
2693 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2694 uint64_t item_flags,
2695 uint8_t target_protocol,
2696 const struct rte_flow_item_tcp *flow_mask,
2697 struct rte_flow_error *error)
2699 const struct rte_flow_item_tcp *mask = item->mask;
2700 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2701 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2702 MLX5_FLOW_LAYER_OUTER_L3;
2703 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2704 MLX5_FLOW_LAYER_OUTER_L4;
2707 MLX5_ASSERT(flow_mask);
2708 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2709 return rte_flow_error_set(error, EINVAL,
2710 RTE_FLOW_ERROR_TYPE_ITEM, item,
2711 "protocol filtering not compatible"
2713 if (!(item_flags & l3m))
2714 return rte_flow_error_set(error, EINVAL,
2715 RTE_FLOW_ERROR_TYPE_ITEM, item,
2716 "L3 is mandatory to filter on L4");
2717 if (item_flags & l4m)
2718 return rte_flow_error_set(error, EINVAL,
2719 RTE_FLOW_ERROR_TYPE_ITEM, item,
2720 "multiple L4 layers not supported");
2722 mask = &rte_flow_item_tcp_mask;
2723 ret = mlx5_flow_item_acceptable
2724 (item, (const uint8_t *)mask,
2725 (const uint8_t *)flow_mask,
2726 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2734 * Validate VXLAN item.
2737 * Pointer to the Ethernet device structure.
2738 * @param[in] udp_dport
2739 * UDP destination port
2741 * Item specification.
2742 * @param[in] item_flags
2743 * Bit-fields that holds the items detected until now.
2745 * Flow rule attributes.
2747 * Pointer to error structure.
2750 * 0 on success, a negative errno value otherwise and rte_errno is set.
2753 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev,
2755 const struct rte_flow_item *item,
2756 uint64_t item_flags,
2757 const struct rte_flow_attr *attr,
2758 struct rte_flow_error *error)
2760 const struct rte_flow_item_vxlan *spec = item->spec;
2761 const struct rte_flow_item_vxlan *mask = item->mask;
2763 struct mlx5_priv *priv = dev->data->dev_private;
2767 } id = { .vlan_id = 0, };
2768 const struct rte_flow_item_vxlan nic_mask = {
2769 .vni = "\xff\xff\xff",
2772 const struct rte_flow_item_vxlan *valid_mask;
2774 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2775 return rte_flow_error_set(error, ENOTSUP,
2776 RTE_FLOW_ERROR_TYPE_ITEM, item,
2777 "multiple tunnel layers not"
2779 valid_mask = &rte_flow_item_vxlan_mask;
2781 * Verify only UDPv4 is present as defined in
2782 * https://tools.ietf.org/html/rfc7348
2784 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2785 return rte_flow_error_set(error, EINVAL,
2786 RTE_FLOW_ERROR_TYPE_ITEM, item,
2787 "no outer UDP layer found");
2789 mask = &rte_flow_item_vxlan_mask;
2791 if (priv->sh->steering_format_version !=
2792 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 ||
2793 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) {
2794 /* FDB domain & NIC domain non-zero group */
2795 if ((attr->transfer || attr->group) && priv->sh->misc5_cap)
2796 valid_mask = &nic_mask;
2797 /* Group zero in NIC domain */
2798 if (!attr->group && !attr->transfer &&
2799 priv->sh->tunnel_header_0_1)
2800 valid_mask = &nic_mask;
2802 ret = mlx5_flow_item_acceptable
2803 (item, (const uint8_t *)mask,
2804 (const uint8_t *)valid_mask,
2805 sizeof(struct rte_flow_item_vxlan),
2806 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2810 memcpy(&id.vni[1], spec->vni, 3);
2811 memcpy(&id.vni[1], mask->vni, 3);
2813 if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2814 return rte_flow_error_set(error, ENOTSUP,
2815 RTE_FLOW_ERROR_TYPE_ITEM, item,
2816 "VXLAN tunnel must be fully defined");
2821 * Validate VXLAN_GPE item.
2824 * Item specification.
2825 * @param[in] item_flags
2826 * Bit-fields that holds the items detected until now.
2828 * Pointer to the private data structure.
2829 * @param[in] target_protocol
2830 * The next protocol in the previous item.
2832 * Pointer to error structure.
2835 * 0 on success, a negative errno value otherwise and rte_errno is set.
2838 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2839 uint64_t item_flags,
2840 struct rte_eth_dev *dev,
2841 struct rte_flow_error *error)
2843 struct mlx5_priv *priv = dev->data->dev_private;
2844 const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2845 const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2850 } id = { .vlan_id = 0, };
2852 if (!priv->sh->config.l3_vxlan_en)
2853 return rte_flow_error_set(error, ENOTSUP,
2854 RTE_FLOW_ERROR_TYPE_ITEM, item,
2855 "L3 VXLAN is not enabled by device"
2856 " parameter and/or not configured in"
2858 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2859 return rte_flow_error_set(error, ENOTSUP,
2860 RTE_FLOW_ERROR_TYPE_ITEM, item,
2861 "multiple tunnel layers not"
2864 * Verify only UDPv4 is present as defined in
2865 * https://tools.ietf.org/html/rfc7348
2867 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2868 return rte_flow_error_set(error, EINVAL,
2869 RTE_FLOW_ERROR_TYPE_ITEM, item,
2870 "no outer UDP layer found");
2872 mask = &rte_flow_item_vxlan_gpe_mask;
2873 ret = mlx5_flow_item_acceptable
2874 (item, (const uint8_t *)mask,
2875 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2876 sizeof(struct rte_flow_item_vxlan_gpe),
2877 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2882 return rte_flow_error_set(error, ENOTSUP,
2883 RTE_FLOW_ERROR_TYPE_ITEM,
2885 "VxLAN-GPE protocol"
2887 memcpy(&id.vni[1], spec->vni, 3);
2888 memcpy(&id.vni[1], mask->vni, 3);
2890 if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2891 return rte_flow_error_set(error, ENOTSUP,
2892 RTE_FLOW_ERROR_TYPE_ITEM, item,
2893 "VXLAN-GPE tunnel must be fully"
2898 * Validate GRE Key item.
2901 * Item specification.
2902 * @param[in] item_flags
2903 * Bit flags to mark detected items.
2904 * @param[in] gre_item
2905 * Pointer to gre_item
2907 * Pointer to error structure.
2910 * 0 on success, a negative errno value otherwise and rte_errno is set.
2913 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2914 uint64_t item_flags,
2915 const struct rte_flow_item *gre_item,
2916 struct rte_flow_error *error)
2918 const rte_be32_t *mask = item->mask;
2920 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2921 const struct rte_flow_item_gre *gre_spec;
2922 const struct rte_flow_item_gre *gre_mask;
2924 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2925 return rte_flow_error_set(error, ENOTSUP,
2926 RTE_FLOW_ERROR_TYPE_ITEM, item,
2927 "Multiple GRE key not support");
2928 if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2929 return rte_flow_error_set(error, ENOTSUP,
2930 RTE_FLOW_ERROR_TYPE_ITEM, item,
2931 "No preceding GRE header");
2932 if (item_flags & MLX5_FLOW_LAYER_INNER)
2933 return rte_flow_error_set(error, ENOTSUP,
2934 RTE_FLOW_ERROR_TYPE_ITEM, item,
2935 "GRE key following a wrong item");
2936 gre_mask = gre_item->mask;
2938 gre_mask = &rte_flow_item_gre_mask;
2939 gre_spec = gre_item->spec;
2940 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2941 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2942 return rte_flow_error_set(error, EINVAL,
2943 RTE_FLOW_ERROR_TYPE_ITEM, item,
2944 "Key bit must be on");
2947 mask = &gre_key_default_mask;
2948 ret = mlx5_flow_item_acceptable
2949 (item, (const uint8_t *)mask,
2950 (const uint8_t *)&gre_key_default_mask,
2951 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2956 * Validate GRE optional item.
2959 * Pointer to the Ethernet device structure.
2961 * Item specification.
2962 * @param[in] item_flags
2963 * Bit flags to mark detected items.
2965 * Flow rule attributes.
2966 * @param[in] gre_item
2967 * Pointer to gre_item
2969 * Pointer to error structure.
2972 * 0 on success, a negative errno value otherwise and rte_errno is set.
2975 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev,
2976 const struct rte_flow_item *item,
2977 uint64_t item_flags,
2978 const struct rte_flow_attr *attr,
2979 const struct rte_flow_item *gre_item,
2980 struct rte_flow_error *error)
2982 const struct rte_flow_item_gre *gre_spec = gre_item->spec;
2983 const struct rte_flow_item_gre *gre_mask = gre_item->mask;
2984 const struct rte_flow_item_gre_opt *spec = item->spec;
2985 const struct rte_flow_item_gre_opt *mask = item->mask;
2986 struct mlx5_priv *priv = dev->data->dev_private;
2988 struct rte_flow_item_gre_opt nic_mask = {
2990 .checksum = RTE_BE16(UINT16_MAX),
2994 .key = RTE_BE32(UINT32_MAX),
2997 .sequence = RTE_BE32(UINT32_MAX),
3001 if (!(item_flags & MLX5_FLOW_LAYER_GRE))
3002 return rte_flow_error_set(error, ENOTSUP,
3003 RTE_FLOW_ERROR_TYPE_ITEM, item,
3004 "No preceding GRE header");
3005 if (item_flags & MLX5_FLOW_LAYER_INNER)
3006 return rte_flow_error_set(error, ENOTSUP,
3007 RTE_FLOW_ERROR_TYPE_ITEM, item,
3008 "GRE option following a wrong item");
3010 return rte_flow_error_set(error, EINVAL,
3011 RTE_FLOW_ERROR_TYPE_ITEM, item,
3012 "At least one field gre_option(checksum/key/sequence) must be specified");
3014 gre_mask = &rte_flow_item_gre_mask;
3015 if (mask->checksum_rsvd.checksum)
3016 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) &&
3017 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000)))
3018 return rte_flow_error_set(error, EINVAL,
3019 RTE_FLOW_ERROR_TYPE_ITEM,
3021 "Checksum bit must be on");
3023 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
3024 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
3025 return rte_flow_error_set(error, EINVAL,
3026 RTE_FLOW_ERROR_TYPE_ITEM,
3027 item, "Key bit must be on");
3028 if (mask->sequence.sequence)
3029 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) &&
3030 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000)))
3031 return rte_flow_error_set(error, EINVAL,
3032 RTE_FLOW_ERROR_TYPE_ITEM,
3034 "Sequence bit must be on");
3035 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) {
3036 if (priv->sh->steering_format_version ==
3037 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 ||
3038 ((attr->group || attr->transfer) &&
3039 !priv->sh->misc5_cap) ||
3040 (!(priv->sh->tunnel_header_0_1 &&
3041 priv->sh->tunnel_header_2_3) &&
3042 !attr->group && !attr->transfer))
3043 return rte_flow_error_set(error, EINVAL,
3044 RTE_FLOW_ERROR_TYPE_ITEM,
3046 "Checksum/Sequence not supported");
3048 ret = mlx5_flow_item_acceptable
3049 (item, (const uint8_t *)mask,
3050 (const uint8_t *)&nic_mask,
3051 sizeof(struct rte_flow_item_gre_opt),
3052 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3057 * Validate GRE item.
3060 * Item specification.
3061 * @param[in] item_flags
3062 * Bit flags to mark detected items.
3063 * @param[in] target_protocol
3064 * The next protocol in the previous item.
3066 * Pointer to error structure.
3069 * 0 on success, a negative errno value otherwise and rte_errno is set.
3072 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
3073 uint64_t item_flags,
3074 uint8_t target_protocol,
3075 struct rte_flow_error *error)
3077 const struct rte_flow_item_gre *spec __rte_unused = item->spec;
3078 const struct rte_flow_item_gre *mask = item->mask;
3080 const struct rte_flow_item_gre nic_mask = {
3081 .c_rsvd0_ver = RTE_BE16(0xB000),
3082 .protocol = RTE_BE16(UINT16_MAX),
3085 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
3086 return rte_flow_error_set(error, EINVAL,
3087 RTE_FLOW_ERROR_TYPE_ITEM, item,
3088 "protocol filtering not compatible"
3089 " with this GRE layer");
3090 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3091 return rte_flow_error_set(error, ENOTSUP,
3092 RTE_FLOW_ERROR_TYPE_ITEM, item,
3093 "multiple tunnel layers not"
3095 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
3096 return rte_flow_error_set(error, ENOTSUP,
3097 RTE_FLOW_ERROR_TYPE_ITEM, item,
3098 "L3 Layer is missing");
3100 mask = &rte_flow_item_gre_mask;
3101 ret = mlx5_flow_item_acceptable
3102 (item, (const uint8_t *)mask,
3103 (const uint8_t *)&nic_mask,
3104 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
3108 #ifndef HAVE_MLX5DV_DR
3109 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
3110 if (spec && (spec->protocol & mask->protocol))
3111 return rte_flow_error_set(error, ENOTSUP,
3112 RTE_FLOW_ERROR_TYPE_ITEM, item,
3113 "without MPLS support the"
3114 " specification cannot be used for"
3122 * Validate Geneve item.
3125 * Item specification.
3126 * @param[in] itemFlags
3127 * Bit-fields that holds the items detected until now.
3129 * Pointer to the private data structure.
3131 * Pointer to error structure.
3134 * 0 on success, a negative errno value otherwise and rte_errno is set.
3138 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
3139 uint64_t item_flags,
3140 struct rte_eth_dev *dev,
3141 struct rte_flow_error *error)
3143 struct mlx5_priv *priv = dev->data->dev_private;
3144 const struct rte_flow_item_geneve *spec = item->spec;
3145 const struct rte_flow_item_geneve *mask = item->mask;
3148 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ?
3149 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
3150 const struct rte_flow_item_geneve nic_mask = {
3151 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
3152 .vni = "\xff\xff\xff",
3153 .protocol = RTE_BE16(UINT16_MAX),
3156 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx)
3157 return rte_flow_error_set(error, ENOTSUP,
3158 RTE_FLOW_ERROR_TYPE_ITEM, item,
3159 "L3 Geneve is not enabled by device"
3160 " parameter and/or not configured in"
3162 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3163 return rte_flow_error_set(error, ENOTSUP,
3164 RTE_FLOW_ERROR_TYPE_ITEM, item,
3165 "multiple tunnel layers not"
3168 * Verify only UDPv4 is present as defined in
3169 * https://tools.ietf.org/html/rfc7348
3171 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
3172 return rte_flow_error_set(error, EINVAL,
3173 RTE_FLOW_ERROR_TYPE_ITEM, item,
3174 "no outer UDP layer found");
3176 mask = &rte_flow_item_geneve_mask;
3177 ret = mlx5_flow_item_acceptable
3178 (item, (const uint8_t *)mask,
3179 (const uint8_t *)&nic_mask,
3180 sizeof(struct rte_flow_item_geneve),
3181 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3185 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
3186 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
3187 MLX5_GENEVE_CRITO_VAL(gbhdr) ||
3188 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
3189 return rte_flow_error_set(error, ENOTSUP,
3190 RTE_FLOW_ERROR_TYPE_ITEM,
3192 "Geneve protocol unsupported"
3193 " fields are being used");
3194 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
3195 return rte_flow_error_set
3197 RTE_FLOW_ERROR_TYPE_ITEM,
3199 "Unsupported Geneve options length");
3201 if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
3202 return rte_flow_error_set
3204 RTE_FLOW_ERROR_TYPE_ITEM, item,
3205 "Geneve tunnel must be fully defined");
3210 * Validate Geneve TLV option item.
3213 * Item specification.
3214 * @param[in] last_item
3215 * Previous validated item in the pattern items.
3216 * @param[in] geneve_item
3217 * Previous GENEVE item specification.
3219 * Pointer to the rte_eth_dev structure.
3221 * Pointer to error structure.
3224 * 0 on success, a negative errno value otherwise and rte_errno is set.
3227 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item,
3229 const struct rte_flow_item *geneve_item,
3230 struct rte_eth_dev *dev,
3231 struct rte_flow_error *error)
3233 struct mlx5_priv *priv = dev->data->dev_private;
3234 struct mlx5_dev_ctx_shared *sh = priv->sh;
3235 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource;
3236 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr;
3237 uint8_t data_max_supported =
3238 hca_attr->max_geneve_tlv_option_data_len * 4;
3239 const struct rte_flow_item_geneve *geneve_spec;
3240 const struct rte_flow_item_geneve *geneve_mask;
3241 const struct rte_flow_item_geneve_opt *spec = item->spec;
3242 const struct rte_flow_item_geneve_opt *mask = item->mask;
3244 unsigned int data_len;
3245 uint8_t tlv_option_len;
3246 uint16_t optlen_m, optlen_v;
3247 const struct rte_flow_item_geneve_opt full_mask = {
3248 .option_class = RTE_BE16(0xffff),
3249 .option_type = 0xff,
3254 mask = &rte_flow_item_geneve_opt_mask;
3256 return rte_flow_error_set
3257 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3258 "Geneve TLV opt class/type/length must be specified");
3259 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK)
3260 return rte_flow_error_set
3261 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3262 "Geneve TLV opt length exceeds the limit (31)");
3263 /* Check if class type and length masks are full. */
3264 if (full_mask.option_class != mask->option_class ||
3265 full_mask.option_type != mask->option_type ||
3266 full_mask.option_len != (mask->option_len & full_mask.option_len))
3267 return rte_flow_error_set
3268 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3269 "Geneve TLV opt class/type/length masks must be full");
3270 /* Check if length is supported */
3271 if ((uint32_t)spec->option_len >
3272 hca_attr->max_geneve_tlv_option_data_len)
3273 return rte_flow_error_set
3274 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3275 "Geneve TLV opt length not supported");
3276 if (hca_attr->max_geneve_tlv_options > 1)
3278 "max_geneve_tlv_options supports more than 1 option");
3279 /* Check GENEVE item preceding. */
3280 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE))
3281 return rte_flow_error_set
3282 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3283 "Geneve opt item must be preceded with Geneve item");
3284 geneve_spec = geneve_item->spec;
3285 geneve_mask = geneve_item->mask ? geneve_item->mask :
3286 &rte_flow_item_geneve_mask;
3287 /* Check if GENEVE TLV option size doesn't exceed option length */
3288 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 ||
3289 geneve_spec->ver_opt_len_o_c_rsvd0)) {
3290 tlv_option_len = spec->option_len & mask->option_len;
3291 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0);
3292 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v);
3293 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0);
3294 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m);
3295 if ((optlen_v & optlen_m) <= tlv_option_len)
3296 return rte_flow_error_set
3297 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3298 "GENEVE TLV option length exceeds optlen");
3300 /* Check if length is 0 or data is 0. */
3301 if (spec->data == NULL || spec->option_len == 0)
3302 return rte_flow_error_set
3303 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3304 "Geneve TLV opt with zero data/length not supported");
3305 /* Check not all data & mask are 0. */
3306 data_len = spec->option_len * 4;
3307 if (mask->data == NULL) {
3308 for (i = 0; i < data_len; i++)
3312 return rte_flow_error_set(error, ENOTSUP,
3313 RTE_FLOW_ERROR_TYPE_ITEM, item,
3314 "Can't match on Geneve option data 0");
3316 for (i = 0; i < data_len; i++)
3317 if (spec->data[i] & mask->data[i])
3320 return rte_flow_error_set(error, ENOTSUP,
3321 RTE_FLOW_ERROR_TYPE_ITEM, item,
3322 "Can't match on Geneve option data and mask 0");
3323 /* Check data mask supported. */
3324 for (i = data_max_supported; i < data_len ; i++)
3326 return rte_flow_error_set(error, ENOTSUP,
3327 RTE_FLOW_ERROR_TYPE_ITEM, item,
3328 "Data mask is of unsupported size");
3330 /* Check GENEVE option is supported in NIC. */
3331 if (!hca_attr->geneve_tlv_opt)
3332 return rte_flow_error_set
3333 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3334 "Geneve TLV opt not supported");
3335 /* Check if we already have geneve option with different type/class. */
3336 rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
3337 geneve_opt_resource = sh->geneve_tlv_option_resource;
3338 if (geneve_opt_resource != NULL)
3339 if (geneve_opt_resource->option_class != spec->option_class ||
3340 geneve_opt_resource->option_type != spec->option_type ||
3341 geneve_opt_resource->length != spec->option_len) {
3342 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
3343 return rte_flow_error_set(error, ENOTSUP,
3344 RTE_FLOW_ERROR_TYPE_ITEM, item,
3345 "Only one Geneve TLV option supported");
3347 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
3352 * Validate MPLS item.
3355 * Pointer to the rte_eth_dev structure.
3357 * Item specification.
3358 * @param[in] item_flags
3359 * Bit-fields that holds the items detected until now.
3360 * @param[in] prev_layer
3361 * The protocol layer indicated in previous item.
3363 * Pointer to error structure.
3366 * 0 on success, a negative errno value otherwise and rte_errno is set.
3369 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
3370 const struct rte_flow_item *item __rte_unused,
3371 uint64_t item_flags __rte_unused,
3372 uint64_t prev_layer __rte_unused,
3373 struct rte_flow_error *error)
3375 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
3376 const struct rte_flow_item_mpls *mask = item->mask;
3377 struct mlx5_priv *priv = dev->data->dev_private;
3380 if (!priv->sh->dev_cap.mpls_en)
3381 return rte_flow_error_set(error, ENOTSUP,
3382 RTE_FLOW_ERROR_TYPE_ITEM, item,
3383 "MPLS not supported or"
3384 " disabled in firmware"
3386 /* MPLS over UDP, GRE is allowed */
3387 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP |
3388 MLX5_FLOW_LAYER_GRE |
3389 MLX5_FLOW_LAYER_GRE_KEY)))
3390 return rte_flow_error_set(error, EINVAL,
3391 RTE_FLOW_ERROR_TYPE_ITEM, item,
3392 "protocol filtering not compatible"
3393 " with MPLS layer");
3394 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
3395 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
3396 !(item_flags & MLX5_FLOW_LAYER_GRE))
3397 return rte_flow_error_set(error, ENOTSUP,
3398 RTE_FLOW_ERROR_TYPE_ITEM, item,
3399 "multiple tunnel layers not"
3402 mask = &rte_flow_item_mpls_mask;
3403 ret = mlx5_flow_item_acceptable
3404 (item, (const uint8_t *)mask,
3405 (const uint8_t *)&rte_flow_item_mpls_mask,
3406 sizeof(struct rte_flow_item_mpls),
3407 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3412 return rte_flow_error_set(error, ENOTSUP,
3413 RTE_FLOW_ERROR_TYPE_ITEM, item,
3414 "MPLS is not supported by Verbs, please"
3420 * Validate NVGRE item.
3423 * Item specification.
3424 * @param[in] item_flags
3425 * Bit flags to mark detected items.
3426 * @param[in] target_protocol
3427 * The next protocol in the previous item.
3429 * Pointer to error structure.
3432 * 0 on success, a negative errno value otherwise and rte_errno is set.
3435 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
3436 uint64_t item_flags,
3437 uint8_t target_protocol,
3438 struct rte_flow_error *error)
3440 const struct rte_flow_item_nvgre *mask = item->mask;
3443 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
3444 return rte_flow_error_set(error, EINVAL,
3445 RTE_FLOW_ERROR_TYPE_ITEM, item,
3446 "protocol filtering not compatible"
3447 " with this GRE layer");
3448 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3449 return rte_flow_error_set(error, ENOTSUP,
3450 RTE_FLOW_ERROR_TYPE_ITEM, item,
3451 "multiple tunnel layers not"
3453 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
3454 return rte_flow_error_set(error, ENOTSUP,
3455 RTE_FLOW_ERROR_TYPE_ITEM, item,
3456 "L3 Layer is missing");
3458 mask = &rte_flow_item_nvgre_mask;
3459 ret = mlx5_flow_item_acceptable
3460 (item, (const uint8_t *)mask,
3461 (const uint8_t *)&rte_flow_item_nvgre_mask,
3462 sizeof(struct rte_flow_item_nvgre),
3463 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3470 * Validate eCPRI item.
3473 * Item specification.
3474 * @param[in] item_flags
3475 * Bit-fields that holds the items detected until now.
3476 * @param[in] last_item
3477 * Previous validated item in the pattern items.
3478 * @param[in] ether_type
3479 * Type in the ethernet layer header (including dot1q).
3480 * @param[in] acc_mask
3481 * Acceptable mask, if NULL default internal default mask
3482 * will be used to check whether item fields are supported.
3484 * Pointer to error structure.
3487 * 0 on success, a negative errno value otherwise and rte_errno is set.
3490 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
3491 uint64_t item_flags,
3493 uint16_t ether_type,
3494 const struct rte_flow_item_ecpri *acc_mask,
3495 struct rte_flow_error *error)
3497 const struct rte_flow_item_ecpri *mask = item->mask;
3498 const struct rte_flow_item_ecpri nic_mask = {
3502 RTE_BE32(((const struct rte_ecpri_common_hdr) {
3506 .dummy[0] = 0xFFFFFFFF,
3509 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
3510 MLX5_FLOW_LAYER_OUTER_VLAN);
3511 struct rte_flow_item_ecpri mask_lo;
3513 if (!(last_item & outer_l2_vlan) &&
3514 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP)
3515 return rte_flow_error_set(error, EINVAL,
3516 RTE_FLOW_ERROR_TYPE_ITEM, item,
3517 "eCPRI can only follow L2/VLAN layer or UDP layer");
3518 if ((last_item & outer_l2_vlan) && ether_type &&
3519 ether_type != RTE_ETHER_TYPE_ECPRI)
3520 return rte_flow_error_set(error, EINVAL,
3521 RTE_FLOW_ERROR_TYPE_ITEM, item,
3522 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE");
3523 if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3524 return rte_flow_error_set(error, EINVAL,
3525 RTE_FLOW_ERROR_TYPE_ITEM, item,
3526 "eCPRI with tunnel is not supported right now");
3527 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
3528 return rte_flow_error_set(error, ENOTSUP,
3529 RTE_FLOW_ERROR_TYPE_ITEM, item,
3530 "multiple L3 layers not supported");
3531 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
3532 return rte_flow_error_set(error, EINVAL,
3533 RTE_FLOW_ERROR_TYPE_ITEM, item,
3534 "eCPRI cannot coexist with a TCP layer");
3535 /* In specification, eCPRI could be over UDP layer. */
3536 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
3537 return rte_flow_error_set(error, EINVAL,
3538 RTE_FLOW_ERROR_TYPE_ITEM, item,
3539 "eCPRI over UDP layer is not yet supported right now");
3540 /* Mask for type field in common header could be zero. */
3542 mask = &rte_flow_item_ecpri_mask;
3543 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
3544 /* Input mask is in big-endian format. */
3545 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
3546 return rte_flow_error_set(error, EINVAL,
3547 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3548 "partial mask is not supported for protocol");
3549 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
3550 return rte_flow_error_set(error, EINVAL,
3551 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3552 "message header mask must be after a type mask");
3553 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
3554 acc_mask ? (const uint8_t *)acc_mask
3555 : (const uint8_t *)&nic_mask,
3556 sizeof(struct rte_flow_item_ecpri),
3557 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3561 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
3562 const struct rte_flow_attr *attr __rte_unused,
3563 const struct rte_flow_item items[] __rte_unused,
3564 const struct rte_flow_action actions[] __rte_unused,
3565 bool external __rte_unused,
3566 int hairpin __rte_unused,
3567 struct rte_flow_error *error)
3569 return rte_flow_error_set(error, ENOTSUP,
3570 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3573 static struct mlx5_flow *
3574 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
3575 const struct rte_flow_attr *attr __rte_unused,
3576 const struct rte_flow_item items[] __rte_unused,
3577 const struct rte_flow_action actions[] __rte_unused,
3578 struct rte_flow_error *error)
3580 rte_flow_error_set(error, ENOTSUP,
3581 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3586 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
3587 struct mlx5_flow *dev_flow __rte_unused,
3588 const struct rte_flow_attr *attr __rte_unused,
3589 const struct rte_flow_item items[] __rte_unused,
3590 const struct rte_flow_action actions[] __rte_unused,
3591 struct rte_flow_error *error)
3593 return rte_flow_error_set(error, ENOTSUP,
3594 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3598 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
3599 struct rte_flow *flow __rte_unused,
3600 struct rte_flow_error *error)
3602 return rte_flow_error_set(error, ENOTSUP,
3603 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3607 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
3608 struct rte_flow *flow __rte_unused)
3613 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
3614 struct rte_flow *flow __rte_unused)
3619 flow_null_query(struct rte_eth_dev *dev __rte_unused,
3620 struct rte_flow *flow __rte_unused,
3621 const struct rte_flow_action *actions __rte_unused,
3622 void *data __rte_unused,
3623 struct rte_flow_error *error)
3625 return rte_flow_error_set(error, ENOTSUP,
3626 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3630 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused,
3631 uint32_t domains __rte_unused,
3632 uint32_t flags __rte_unused)
3637 /* Void driver to protect from null pointer reference. */
3638 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
3639 .validate = flow_null_validate,
3640 .prepare = flow_null_prepare,
3641 .translate = flow_null_translate,
3642 .apply = flow_null_apply,
3643 .remove = flow_null_remove,
3644 .destroy = flow_null_destroy,
3645 .query = flow_null_query,
3646 .sync_domain = flow_null_sync_domain,
3650 * Select flow driver type according to flow attributes and device
3654 * Pointer to the dev structure.
3656 * Pointer to the flow attributes.
3659 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
3661 static enum mlx5_flow_drv_type
3662 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
3664 struct mlx5_priv *priv = dev->data->dev_private;
3665 /* The OS can determine first a specific flow type (DV, VERBS) */
3666 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
3668 if (type != MLX5_FLOW_TYPE_MAX)
3671 * Currently when dv_flow_en == 2, only HW steering engine is
3672 * supported. New engines can also be chosen here if ready.
3674 if (priv->sh->config.dv_flow_en == 2)
3675 return MLX5_FLOW_TYPE_HW;
3676 /* If no OS specific type - continue with DV/VERBS selection */
3677 if (attr->transfer && priv->sh->config.dv_esw_en)
3678 type = MLX5_FLOW_TYPE_DV;
3679 if (!attr->transfer)
3680 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
3681 MLX5_FLOW_TYPE_VERBS;
3685 #define flow_get_drv_ops(type) flow_drv_ops[type]
3688 * Flow driver validation API. This abstracts calling driver specific functions.
3689 * The type of flow driver is determined according to flow attributes.
3692 * Pointer to the dev structure.
3694 * Pointer to the flow attributes.
3696 * Pointer to the list of items.
3697 * @param[in] actions
3698 * Pointer to the list of actions.
3699 * @param[in] external
3700 * This flow rule is created by request external to PMD.
3701 * @param[in] hairpin
3702 * Number of hairpin TX actions, 0 means classic flow.
3704 * Pointer to the error structure.
3707 * 0 on success, a negative errno value otherwise and rte_errno is set.
3710 flow_drv_validate(struct rte_eth_dev *dev,
3711 const struct rte_flow_attr *attr,
3712 const struct rte_flow_item items[],
3713 const struct rte_flow_action actions[],
3714 bool external, int hairpin, struct rte_flow_error *error)
3716 const struct mlx5_flow_driver_ops *fops;
3717 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
3719 fops = flow_get_drv_ops(type);
3720 return fops->validate(dev, attr, items, actions, external,
3725 * Flow driver preparation API. This abstracts calling driver specific
3726 * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3727 * calculates the size of memory required for device flow, allocates the memory,
3728 * initializes the device flow and returns the pointer.
3731 * This function initializes device flow structure such as dv or verbs in
3732 * struct mlx5_flow. However, it is caller's responsibility to initialize the
3733 * rest. For example, adding returning device flow to flow->dev_flow list and
3734 * setting backward reference to the flow should be done out of this function.
3735 * layers field is not filled either.
3738 * Pointer to the dev structure.
3740 * Pointer to the flow attributes.
3742 * Pointer to the list of items.
3743 * @param[in] actions
3744 * Pointer to the list of actions.
3745 * @param[in] flow_idx
3746 * This memory pool index to the flow.
3748 * Pointer to the error structure.
3751 * Pointer to device flow on success, otherwise NULL and rte_errno is set.
3753 static inline struct mlx5_flow *
3754 flow_drv_prepare(struct rte_eth_dev *dev,
3755 const struct rte_flow *flow,
3756 const struct rte_flow_attr *attr,
3757 const struct rte_flow_item items[],
3758 const struct rte_flow_action actions[],
3760 struct rte_flow_error *error)
3762 const struct mlx5_flow_driver_ops *fops;
3763 enum mlx5_flow_drv_type type = flow->drv_type;
3764 struct mlx5_flow *mlx5_flow = NULL;
3766 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3767 fops = flow_get_drv_ops(type);
3768 mlx5_flow = fops->prepare(dev, attr, items, actions, error);
3770 mlx5_flow->flow_idx = flow_idx;
3775 * Flow driver translation API. This abstracts calling driver specific
3776 * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3777 * translates a generic flow into a driver flow. flow_drv_prepare() must
3781 * dev_flow->layers could be filled as a result of parsing during translation
3782 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
3783 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
3784 * flow->actions could be overwritten even though all the expanded dev_flows
3785 * have the same actions.
3788 * Pointer to the rte dev structure.
3789 * @param[in, out] dev_flow
3790 * Pointer to the mlx5 flow.
3792 * Pointer to the flow attributes.
3794 * Pointer to the list of items.
3795 * @param[in] actions
3796 * Pointer to the list of actions.
3798 * Pointer to the error structure.
3801 * 0 on success, a negative errno value otherwise and rte_errno is set.
3804 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3805 const struct rte_flow_attr *attr,
3806 const struct rte_flow_item items[],
3807 const struct rte_flow_action actions[],
3808 struct rte_flow_error *error)
3810 const struct mlx5_flow_driver_ops *fops;
3811 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
3813 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3814 fops = flow_get_drv_ops(type);
3815 return fops->translate(dev, dev_flow, attr, items, actions, error);
3819 * Flow driver apply API. This abstracts calling driver specific functions.
3820 * Parent flow (rte_flow) should have driver type (drv_type). It applies
3821 * translated driver flows on to device. flow_drv_translate() must precede.
3824 * Pointer to Ethernet device structure.
3825 * @param[in, out] flow
3826 * Pointer to flow structure.
3828 * Pointer to error structure.
3831 * 0 on success, a negative errno value otherwise and rte_errno is set.
3834 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3835 struct rte_flow_error *error)
3837 const struct mlx5_flow_driver_ops *fops;
3838 enum mlx5_flow_drv_type type = flow->drv_type;
3840 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3841 fops = flow_get_drv_ops(type);
3842 return fops->apply(dev, flow, error);
3846 * Flow driver destroy API. This abstracts calling driver specific functions.
3847 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3848 * on device and releases resources of the flow.
3851 * Pointer to Ethernet device.
3852 * @param[in, out] flow
3853 * Pointer to flow structure.
3856 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3858 const struct mlx5_flow_driver_ops *fops;
3859 enum mlx5_flow_drv_type type = flow->drv_type;
3861 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3862 fops = flow_get_drv_ops(type);
3863 fops->destroy(dev, flow);
3867 * Flow driver find RSS policy tbl API. This abstracts calling driver
3868 * specific functions. Parent flow (rte_flow) should have driver
3869 * type (drv_type). It will find the RSS policy table that has the rss_desc.
3872 * Pointer to Ethernet device.
3873 * @param[in, out] flow
3874 * Pointer to flow structure.
3876 * Pointer to meter policy table.
3877 * @param[in] rss_desc
3878 * Pointer to rss_desc
3880 static struct mlx5_flow_meter_sub_policy *
3881 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
3882 struct rte_flow *flow,
3883 struct mlx5_flow_meter_policy *policy,
3884 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
3886 const struct mlx5_flow_driver_ops *fops;
3887 enum mlx5_flow_drv_type type = flow->drv_type;
3889 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3890 fops = flow_get_drv_ops(type);
3891 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc);
3895 * Flow driver color tag rule API. This abstracts calling driver
3896 * specific functions. Parent flow (rte_flow) should have driver
3897 * type (drv_type). It will create the color tag rules in hierarchy meter.
3900 * Pointer to Ethernet device.
3901 * @param[in, out] flow
3902 * Pointer to flow structure.
3904 * Pointer to flow meter structure.
3905 * @param[in] src_port
3906 * The src port this extra rule should use.
3908 * The src port id match item.
3910 * Pointer to error structure.
3913 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev,
3914 struct rte_flow *flow,
3915 struct mlx5_flow_meter_info *fm,
3917 const struct rte_flow_item *item,
3918 struct rte_flow_error *error)
3920 const struct mlx5_flow_driver_ops *fops;
3921 enum mlx5_flow_drv_type type = flow->drv_type;
3923 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3924 fops = flow_get_drv_ops(type);
3925 return fops->meter_hierarchy_rule_create(dev, fm,
3926 src_port, item, error);
3930 * Get RSS action from the action list.
3933 * Pointer to Ethernet device.
3934 * @param[in] actions
3935 * Pointer to the list of actions.
3937 * Parent flow structure pointer.
3940 * Pointer to the RSS action if exist, else return NULL.
3942 static const struct rte_flow_action_rss*
3943 flow_get_rss_action(struct rte_eth_dev *dev,
3944 const struct rte_flow_action actions[])
3946 struct mlx5_priv *priv = dev->data->dev_private;
3947 const struct rte_flow_action_rss *rss = NULL;
3948 struct mlx5_meter_policy_action_container *acg;
3949 struct mlx5_meter_policy_action_container *acy;
3951 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3952 switch (actions->type) {
3953 case RTE_FLOW_ACTION_TYPE_RSS:
3954 rss = actions->conf;
3956 case RTE_FLOW_ACTION_TYPE_SAMPLE:
3958 const struct rte_flow_action_sample *sample =
3960 const struct rte_flow_action *act = sample->actions;
3961 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++)
3962 if (act->type == RTE_FLOW_ACTION_TYPE_RSS)
3966 case RTE_FLOW_ACTION_TYPE_METER:
3969 struct mlx5_flow_meter_info *fm;
3970 struct mlx5_flow_meter_policy *policy;
3971 const struct rte_flow_action_meter *mtr = actions->conf;
3973 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx);
3974 if (fm && !fm->def_policy) {
3975 policy = mlx5_flow_meter_policy_find(dev,
3976 fm->policy_id, NULL);
3977 MLX5_ASSERT(policy);
3978 if (policy->is_hierarchy) {
3980 mlx5_flow_meter_hierarchy_get_final_policy(dev,
3985 if (policy->is_rss) {
3987 &policy->act_cnt[RTE_COLOR_GREEN];
3989 &policy->act_cnt[RTE_COLOR_YELLOW];
3990 if (acg->fate_action ==
3991 MLX5_FLOW_FATE_SHARED_RSS)
3992 rss = acg->rss->conf;
3993 else if (acy->fate_action ==
3994 MLX5_FLOW_FATE_SHARED_RSS)
3995 rss = acy->rss->conf;
4008 * Get ASO age action by index.
4011 * Pointer to the Ethernet device structure.
4012 * @param[in] age_idx
4013 * Index to the ASO age action.
4016 * The specified ASO age action.
4018 struct mlx5_aso_age_action*
4019 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx)
4021 uint16_t pool_idx = age_idx & UINT16_MAX;
4022 uint16_t offset = (age_idx >> 16) & UINT16_MAX;
4023 struct mlx5_priv *priv = dev->data->dev_private;
4024 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
4025 struct mlx5_aso_age_pool *pool;
4027 rte_rwlock_read_lock(&mng->resize_rwl);
4028 pool = mng->pools[pool_idx];
4029 rte_rwlock_read_unlock(&mng->resize_rwl);
4030 return &pool->actions[offset - 1];
4033 /* maps indirect action to translated direct in some actions array */
4034 struct mlx5_translated_action_handle {
4035 struct rte_flow_action_handle *action; /**< Indirect action handle. */
4036 int index; /**< Index in related array of rte_flow_action. */
4040 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related
4041 * direct action if translation possible.
4042 * This functionality used to run same execution path for both direct and
4043 * indirect actions on flow create. All necessary preparations for indirect
4044 * action handling should be performed on *handle* actions list returned
4048 * Pointer to Ethernet device.
4049 * @param[in] actions
4050 * List of actions to translate.
4051 * @param[out] handle
4052 * List to store translated indirect action object handles.
4053 * @param[in, out] indir_n
4054 * Size of *handle* array. On return should be updated with number of
4055 * indirect actions retrieved from the *actions* list.
4056 * @param[out] translated_actions
4057 * List of actions where all indirect actions were translated to direct
4058 * if possible. NULL if no translation took place.
4060 * Pointer to the error structure.
4063 * 0 on success, a negative errno value otherwise and rte_errno is set.
4066 flow_action_handles_translate(struct rte_eth_dev *dev,
4067 const struct rte_flow_action actions[],
4068 struct mlx5_translated_action_handle *handle,
4070 struct rte_flow_action **translated_actions,
4071 struct rte_flow_error *error)
4073 struct mlx5_priv *priv = dev->data->dev_private;
4074 struct rte_flow_action *translated = NULL;
4075 size_t actions_size;
4078 struct mlx5_translated_action_handle *handle_end = NULL;
4080 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
4081 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT)
4083 if (copied_n == *indir_n) {
4084 return rte_flow_error_set
4085 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
4086 NULL, "too many shared actions");
4088 rte_memcpy(&handle[copied_n].action, &actions[n].conf,
4089 sizeof(actions[n].conf));
4090 handle[copied_n].index = n;
4094 *indir_n = copied_n;
4097 actions_size = sizeof(struct rte_flow_action) * n;
4098 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
4103 memcpy(translated, actions, actions_size);
4104 for (handle_end = handle + copied_n; handle < handle_end; handle++) {
4105 struct mlx5_shared_action_rss *shared_rss;
4106 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
4107 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
4108 uint32_t idx = act_idx &
4109 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
4112 case MLX5_INDIRECT_ACTION_TYPE_RSS:
4113 shared_rss = mlx5_ipool_get
4114 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
4115 translated[handle->index].type =
4116 RTE_FLOW_ACTION_TYPE_RSS;
4117 translated[handle->index].conf =
4118 &shared_rss->origin;
4120 case MLX5_INDIRECT_ACTION_TYPE_COUNT:
4121 translated[handle->index].type =
4122 (enum rte_flow_action_type)
4123 MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
4124 translated[handle->index].conf = (void *)(uintptr_t)idx;
4126 case MLX5_INDIRECT_ACTION_TYPE_AGE:
4127 if (priv->sh->flow_hit_aso_en) {
4128 translated[handle->index].type =
4129 (enum rte_flow_action_type)
4130 MLX5_RTE_FLOW_ACTION_TYPE_AGE;
4131 translated[handle->index].conf =
4132 (void *)(uintptr_t)idx;
4136 case MLX5_INDIRECT_ACTION_TYPE_CT:
4137 if (priv->sh->ct_aso_en) {
4138 translated[handle->index].type =
4139 RTE_FLOW_ACTION_TYPE_CONNTRACK;
4140 translated[handle->index].conf =
4141 (void *)(uintptr_t)idx;
4146 mlx5_free(translated);
4147 return rte_flow_error_set
4148 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
4149 NULL, "invalid indirect action type");
4152 *translated_actions = translated;
4157 * Get Shared RSS action from the action list.
4160 * Pointer to Ethernet device.
4162 * Pointer to the list of actions.
4163 * @param[in] shared_n
4164 * Actions list length.
4167 * The MLX5 RSS action ID if exists, otherwise return 0.
4170 flow_get_shared_rss_action(struct rte_eth_dev *dev,
4171 struct mlx5_translated_action_handle *handle,
4174 struct mlx5_translated_action_handle *handle_end;
4175 struct mlx5_priv *priv = dev->data->dev_private;
4176 struct mlx5_shared_action_rss *shared_rss;
4179 for (handle_end = handle + shared_n; handle < handle_end; handle++) {
4180 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
4181 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
4182 uint32_t idx = act_idx &
4183 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
4185 case MLX5_INDIRECT_ACTION_TYPE_RSS:
4186 shared_rss = mlx5_ipool_get
4187 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
4189 __atomic_add_fetch(&shared_rss->refcnt, 1,
4200 find_graph_root(uint32_t rss_level)
4202 return rss_level < 2 ? MLX5_EXPANSION_ROOT :
4203 MLX5_EXPANSION_ROOT_OUTER;
4207 * Get layer flags from the prefix flow.
4209 * Some flows may be split to several subflows, the prefix subflow gets the
4210 * match items and the suffix sub flow gets the actions.
4211 * Some actions need the user defined match item flags to get the detail for
4213 * This function helps the suffix flow to get the item layer flags from prefix
4216 * @param[in] dev_flow
4217 * Pointer the created prefix subflow.
4220 * The layers get from prefix subflow.
4222 static inline uint64_t
4223 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
4225 uint64_t layers = 0;
4228 * Layers bits could be localization, but usually the compiler will
4229 * help to do the optimization work for source code.
4230 * If no decap actions, use the layers directly.
4232 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
4233 return dev_flow->handle->layers;
4234 /* Convert L3 layers with decap action. */
4235 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
4236 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
4237 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
4238 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
4239 /* Convert L4 layers with decap action. */
4240 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
4241 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
4242 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
4243 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
4248 * Get metadata split action information.
4250 * @param[in] actions
4251 * Pointer to the list of actions.
4253 * Pointer to the return pointer.
4254 * @param[out] qrss_type
4255 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
4256 * if no QUEUE/RSS is found.
4257 * @param[out] encap_idx
4258 * Pointer to the index of the encap action if exists, otherwise the last
4262 * Total number of actions.
4265 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
4266 const struct rte_flow_action **qrss,
4269 const struct rte_flow_action_raw_encap *raw_encap;
4271 int raw_decap_idx = -1;
4274 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4275 switch (actions->type) {
4276 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4277 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4278 *encap_idx = actions_n;
4280 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4281 raw_decap_idx = actions_n;
4283 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4284 raw_encap = actions->conf;
4285 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4286 *encap_idx = raw_decap_idx != -1 ?
4287 raw_decap_idx : actions_n;
4289 case RTE_FLOW_ACTION_TYPE_QUEUE:
4290 case RTE_FLOW_ACTION_TYPE_RSS:
4298 if (*encap_idx == -1)
4299 *encap_idx = actions_n;
4300 /* Count RTE_FLOW_ACTION_TYPE_END. */
4301 return actions_n + 1;
4305 * Check if the action will change packet.
4308 * Pointer to Ethernet device.
4313 * true if action will change packet, false otherwise.
4315 static bool flow_check_modify_action_type(struct rte_eth_dev *dev,
4316 enum rte_flow_action_type type)
4318 struct mlx5_priv *priv = dev->data->dev_private;
4321 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
4322 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
4323 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
4324 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
4325 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
4326 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
4327 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
4328 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
4329 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
4330 case RTE_FLOW_ACTION_TYPE_SET_TTL:
4331 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
4332 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
4333 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
4334 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
4335 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
4336 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
4337 case RTE_FLOW_ACTION_TYPE_SET_META:
4338 case RTE_FLOW_ACTION_TYPE_SET_TAG:
4339 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
4340 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4341 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4342 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4343 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4344 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4345 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4346 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4347 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4348 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4349 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
4351 case RTE_FLOW_ACTION_TYPE_FLAG:
4352 case RTE_FLOW_ACTION_TYPE_MARK:
4353 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
4363 * Check meter action from the action list.
4366 * Pointer to Ethernet device.
4367 * @param[in] actions
4368 * Pointer to the list of actions.
4369 * @param[out] has_mtr
4370 * Pointer to the meter exist flag.
4371 * @param[out] has_modify
4372 * Pointer to the flag showing there's packet change action.
4373 * @param[out] meter_id
4374 * Pointer to the meter id.
4377 * Total number of actions.
4380 flow_check_meter_action(struct rte_eth_dev *dev,
4381 const struct rte_flow_action actions[],
4382 bool *has_mtr, bool *has_modify, uint32_t *meter_id)
4384 const struct rte_flow_action_meter *mtr = NULL;
4387 MLX5_ASSERT(has_mtr);
4389 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4390 switch (actions->type) {
4391 case RTE_FLOW_ACTION_TYPE_METER:
4392 mtr = actions->conf;
4393 *meter_id = mtr->mtr_id;
4400 *has_modify |= flow_check_modify_action_type(dev,
4404 /* Count RTE_FLOW_ACTION_TYPE_END. */
4405 return actions_n + 1;
4409 * Check if the flow should be split due to hairpin.
4410 * The reason for the split is that in current HW we can't
4411 * support encap and push-vlan on Rx, so if a flow contains
4412 * these actions we move it to Tx.
4415 * Pointer to Ethernet device.
4417 * Flow rule attributes.
4418 * @param[in] actions
4419 * Associated actions (list terminated by the END action).
4422 * > 0 the number of actions and the flow should be split,
4423 * 0 when no split required.
4426 flow_check_hairpin_split(struct rte_eth_dev *dev,
4427 const struct rte_flow_attr *attr,
4428 const struct rte_flow_action actions[])
4430 int queue_action = 0;
4433 const struct rte_flow_action_queue *queue;
4434 const struct rte_flow_action_rss *rss;
4435 const struct rte_flow_action_raw_encap *raw_encap;
4436 const struct rte_eth_hairpin_conf *conf;
4440 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4441 switch (actions->type) {
4442 case RTE_FLOW_ACTION_TYPE_QUEUE:
4443 queue = actions->conf;
4446 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
4447 if (conf == NULL || conf->tx_explicit != 0)
4452 case RTE_FLOW_ACTION_TYPE_RSS:
4453 rss = actions->conf;
4454 if (rss == NULL || rss->queue_num == 0)
4456 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
4457 if (conf == NULL || conf->tx_explicit != 0)
4462 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4463 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4464 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4465 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4466 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4470 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4471 raw_encap = actions->conf;
4472 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4481 if (split && queue_action)
4486 /* Declare flow create/destroy prototype in advance. */
4488 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
4489 const struct rte_flow_attr *attr,
4490 const struct rte_flow_item items[],
4491 const struct rte_flow_action actions[],
4492 bool external, struct rte_flow_error *error);
4495 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
4499 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused,
4500 struct mlx5_list_entry *entry, void *cb_ctx)
4502 struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4503 struct mlx5_flow_mreg_copy_resource *mcp_res =
4504 container_of(entry, typeof(*mcp_res), hlist_ent);
4506 return mcp_res->mark_id != *(uint32_t *)(ctx->data);
4509 struct mlx5_list_entry *
4510 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx)
4512 struct rte_eth_dev *dev = tool_ctx;
4513 struct mlx5_priv *priv = dev->data->dev_private;
4514 struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4515 struct mlx5_flow_mreg_copy_resource *mcp_res;
4516 struct rte_flow_error *error = ctx->error;
4519 uint32_t mark_id = *(uint32_t *)(ctx->data);
4520 struct rte_flow_attr attr = {
4521 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4524 struct mlx5_rte_flow_item_tag tag_spec = {
4527 struct rte_flow_item items[] = {
4528 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
4530 struct rte_flow_action_mark ftag = {
4533 struct mlx5_flow_action_copy_mreg cp_mreg = {
4537 struct rte_flow_action_jump jump = {
4538 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4540 struct rte_flow_action actions[] = {
4541 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
4544 /* Fill the register fields in the flow. */
4545 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
4549 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4553 /* Provide the full width of FLAG specific value. */
4554 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
4555 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
4556 /* Build a new flow. */
4557 if (mark_id != MLX5_DEFAULT_COPY_ID) {
4558 items[0] = (struct rte_flow_item){
4559 .type = (enum rte_flow_item_type)
4560 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4563 items[1] = (struct rte_flow_item){
4564 .type = RTE_FLOW_ITEM_TYPE_END,
4566 actions[0] = (struct rte_flow_action){
4567 .type = (enum rte_flow_action_type)
4568 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
4571 actions[1] = (struct rte_flow_action){
4572 .type = (enum rte_flow_action_type)
4573 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4576 actions[2] = (struct rte_flow_action){
4577 .type = RTE_FLOW_ACTION_TYPE_JUMP,
4580 actions[3] = (struct rte_flow_action){
4581 .type = RTE_FLOW_ACTION_TYPE_END,
4584 /* Default rule, wildcard match. */
4585 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR;
4586 items[0] = (struct rte_flow_item){
4587 .type = RTE_FLOW_ITEM_TYPE_END,
4589 actions[0] = (struct rte_flow_action){
4590 .type = (enum rte_flow_action_type)
4591 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4594 actions[1] = (struct rte_flow_action){
4595 .type = RTE_FLOW_ACTION_TYPE_JUMP,
4598 actions[2] = (struct rte_flow_action){
4599 .type = RTE_FLOW_ACTION_TYPE_END,
4602 /* Build a new entry. */
4603 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
4609 mcp_res->mark_id = mark_id;
4611 * The copy Flows are not included in any list. There
4612 * ones are referenced from other Flows and can not
4613 * be applied, removed, deleted in arbitrary order
4614 * by list traversing.
4616 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP,
4617 &attr, items, actions, false, error);
4618 if (!mcp_res->rix_flow) {
4619 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
4622 return &mcp_res->hlist_ent;
4625 struct mlx5_list_entry *
4626 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
4627 void *cb_ctx __rte_unused)
4629 struct rte_eth_dev *dev = tool_ctx;
4630 struct mlx5_priv *priv = dev->data->dev_private;
4631 struct mlx5_flow_mreg_copy_resource *mcp_res;
4634 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
4639 memcpy(mcp_res, oentry, sizeof(*mcp_res));
4641 return &mcp_res->hlist_ent;
4645 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
4647 struct mlx5_flow_mreg_copy_resource *mcp_res =
4648 container_of(entry, typeof(*mcp_res), hlist_ent);
4649 struct rte_eth_dev *dev = tool_ctx;
4650 struct mlx5_priv *priv = dev->data->dev_private;
4652 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
4656 * Add a flow of copying flow metadata registers in RX_CP_TBL.
4658 * As mark_id is unique, if there's already a registered flow for the mark_id,
4659 * return by increasing the reference counter of the resource. Otherwise, create
4660 * the resource (mcp_res) and flow.
4663 * - If ingress port is ANY and reg_c[1] is mark_id,
4664 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4666 * For default flow (zero mark_id), flow is like,
4667 * - If ingress port is ANY,
4668 * reg_b := reg_c[0] and jump to RX_ACT_TBL.
4671 * Pointer to Ethernet device.
4673 * ID of MARK action, zero means default flow for META.
4675 * Perform verbose error reporting if not NULL.
4678 * Associated resource on success, NULL otherwise and rte_errno is set.
4680 static struct mlx5_flow_mreg_copy_resource *
4681 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
4682 struct rte_flow_error *error)
4684 struct mlx5_priv *priv = dev->data->dev_private;
4685 struct mlx5_list_entry *entry;
4686 struct mlx5_flow_cb_ctx ctx = {
4692 /* Check if already registered. */
4693 MLX5_ASSERT(priv->mreg_cp_tbl);
4694 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx);
4697 return container_of(entry, struct mlx5_flow_mreg_copy_resource,
4702 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
4704 struct mlx5_flow_mreg_copy_resource *mcp_res =
4705 container_of(entry, typeof(*mcp_res), hlist_ent);
4706 struct rte_eth_dev *dev = tool_ctx;
4707 struct mlx5_priv *priv = dev->data->dev_private;
4709 MLX5_ASSERT(mcp_res->rix_flow);
4710 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow);
4711 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
4715 * Release flow in RX_CP_TBL.
4718 * Pointer to Ethernet device.
4720 * Parent flow for wich copying is provided.
4723 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
4724 struct rte_flow *flow)
4726 struct mlx5_flow_mreg_copy_resource *mcp_res;
4727 struct mlx5_priv *priv = dev->data->dev_private;
4729 if (!flow->rix_mreg_copy)
4731 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
4732 flow->rix_mreg_copy);
4733 if (!mcp_res || !priv->mreg_cp_tbl)
4735 MLX5_ASSERT(mcp_res->rix_flow);
4736 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
4737 flow->rix_mreg_copy = 0;
4741 * Remove the default copy action from RX_CP_TBL.
4743 * This functions is called in the mlx5_dev_start(). No thread safe
4747 * Pointer to Ethernet device.
4750 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
4752 struct mlx5_list_entry *entry;
4753 struct mlx5_priv *priv = dev->data->dev_private;
4754 struct mlx5_flow_cb_ctx ctx;
4757 /* Check if default flow is registered. */
4758 if (!priv->mreg_cp_tbl)
4760 mark_id = MLX5_DEFAULT_COPY_ID;
4761 ctx.data = &mark_id;
4762 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx);
4765 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
4769 * Add the default copy action in in RX_CP_TBL.
4771 * This functions is called in the mlx5_dev_start(). No thread safe
4775 * Pointer to Ethernet device.
4777 * Perform verbose error reporting if not NULL.
4780 * 0 for success, negative value otherwise and rte_errno is set.
4783 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
4784 struct rte_flow_error *error)
4786 struct mlx5_priv *priv = dev->data->dev_private;
4787 struct mlx5_flow_mreg_copy_resource *mcp_res;
4788 struct mlx5_flow_cb_ctx ctx;
4791 /* Check whether extensive metadata feature is engaged. */
4792 if (!priv->sh->config.dv_flow_en ||
4793 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4794 !mlx5_flow_ext_mreg_supported(dev) ||
4795 !priv->sh->dv_regc0_mask)
4798 * Add default mreg copy flow may be called multiple time, but
4799 * only be called once in stop. Avoid register it twice.
4801 mark_id = MLX5_DEFAULT_COPY_ID;
4802 ctx.data = &mark_id;
4803 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx))
4805 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error);
4812 * Add a flow of copying flow metadata registers in RX_CP_TBL.
4814 * All the flow having Q/RSS action should be split by
4815 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
4816 * performs the following,
4817 * - CQE->flow_tag := reg_c[1] (MARK)
4818 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4819 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
4820 * but there should be a flow per each MARK ID set by MARK action.
4822 * For the aforementioned reason, if there's a MARK action in flow's action
4823 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
4824 * the MARK ID to CQE's flow_tag like,
4825 * - If reg_c[1] is mark_id,
4826 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4828 * For SET_META action which stores value in reg_c[0], as the destination is
4829 * also a flow metadata register (reg_b), adding a default flow is enough. Zero
4830 * MARK ID means the default flow. The default flow looks like,
4831 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4834 * Pointer to Ethernet device.
4836 * Pointer to flow structure.
4837 * @param[in] actions
4838 * Pointer to the list of actions.
4840 * Perform verbose error reporting if not NULL.
4843 * 0 on success, negative value otherwise and rte_errno is set.
4846 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
4847 struct rte_flow *flow,
4848 const struct rte_flow_action *actions,
4849 struct rte_flow_error *error)
4851 struct mlx5_priv *priv = dev->data->dev_private;
4852 struct mlx5_sh_config *config = &priv->sh->config;
4853 struct mlx5_flow_mreg_copy_resource *mcp_res;
4854 const struct rte_flow_action_mark *mark;
4856 /* Check whether extensive metadata feature is engaged. */
4857 if (!config->dv_flow_en ||
4858 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4859 !mlx5_flow_ext_mreg_supported(dev) ||
4860 !priv->sh->dv_regc0_mask)
4862 /* Find MARK action. */
4863 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4864 switch (actions->type) {
4865 case RTE_FLOW_ACTION_TYPE_FLAG:
4866 mcp_res = flow_mreg_add_copy_action
4867 (dev, MLX5_FLOW_MARK_DEFAULT, error);
4870 flow->rix_mreg_copy = mcp_res->idx;
4872 case RTE_FLOW_ACTION_TYPE_MARK:
4873 mark = (const struct rte_flow_action_mark *)
4876 flow_mreg_add_copy_action(dev, mark->id, error);
4879 flow->rix_mreg_copy = mcp_res->idx;
4888 #define MLX5_MAX_SPLIT_ACTIONS 24
4889 #define MLX5_MAX_SPLIT_ITEMS 24
4892 * Split the hairpin flow.
4893 * Since HW can't support encap and push-vlan on Rx, we move these
4895 * If the count action is after the encap then we also
4896 * move the count action. in this case the count will also measure
4900 * Pointer to Ethernet device.
4901 * @param[in] actions
4902 * Associated actions (list terminated by the END action).
4903 * @param[out] actions_rx
4905 * @param[out] actions_tx
4907 * @param[out] pattern_tx
4908 * The pattern items for the Tx flow.
4909 * @param[out] flow_id
4910 * The flow ID connected to this flow.
4916 flow_hairpin_split(struct rte_eth_dev *dev,
4917 const struct rte_flow_action actions[],
4918 struct rte_flow_action actions_rx[],
4919 struct rte_flow_action actions_tx[],
4920 struct rte_flow_item pattern_tx[],
4923 const struct rte_flow_action_raw_encap *raw_encap;
4924 const struct rte_flow_action_raw_decap *raw_decap;
4925 struct mlx5_rte_flow_action_set_tag *set_tag;
4926 struct rte_flow_action *tag_action;
4927 struct mlx5_rte_flow_item_tag *tag_item;
4928 struct rte_flow_item *item;
4932 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4933 switch (actions->type) {
4934 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4935 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4936 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4937 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4938 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4939 rte_memcpy(actions_tx, actions,
4940 sizeof(struct rte_flow_action));
4943 case RTE_FLOW_ACTION_TYPE_COUNT:
4945 rte_memcpy(actions_tx, actions,
4946 sizeof(struct rte_flow_action));
4949 rte_memcpy(actions_rx, actions,
4950 sizeof(struct rte_flow_action));
4954 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4955 raw_encap = actions->conf;
4956 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) {
4957 memcpy(actions_tx, actions,
4958 sizeof(struct rte_flow_action));
4962 rte_memcpy(actions_rx, actions,
4963 sizeof(struct rte_flow_action));
4967 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4968 raw_decap = actions->conf;
4969 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) {
4970 memcpy(actions_tx, actions,
4971 sizeof(struct rte_flow_action));
4974 rte_memcpy(actions_rx, actions,
4975 sizeof(struct rte_flow_action));
4980 rte_memcpy(actions_rx, actions,
4981 sizeof(struct rte_flow_action));
4986 /* Add set meta action and end action for the Rx flow. */
4987 tag_action = actions_rx;
4988 tag_action->type = (enum rte_flow_action_type)
4989 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4991 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
4993 set_tag = (void *)actions_rx;
4994 *set_tag = (struct mlx5_rte_flow_action_set_tag) {
4995 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL),
4998 MLX5_ASSERT(set_tag->id > REG_NON);
4999 tag_action->conf = set_tag;
5000 /* Create Tx item list. */
5001 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
5002 addr = (void *)&pattern_tx[2];
5004 item->type = (enum rte_flow_item_type)
5005 MLX5_RTE_FLOW_ITEM_TYPE_TAG;
5006 tag_item = (void *)addr;
5007 tag_item->data = flow_id;
5008 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
5009 MLX5_ASSERT(set_tag->id > REG_NON);
5010 item->spec = tag_item;
5011 addr += sizeof(struct mlx5_rte_flow_item_tag);
5012 tag_item = (void *)addr;
5013 tag_item->data = UINT32_MAX;
5014 tag_item->id = UINT16_MAX;
5015 item->mask = tag_item;
5018 item->type = RTE_FLOW_ITEM_TYPE_END;
5023 * The last stage of splitting chain, just creates the subflow
5024 * without any modification.
5027 * Pointer to Ethernet device.
5029 * Parent flow structure pointer.
5030 * @param[in, out] sub_flow
5031 * Pointer to return the created subflow, may be NULL.
5033 * Flow rule attributes.
5035 * Pattern specification (list terminated by the END pattern item).
5036 * @param[in] actions
5037 * Associated actions (list terminated by the END action).
5038 * @param[in] flow_split_info
5039 * Pointer to flow split info structure.
5041 * Perform verbose error reporting if not NULL.
5043 * 0 on success, negative value otherwise
5046 flow_create_split_inner(struct rte_eth_dev *dev,
5047 struct rte_flow *flow,
5048 struct mlx5_flow **sub_flow,
5049 const struct rte_flow_attr *attr,
5050 const struct rte_flow_item items[],
5051 const struct rte_flow_action actions[],
5052 struct mlx5_flow_split_info *flow_split_info,
5053 struct rte_flow_error *error)
5055 struct mlx5_flow *dev_flow;
5056 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
5058 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
5059 flow_split_info->flow_idx, error);
5062 dev_flow->flow = flow;
5063 dev_flow->external = flow_split_info->external;
5064 dev_flow->skip_scale = flow_split_info->skip_scale;
5065 /* Subflow object was created, we must include one in the list. */
5066 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
5067 dev_flow->handle, next);
5069 * If dev_flow is as one of the suffix flow, some actions in suffix
5070 * flow may need some user defined item layer flags, and pass the
5071 * Metadata rxq mark flag to suffix flow as well.
5073 if (flow_split_info->prefix_layers)
5074 dev_flow->handle->layers = flow_split_info->prefix_layers;
5075 if (flow_split_info->prefix_mark) {
5080 *sub_flow = dev_flow;
5081 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5082 dev_flow->dv.table_id = flow_split_info->table_id;
5084 return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
5088 * Get the sub policy of a meter.
5091 * Pointer to Ethernet device.
5093 * Parent flow structure pointer.
5095 * Pointer to thread flow work space.
5097 * Flow rule attributes.
5099 * Pattern specification (list terminated by the END pattern item).
5101 * Perform verbose error reporting if not NULL.
5104 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set.
5106 static struct mlx5_flow_meter_sub_policy *
5107 get_meter_sub_policy(struct rte_eth_dev *dev,
5108 struct rte_flow *flow,
5109 struct mlx5_flow_workspace *wks,
5110 const struct rte_flow_attr *attr,
5111 const struct rte_flow_item items[],
5112 struct rte_flow_error *error)
5114 struct mlx5_flow_meter_policy *policy;
5115 struct mlx5_flow_meter_policy *final_policy;
5116 struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
5118 policy = wks->policy;
5119 final_policy = policy->is_hierarchy ? wks->final_policy : policy;
5120 if (final_policy->is_rss || final_policy->is_queue) {
5121 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS];
5122 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0};
5126 * This is a tmp dev_flow,
5127 * no need to register any matcher for it in translate.
5129 wks->skip_matcher_reg = 1;
5130 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
5131 struct mlx5_flow dev_flow = {0};
5132 struct mlx5_flow_handle dev_handle = { {0} };
5133 uint8_t fate = final_policy->act_cnt[i].fate_action;
5135 if (fate == MLX5_FLOW_FATE_SHARED_RSS) {
5136 const struct rte_flow_action_rss *rss_act =
5137 final_policy->act_cnt[i].rss->conf;
5138 struct rte_flow_action rss_actions[2] = {
5140 .type = RTE_FLOW_ACTION_TYPE_RSS,
5144 .type = RTE_FLOW_ACTION_TYPE_END,
5149 dev_flow.handle = &dev_handle;
5150 dev_flow.ingress = attr->ingress;
5151 dev_flow.flow = flow;
5152 dev_flow.external = 0;
5153 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5154 dev_flow.dv.transfer = attr->transfer;
5157 * Translate RSS action to get rss hash fields.
5159 if (flow_drv_translate(dev, &dev_flow, attr,
5160 items, rss_actions, error))
5162 rss_desc_v[i] = wks->rss_desc;
5163 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN;
5164 rss_desc_v[i].hash_fields =
5165 dev_flow.hash_fields;
5166 rss_desc_v[i].queue_num =
5167 rss_desc_v[i].hash_fields ?
5168 rss_desc_v[i].queue_num : 1;
5169 rss_desc_v[i].tunnel =
5170 !!(dev_flow.handle->layers &
5171 MLX5_FLOW_LAYER_TUNNEL);
5172 /* Use the RSS queues in the containers. */
5173 rss_desc_v[i].queue =
5174 (uint16_t *)(uintptr_t)rss_act->queue;
5175 rss_desc[i] = &rss_desc_v[i];
5176 } else if (fate == MLX5_FLOW_FATE_QUEUE) {
5177 /* This is queue action. */
5178 rss_desc_v[i] = wks->rss_desc;
5179 rss_desc_v[i].key_len = 0;
5180 rss_desc_v[i].hash_fields = 0;
5181 rss_desc_v[i].queue =
5182 &final_policy->act_cnt[i].queue;
5183 rss_desc_v[i].queue_num = 1;
5184 rss_desc[i] = &rss_desc_v[i];
5189 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev,
5190 flow, policy, rss_desc);
5192 enum mlx5_meter_domain mtr_domain =
5193 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
5194 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
5195 MLX5_MTR_DOMAIN_INGRESS);
5196 sub_policy = policy->sub_policys[mtr_domain][0];
5199 rte_flow_error_set(error, EINVAL,
5200 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5201 "Failed to get meter sub-policy.");
5207 * Split the meter flow.
5209 * As meter flow will split to three sub flow, other than meter
5210 * action, the other actions make sense to only meter accepts
5211 * the packet. If it need to be dropped, no other additional
5212 * actions should be take.
5214 * One kind of special action which decapsulates the L3 tunnel
5215 * header will be in the prefix sub flow, as not to take the
5216 * L3 tunnel header into account.
5219 * Pointer to Ethernet device.
5221 * Parent flow structure pointer.
5223 * Pointer to thread flow work space.
5225 * Flow rule attributes.
5227 * Pattern specification (list terminated by the END pattern item).
5228 * @param[out] sfx_items
5229 * Suffix flow match items (list terminated by the END pattern item).
5230 * @param[in] actions
5231 * Associated actions (list terminated by the END action).
5232 * @param[out] actions_sfx
5233 * Suffix flow actions.
5234 * @param[out] actions_pre
5235 * Prefix flow actions.
5236 * @param[out] mtr_flow_id
5237 * Pointer to meter flow id.
5239 * Perform verbose error reporting if not NULL.
5242 * 0 on success, a negative errno value otherwise and rte_errno is set.
5245 flow_meter_split_prep(struct rte_eth_dev *dev,
5246 struct rte_flow *flow,
5247 struct mlx5_flow_workspace *wks,
5248 const struct rte_flow_attr *attr,
5249 const struct rte_flow_item items[],
5250 struct rte_flow_item sfx_items[],
5251 const struct rte_flow_action actions[],
5252 struct rte_flow_action actions_sfx[],
5253 struct rte_flow_action actions_pre[],
5254 uint32_t *mtr_flow_id,
5255 struct rte_flow_error *error)
5257 struct mlx5_priv *priv = dev->data->dev_private;
5258 struct mlx5_flow_meter_info *fm = wks->fm;
5259 struct rte_flow_action *tag_action = NULL;
5260 struct rte_flow_item *tag_item;
5261 struct mlx5_rte_flow_action_set_tag *set_tag;
5262 const struct rte_flow_action_raw_encap *raw_encap;
5263 const struct rte_flow_action_raw_decap *raw_decap;
5264 struct mlx5_rte_flow_item_tag *tag_item_spec;
5265 struct mlx5_rte_flow_item_tag *tag_item_mask;
5266 uint32_t tag_id = 0;
5267 struct rte_flow_item *vlan_item_dst = NULL;
5268 const struct rte_flow_item *vlan_item_src = NULL;
5269 const struct rte_flow_item *orig_items = items;
5270 struct rte_flow_action *hw_mtr_action;
5271 struct rte_flow_action *action_pre_head = NULL;
5272 int32_t flow_src_port = priv->representor_id;
5274 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
5275 uint8_t mtr_reg_bits = priv->mtr_reg_share ?
5276 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
5277 uint32_t flow_id = 0;
5278 uint32_t flow_id_reversed = 0;
5279 uint8_t flow_id_bits = 0;
5280 bool after_meter = false;
5283 /* Prepare the suffix subflow items. */
5284 tag_item = sfx_items++;
5285 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5286 struct mlx5_priv *port_priv;
5287 const struct rte_flow_item_port_id *pid_v;
5288 int item_type = items->type;
5290 switch (item_type) {
5291 case RTE_FLOW_ITEM_TYPE_PORT_ID:
5292 pid_v = items->spec;
5294 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false);
5296 return rte_flow_error_set(error,
5298 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5300 "Failed to get port info.");
5301 flow_src_port = port_priv->representor_id;
5302 if (!fm->def_policy && wks->policy->is_hierarchy &&
5303 flow_src_port != priv->representor_id) {
5304 if (flow_drv_mtr_hierarchy_rule_create(dev,
5311 memcpy(sfx_items, items, sizeof(*sfx_items));
5314 case RTE_FLOW_ITEM_TYPE_VLAN:
5315 /* Determine if copy vlan item below. */
5316 vlan_item_src = items;
5317 vlan_item_dst = sfx_items++;
5318 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID;
5324 sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
5326 mtr_first = priv->sh->meter_aso_en &&
5327 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX));
5328 /* For ASO meter, meter must be before tag in TX direction. */
5330 action_pre_head = actions_pre++;
5331 /* Leave space for tag action. */
5332 tag_action = actions_pre++;
5334 /* Prepare the actions for prefix and suffix flow. */
5335 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5336 struct rte_flow_action *action_cur = NULL;
5338 switch (actions->type) {
5339 case RTE_FLOW_ACTION_TYPE_METER:
5341 action_cur = action_pre_head;
5343 /* Leave space for tag action. */
5344 tag_action = actions_pre++;
5345 action_cur = actions_pre++;
5349 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5350 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5351 action_cur = actions_pre++;
5353 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5354 raw_encap = actions->conf;
5355 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
5356 action_cur = actions_pre++;
5358 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5359 raw_decap = actions->conf;
5360 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
5361 action_cur = actions_pre++;
5363 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5364 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5365 if (vlan_item_dst && vlan_item_src) {
5366 memcpy(vlan_item_dst, vlan_item_src,
5367 sizeof(*vlan_item_dst));
5369 * Convert to internal match item, it is used
5370 * for vlan push and set vid.
5372 vlan_item_dst->type = (enum rte_flow_item_type)
5373 MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
5376 case RTE_FLOW_ACTION_TYPE_COUNT:
5378 action_cur = after_meter ?
5379 actions_sfx++ : actions_pre++;
5385 action_cur = (fm->def_policy) ?
5386 actions_sfx++ : actions_pre++;
5387 memcpy(action_cur, actions, sizeof(struct rte_flow_action));
5389 /* Add end action to the actions. */
5390 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
5391 if (priv->sh->meter_aso_en) {
5393 * For ASO meter, need to add an extra jump action explicitly,
5394 * to jump from meter to policer table.
5396 struct mlx5_flow_meter_sub_policy *sub_policy;
5397 struct mlx5_flow_tbl_data_entry *tbl_data;
5399 if (!fm->def_policy) {
5400 sub_policy = get_meter_sub_policy(dev, flow, wks,
5406 enum mlx5_meter_domain mtr_domain =
5407 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
5408 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
5409 MLX5_MTR_DOMAIN_INGRESS);
5412 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy;
5414 tbl_data = container_of(sub_policy->tbl_rsc,
5415 struct mlx5_flow_tbl_data_entry, tbl);
5416 hw_mtr_action = actions_pre++;
5417 hw_mtr_action->type = (enum rte_flow_action_type)
5418 MLX5_RTE_FLOW_ACTION_TYPE_JUMP;
5419 hw_mtr_action->conf = tbl_data->jump.action;
5421 actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
5424 return rte_flow_error_set(error, ENOMEM,
5425 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5426 NULL, "No tag action space.");
5428 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID;
5431 /* Only default-policy Meter creates mtr flow id. */
5432 if (fm->def_policy) {
5433 mlx5_ipool_malloc(fm->flow_ipool, &tag_id);
5435 return rte_flow_error_set(error, ENOMEM,
5436 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5437 "Failed to allocate meter flow id.");
5438 flow_id = tag_id - 1;
5439 flow_id_bits = (!flow_id) ? 1 :
5440 (MLX5_REG_BITS - __builtin_clz(flow_id));
5441 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) >
5443 mlx5_ipool_free(fm->flow_ipool, tag_id);
5444 return rte_flow_error_set(error, EINVAL,
5445 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5446 "Meter flow id exceeds max limit.");
5448 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits)
5449 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits;
5451 /* Build tag actions and items for meter_id/meter flow_id. */
5452 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre;
5453 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
5454 tag_item_mask = tag_item_spec + 1;
5455 /* Both flow_id and meter_id share the same register. */
5456 *set_tag = (struct mlx5_rte_flow_action_set_tag) {
5457 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
5459 .offset = mtr_id_offset,
5460 .length = mtr_reg_bits,
5461 .data = flow->meter,
5464 * The color Reg bits used by flow_id are growing from
5465 * msb to lsb, so must do bit reverse for flow_id val in RegC.
5467 for (shift = 0; shift < flow_id_bits; shift++)
5468 flow_id_reversed = (flow_id_reversed << 1) |
5469 ((flow_id >> shift) & 0x1);
5471 flow_id_reversed << (mtr_reg_bits - flow_id_bits);
5472 tag_item_spec->id = set_tag->id;
5473 tag_item_spec->data = set_tag->data << mtr_id_offset;
5474 tag_item_mask->data = UINT32_MAX << mtr_id_offset;
5475 tag_action->type = (enum rte_flow_action_type)
5476 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5477 tag_action->conf = set_tag;
5478 tag_item->type = (enum rte_flow_item_type)
5479 MLX5_RTE_FLOW_ITEM_TYPE_TAG;
5480 tag_item->spec = tag_item_spec;
5481 tag_item->last = NULL;
5482 tag_item->mask = tag_item_mask;
5485 *mtr_flow_id = tag_id;
5490 * Split action list having QUEUE/RSS for metadata register copy.
5492 * Once Q/RSS action is detected in user's action list, the flow action
5493 * should be split in order to copy metadata registers, which will happen in
5495 * - CQE->flow_tag := reg_c[1] (MARK)
5496 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
5497 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
5498 * This is because the last action of each flow must be a terminal action
5499 * (QUEUE, RSS or DROP).
5501 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
5502 * stored and kept in the mlx5_flow structure per each sub_flow.
5504 * The Q/RSS action is replaced with,
5505 * - SET_TAG, setting the allocated flow ID to reg_c[2].
5506 * And the following JUMP action is added at the end,
5507 * - JUMP, to RX_CP_TBL.
5509 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
5510 * flow_create_split_metadata() routine. The flow will look like,
5511 * - If flow ID matches (reg_c[2]), perform Q/RSS.
5514 * Pointer to Ethernet device.
5515 * @param[out] split_actions
5516 * Pointer to store split actions to jump to CP_TBL.
5517 * @param[in] actions
5518 * Pointer to the list of original flow actions.
5520 * Pointer to the Q/RSS action.
5521 * @param[in] actions_n
5522 * Number of original actions.
5523 * @param[in] mtr_sfx
5524 * Check if it is in meter suffix table.
5526 * Perform verbose error reporting if not NULL.
5529 * non-zero unique flow_id on success, otherwise 0 and
5530 * error/rte_error are set.
5533 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
5534 struct rte_flow_action *split_actions,
5535 const struct rte_flow_action *actions,
5536 const struct rte_flow_action *qrss,
5537 int actions_n, int mtr_sfx,
5538 struct rte_flow_error *error)
5540 struct mlx5_priv *priv = dev->data->dev_private;
5541 struct mlx5_rte_flow_action_set_tag *set_tag;
5542 struct rte_flow_action_jump *jump;
5543 const int qrss_idx = qrss - actions;
5544 uint32_t flow_id = 0;
5548 * Given actions will be split
5549 * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
5550 * - Add jump to mreg CP_TBL.
5551 * As a result, there will be one more action.
5553 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
5554 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */
5556 set_tag = (void *)(split_actions + actions_n);
5558 * If we are not the meter suffix flow, add the tag action.
5559 * Since meter suffix flow already has the tag added.
5563 * Allocate the new subflow ID. This one is unique within
5564 * device and not shared with representors. Otherwise,
5565 * we would have to resolve multi-thread access synch
5566 * issue. Each flow on the shared device is appended
5567 * with source vport identifier, so the resulting
5568 * flows will be unique in the shared (by master and
5569 * representors) domain even if they have coinciding
5572 mlx5_ipool_malloc(priv->sh->ipool
5573 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id);
5575 return rte_flow_error_set(error, ENOMEM,
5576 RTE_FLOW_ERROR_TYPE_ACTION,
5577 NULL, "can't allocate id "
5578 "for split Q/RSS subflow");
5579 /* Internal SET_TAG action to set flow ID. */
5580 *set_tag = (struct mlx5_rte_flow_action_set_tag){
5583 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
5587 /* Construct new actions array. */
5588 /* Replace QUEUE/RSS action. */
5589 split_actions[qrss_idx] = (struct rte_flow_action){
5590 .type = (enum rte_flow_action_type)
5591 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
5596 * If we are the suffix flow of meter, tag already exist.
5597 * Set the QUEUE/RSS action to void.
5599 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID;
5601 /* JUMP action to jump to mreg copy table (CP_TBL). */
5602 jump = (void *)(set_tag + 1);
5603 *jump = (struct rte_flow_action_jump){
5604 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
5606 split_actions[actions_n - 2] = (struct rte_flow_action){
5607 .type = RTE_FLOW_ACTION_TYPE_JUMP,
5610 split_actions[actions_n - 1] = (struct rte_flow_action){
5611 .type = RTE_FLOW_ACTION_TYPE_END,
5617 * Extend the given action list for Tx metadata copy.
5619 * Copy the given action list to the ext_actions and add flow metadata register
5620 * copy action in order to copy reg_a set by WQE to reg_c[0].
5622 * @param[out] ext_actions
5623 * Pointer to the extended action list.
5624 * @param[in] actions
5625 * Pointer to the list of actions.
5626 * @param[in] actions_n
5627 * Number of actions in the list.
5629 * Perform verbose error reporting if not NULL.
5630 * @param[in] encap_idx
5631 * The encap action index.
5634 * 0 on success, negative value otherwise
5637 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
5638 struct rte_flow_action *ext_actions,
5639 const struct rte_flow_action *actions,
5640 int actions_n, struct rte_flow_error *error,
5643 struct mlx5_flow_action_copy_mreg *cp_mreg =
5644 (struct mlx5_flow_action_copy_mreg *)
5645 (ext_actions + actions_n + 1);
5648 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
5652 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
5657 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
5658 if (encap_idx == actions_n - 1) {
5659 ext_actions[actions_n - 1] = (struct rte_flow_action){
5660 .type = (enum rte_flow_action_type)
5661 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5664 ext_actions[actions_n] = (struct rte_flow_action){
5665 .type = RTE_FLOW_ACTION_TYPE_END,
5668 ext_actions[encap_idx] = (struct rte_flow_action){
5669 .type = (enum rte_flow_action_type)
5670 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5673 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
5674 sizeof(*ext_actions) * (actions_n - encap_idx));
5680 * Check the match action from the action list.
5682 * @param[in] actions
5683 * Pointer to the list of actions.
5685 * Flow rule attributes.
5687 * The action to be check if exist.
5688 * @param[out] match_action_pos
5689 * Pointer to the position of the matched action if exists, otherwise is -1.
5690 * @param[out] qrss_action_pos
5691 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
5692 * @param[out] modify_after_mirror
5693 * Pointer to the flag of modify action after FDB mirroring.
5696 * > 0 the total number of actions.
5697 * 0 if not found match action in action list.
5700 flow_check_match_action(const struct rte_flow_action actions[],
5701 const struct rte_flow_attr *attr,
5702 enum rte_flow_action_type action,
5703 int *match_action_pos, int *qrss_action_pos,
5704 int *modify_after_mirror)
5706 const struct rte_flow_action_sample *sample;
5707 const struct rte_flow_action_raw_decap *decap;
5714 *match_action_pos = -1;
5715 *qrss_action_pos = -1;
5716 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5717 if (actions->type == action) {
5719 *match_action_pos = actions_n;
5721 switch (actions->type) {
5722 case RTE_FLOW_ACTION_TYPE_QUEUE:
5723 case RTE_FLOW_ACTION_TYPE_RSS:
5724 *qrss_action_pos = actions_n;
5726 case RTE_FLOW_ACTION_TYPE_SAMPLE:
5727 sample = actions->conf;
5728 ratio = sample->ratio;
5729 sub_type = ((const struct rte_flow_action *)
5730 (sample->actions))->type;
5731 if (ratio == 1 && attr->transfer)
5734 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5735 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5736 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5737 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5738 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5739 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5740 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5741 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5742 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5743 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5744 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5745 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5746 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5747 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5748 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5749 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5750 case RTE_FLOW_ACTION_TYPE_FLAG:
5751 case RTE_FLOW_ACTION_TYPE_MARK:
5752 case RTE_FLOW_ACTION_TYPE_SET_META:
5753 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5754 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5755 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5756 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5757 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5758 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5759 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5760 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
5761 case RTE_FLOW_ACTION_TYPE_METER:
5763 *modify_after_mirror = 1;
5765 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5766 decap = actions->conf;
5767 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5770 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5771 const struct rte_flow_action_raw_encap *encap =
5774 MLX5_ENCAPSULATION_DECISION_SIZE &&
5776 MLX5_ENCAPSULATION_DECISION_SIZE)
5781 *modify_after_mirror = 1;
5788 if (flag && fdb_mirror && !*modify_after_mirror) {
5789 /* FDB mirroring uses the destination array to implement
5790 * instead of FLOW_SAMPLER object.
5792 if (sub_type != RTE_FLOW_ACTION_TYPE_END)
5795 /* Count RTE_FLOW_ACTION_TYPE_END. */
5796 return flag ? actions_n + 1 : 0;
5799 #define SAMPLE_SUFFIX_ITEM 3
5802 * Split the sample flow.
5804 * As sample flow will split to two sub flow, sample flow with
5805 * sample action, the other actions will move to new suffix flow.
5807 * Also add unique tag id with tag action in the sample flow,
5808 * the same tag id will be as match in the suffix flow.
5811 * Pointer to Ethernet device.
5812 * @param[in] add_tag
5813 * Add extra tag action flag.
5814 * @param[out] sfx_items
5815 * Suffix flow match items (list terminated by the END pattern item).
5816 * @param[in] actions
5817 * Associated actions (list terminated by the END action).
5818 * @param[out] actions_sfx
5819 * Suffix flow actions.
5820 * @param[out] actions_pre
5821 * Prefix flow actions.
5822 * @param[in] actions_n
5823 * The total number of actions.
5824 * @param[in] sample_action_pos
5825 * The sample action position.
5826 * @param[in] qrss_action_pos
5827 * The Queue/RSS action position.
5828 * @param[in] jump_table
5829 * Add extra jump action flag.
5831 * Perform verbose error reporting if not NULL.
5834 * 0 on success, or unique flow_id, a negative errno value
5835 * otherwise and rte_errno is set.
5838 flow_sample_split_prep(struct rte_eth_dev *dev,
5840 const struct rte_flow_item items[],
5841 struct rte_flow_item sfx_items[],
5842 const struct rte_flow_action actions[],
5843 struct rte_flow_action actions_sfx[],
5844 struct rte_flow_action actions_pre[],
5846 int sample_action_pos,
5847 int qrss_action_pos,
5849 struct rte_flow_error *error)
5851 struct mlx5_priv *priv = dev->data->dev_private;
5852 struct mlx5_rte_flow_action_set_tag *set_tag;
5853 struct mlx5_rte_flow_item_tag *tag_spec;
5854 struct mlx5_rte_flow_item_tag *tag_mask;
5855 struct rte_flow_action_jump *jump_action;
5856 uint32_t tag_id = 0;
5857 int append_index = 0;
5858 int set_tag_idx = -1;
5862 if (sample_action_pos < 0)
5863 return rte_flow_error_set(error, EINVAL,
5864 RTE_FLOW_ERROR_TYPE_ACTION,
5865 NULL, "invalid position of sample "
5867 /* Prepare the actions for prefix and suffix flow. */
5869 /* Update the new added tag action index preceding
5870 * the PUSH_VLAN or ENCAP action.
5872 const struct rte_flow_action_raw_encap *raw_encap;
5873 const struct rte_flow_action *action = actions;
5876 int raw_decap_idx = -1;
5877 int push_vlan_idx = -1;
5878 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) {
5879 switch (action->type) {
5880 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5881 raw_decap_idx = action_idx;
5883 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5884 raw_encap = action->conf;
5885 if (raw_encap->size >
5886 MLX5_ENCAPSULATION_DECISION_SIZE) {
5887 encap_idx = raw_decap_idx != -1 ?
5888 raw_decap_idx : action_idx;
5889 if (encap_idx < sample_action_pos &&
5890 push_vlan_idx == -1)
5891 set_tag_idx = encap_idx;
5894 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5895 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5896 encap_idx = action_idx;
5897 if (encap_idx < sample_action_pos &&
5898 push_vlan_idx == -1)
5899 set_tag_idx = encap_idx;
5901 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5902 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5903 push_vlan_idx = action_idx;
5904 if (push_vlan_idx < sample_action_pos)
5905 set_tag_idx = action_idx;
5913 /* Prepare the actions for prefix and suffix flow. */
5914 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
5915 index = qrss_action_pos;
5916 /* Put the preceding the Queue/RSS action into prefix flow. */
5918 memcpy(actions_pre, actions,
5919 sizeof(struct rte_flow_action) * index);
5920 /* Put others preceding the sample action into prefix flow. */
5921 if (sample_action_pos > index + 1)
5922 memcpy(actions_pre + index, actions + index + 1,
5923 sizeof(struct rte_flow_action) *
5924 (sample_action_pos - index - 1));
5925 index = sample_action_pos - 1;
5926 /* Put Queue/RSS action into Suffix flow. */
5927 memcpy(actions_sfx, actions + qrss_action_pos,
5928 sizeof(struct rte_flow_action));
5930 } else if (add_tag && set_tag_idx >= 0) {
5931 if (set_tag_idx > 0)
5932 memcpy(actions_pre, actions,
5933 sizeof(struct rte_flow_action) * set_tag_idx);
5934 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx,
5935 sizeof(struct rte_flow_action) *
5936 (sample_action_pos - set_tag_idx));
5937 index = sample_action_pos;
5939 index = sample_action_pos;
5941 memcpy(actions_pre, actions,
5942 sizeof(struct rte_flow_action) * index);
5944 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress.
5945 * For CX6DX and above, metadata registers Cx preserve their value,
5946 * add an extra tag action for NIC-RX and E-Switch Domain.
5949 /* Prepare the prefix tag action. */
5951 set_tag = (void *)(actions_pre + actions_n + append_index);
5952 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error);
5953 /* Trust VF/SF on CX5 not supported meter so that the reserved
5954 * metadata regC is REG_NON, back to use application tag
5957 if (unlikely(ret == REG_NON))
5958 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
5961 mlx5_ipool_malloc(priv->sh->ipool
5962 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id);
5963 *set_tag = (struct mlx5_rte_flow_action_set_tag) {
5967 /* Prepare the suffix subflow items. */
5968 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5969 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID) {
5970 memcpy(sfx_items, items, sizeof(*sfx_items));
5974 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
5975 tag_spec->data = tag_id;
5976 tag_spec->id = set_tag->id;
5977 tag_mask = tag_spec + 1;
5978 tag_mask->data = UINT32_MAX;
5979 sfx_items[0] = (struct rte_flow_item){
5980 .type = (enum rte_flow_item_type)
5981 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5986 sfx_items[1] = (struct rte_flow_item){
5987 .type = (enum rte_flow_item_type)
5988 RTE_FLOW_ITEM_TYPE_END,
5990 /* Prepare the tag action in prefix subflow. */
5991 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx;
5992 actions_pre[set_tag_idx] =
5993 (struct rte_flow_action){
5994 .type = (enum rte_flow_action_type)
5995 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
5998 /* Update next sample position due to add one tag action */
6001 /* Copy the sample action into prefix flow. */
6002 memcpy(actions_pre + index, actions + sample_action_pos,
6003 sizeof(struct rte_flow_action));
6005 /* For the modify action after the sample action in E-Switch mirroring,
6006 * Add the extra jump action in prefix subflow and jump into the next
6007 * table, then do the modify action in the new table.
6010 /* Prepare the prefix jump action. */
6012 jump_action = (void *)(actions_pre + actions_n + append_index);
6013 jump_action->group = jump_table;
6014 actions_pre[index++] =
6015 (struct rte_flow_action){
6016 .type = (enum rte_flow_action_type)
6017 RTE_FLOW_ACTION_TYPE_JUMP,
6018 .conf = jump_action,
6021 actions_pre[index] = (struct rte_flow_action){
6022 .type = (enum rte_flow_action_type)
6023 RTE_FLOW_ACTION_TYPE_END,
6025 /* Put the actions after sample into Suffix flow. */
6026 memcpy(actions_sfx, actions + sample_action_pos + 1,
6027 sizeof(struct rte_flow_action) *
6028 (actions_n - sample_action_pos - 1));
6033 * The splitting for metadata feature.
6035 * - Q/RSS action on NIC Rx should be split in order to pass by
6036 * the mreg copy table (RX_CP_TBL) and then it jumps to the
6037 * action table (RX_ACT_TBL) which has the split Q/RSS action.
6039 * - All the actions on NIC Tx should have a mreg copy action to
6040 * copy reg_a from WQE to reg_c[0].
6043 * Pointer to Ethernet device.
6045 * Parent flow structure pointer.
6047 * Flow rule attributes.
6049 * Pattern specification (list terminated by the END pattern item).
6050 * @param[in] actions
6051 * Associated actions (list terminated by the END action).
6052 * @param[in] flow_split_info
6053 * Pointer to flow split info structure.
6055 * Perform verbose error reporting if not NULL.
6057 * 0 on success, negative value otherwise
6060 flow_create_split_metadata(struct rte_eth_dev *dev,
6061 struct rte_flow *flow,
6062 const struct rte_flow_attr *attr,
6063 const struct rte_flow_item items[],
6064 const struct rte_flow_action actions[],
6065 struct mlx5_flow_split_info *flow_split_info,
6066 struct rte_flow_error *error)
6068 struct mlx5_priv *priv = dev->data->dev_private;
6069 struct mlx5_sh_config *config = &priv->sh->config;
6070 const struct rte_flow_action *qrss = NULL;
6071 struct rte_flow_action *ext_actions = NULL;
6072 struct mlx5_flow *dev_flow = NULL;
6073 uint32_t qrss_id = 0;
6080 /* Check whether extensive metadata feature is engaged. */
6081 if (!config->dv_flow_en ||
6082 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
6083 !mlx5_flow_ext_mreg_supported(dev))
6084 return flow_create_split_inner(dev, flow, NULL, attr, items,
6085 actions, flow_split_info, error);
6086 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
6089 /* Exclude hairpin flows from splitting. */
6090 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
6091 const struct rte_flow_action_queue *queue;
6094 if (mlx5_rxq_is_hairpin(dev, queue->index))
6096 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
6097 const struct rte_flow_action_rss *rss;
6100 if (mlx5_rxq_is_hairpin(dev, rss->queue[0]))
6105 /* Check if it is in meter suffix table. */
6106 mtr_sfx = attr->group == (attr->transfer ?
6107 (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
6108 MLX5_FLOW_TABLE_LEVEL_METER);
6110 * Q/RSS action on NIC Rx should be split in order to pass by
6111 * the mreg copy table (RX_CP_TBL) and then it jumps to the
6112 * action table (RX_ACT_TBL) which has the split Q/RSS action.
6114 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
6115 sizeof(struct rte_flow_action_set_tag) +
6116 sizeof(struct rte_flow_action_jump);
6117 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
6120 return rte_flow_error_set(error, ENOMEM,
6121 RTE_FLOW_ERROR_TYPE_ACTION,
6122 NULL, "no memory to split "
6125 * Create the new actions list with removed Q/RSS action
6126 * and appended set tag and jump to register copy table
6127 * (RX_CP_TBL). We should preallocate unique tag ID here
6128 * in advance, because it is needed for set tag action.
6130 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
6133 if (!mtr_sfx && !qrss_id) {
6137 } else if (attr->egress && !attr->transfer) {
6139 * All the actions on NIC Tx should have a metadata register
6140 * copy action to copy reg_a from WQE to reg_c[meta]
6142 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
6143 sizeof(struct mlx5_flow_action_copy_mreg);
6144 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
6147 return rte_flow_error_set(error, ENOMEM,
6148 RTE_FLOW_ERROR_TYPE_ACTION,
6149 NULL, "no memory to split "
6151 /* Create the action list appended with copy register. */
6152 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
6153 actions_n, error, encap_idx);
6157 /* Add the unmodified original or prefix subflow. */
6158 ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
6159 items, ext_actions ? ext_actions :
6160 actions, flow_split_info, error);
6163 MLX5_ASSERT(dev_flow);
6165 const struct rte_flow_attr q_attr = {
6166 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
6169 /* Internal PMD action to set register. */
6170 struct mlx5_rte_flow_item_tag q_tag_spec = {
6174 struct rte_flow_item q_items[] = {
6176 .type = (enum rte_flow_item_type)
6177 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
6178 .spec = &q_tag_spec,
6183 .type = RTE_FLOW_ITEM_TYPE_END,
6186 struct rte_flow_action q_actions[] = {
6192 .type = RTE_FLOW_ACTION_TYPE_END,
6195 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
6198 * Configure the tag item only if there is no meter subflow.
6199 * Since tag is already marked in the meter suffix subflow
6200 * we can just use the meter suffix items as is.
6203 /* Not meter subflow. */
6204 MLX5_ASSERT(!mtr_sfx);
6206 * Put unique id in prefix flow due to it is destroyed
6207 * after suffix flow and id will be freed after there
6208 * is no actual flows with this id and identifier
6209 * reallocation becomes possible (for example, for
6210 * other flows in other threads).
6212 dev_flow->handle->split_flow_id = qrss_id;
6213 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
6217 q_tag_spec.id = ret;
6220 /* Add suffix subflow to execute Q/RSS. */
6221 flow_split_info->prefix_layers = layers;
6222 flow_split_info->prefix_mark = 0;
6223 flow_split_info->table_id = 0;
6224 ret = flow_create_split_inner(dev, flow, &dev_flow,
6225 &q_attr, mtr_sfx ? items :
6227 flow_split_info, error);
6230 /* qrss ID should be freed if failed. */
6232 MLX5_ASSERT(dev_flow);
6237 * We do not destroy the partially created sub_flows in case of error.
6238 * These ones are included into parent flow list and will be destroyed
6239 * by flow_drv_destroy.
6241 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
6243 mlx5_free(ext_actions);
6248 * Create meter internal drop flow with the original pattern.
6251 * Pointer to Ethernet device.
6253 * Parent flow structure pointer.
6255 * Flow rule attributes.
6257 * Pattern specification (list terminated by the END pattern item).
6258 * @param[in] flow_split_info
6259 * Pointer to flow split info structure.
6261 * Pointer to flow meter structure.
6263 * Perform verbose error reporting if not NULL.
6265 * 0 on success, negative value otherwise
6268 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev,
6269 struct rte_flow *flow,
6270 const struct rte_flow_attr *attr,
6271 const struct rte_flow_item items[],
6272 struct mlx5_flow_split_info *flow_split_info,
6273 struct mlx5_flow_meter_info *fm,
6274 struct rte_flow_error *error)
6276 struct mlx5_flow *dev_flow = NULL;
6277 struct rte_flow_attr drop_attr = *attr;
6278 struct rte_flow_action drop_actions[3];
6279 struct mlx5_flow_split_info drop_split_info = *flow_split_info;
6281 MLX5_ASSERT(fm->drop_cnt);
6282 drop_actions[0].type =
6283 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
6284 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt;
6285 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP;
6286 drop_actions[1].conf = NULL;
6287 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END;
6288 drop_actions[2].conf = NULL;
6289 drop_split_info.external = false;
6290 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT;
6291 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP;
6292 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER;
6293 return flow_create_split_inner(dev, flow, &dev_flow,
6294 &drop_attr, items, drop_actions,
6295 &drop_split_info, error);
6299 * The splitting for meter feature.
6301 * - The meter flow will be split to two flows as prefix and
6302 * suffix flow. The packets make sense only it pass the prefix
6305 * - Reg_C_5 is used for the packet to match betweend prefix and
6309 * Pointer to Ethernet device.
6311 * Parent flow structure pointer.
6313 * Flow rule attributes.
6315 * Pattern specification (list terminated by the END pattern item).
6316 * @param[in] actions
6317 * Associated actions (list terminated by the END action).
6318 * @param[in] flow_split_info
6319 * Pointer to flow split info structure.
6321 * Perform verbose error reporting if not NULL.
6323 * 0 on success, negative value otherwise
6326 flow_create_split_meter(struct rte_eth_dev *dev,
6327 struct rte_flow *flow,
6328 const struct rte_flow_attr *attr,
6329 const struct rte_flow_item items[],
6330 const struct rte_flow_action actions[],
6331 struct mlx5_flow_split_info *flow_split_info,
6332 struct rte_flow_error *error)
6334 struct mlx5_priv *priv = dev->data->dev_private;
6335 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
6336 struct rte_flow_action *sfx_actions = NULL;
6337 struct rte_flow_action *pre_actions = NULL;
6338 struct rte_flow_item *sfx_items = NULL;
6339 struct mlx5_flow *dev_flow = NULL;
6340 struct rte_flow_attr sfx_attr = *attr;
6341 struct mlx5_flow_meter_info *fm = NULL;
6342 uint8_t skip_scale_restore;
6343 bool has_mtr = false;
6344 bool has_modify = false;
6345 bool set_mtr_reg = true;
6346 bool is_mtr_hierarchy = false;
6347 uint32_t meter_id = 0;
6348 uint32_t mtr_idx = 0;
6349 uint32_t mtr_flow_id = 0;
6356 actions_n = flow_check_meter_action(dev, actions, &has_mtr,
6357 &has_modify, &meter_id);
6360 fm = flow_dv_meter_find_by_idx(priv, flow->meter);
6362 return rte_flow_error_set(error, EINVAL,
6363 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6364 NULL, "Meter not found.");
6366 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
6368 return rte_flow_error_set(error, EINVAL,
6369 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6370 NULL, "Meter not found.");
6371 ret = mlx5_flow_meter_attach(priv, fm,
6375 flow->meter = mtr_idx;
6379 if (!fm->def_policy) {
6380 wks->policy = mlx5_flow_meter_policy_find(dev,
6383 MLX5_ASSERT(wks->policy);
6384 if (wks->policy->mark)
6386 if (wks->policy->is_hierarchy) {
6388 mlx5_flow_meter_hierarchy_get_final_policy(dev,
6390 if (!wks->final_policy)
6391 return rte_flow_error_set(error,
6393 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
6394 "Failed to find terminal policy of hierarchy.");
6395 is_mtr_hierarchy = true;
6399 * If it isn't default-policy Meter, and
6400 * 1. There's no action in flow to change
6401 * packet (modify/encap/decap etc.), OR
6402 * 2. No drop count needed for this meter.
6403 * 3. It's not meter hierarchy.
6404 * Then no need to use regC to save meter id anymore.
6406 if (!fm->def_policy && !is_mtr_hierarchy &&
6407 (!has_modify || !fm->drop_cnt))
6408 set_mtr_reg = false;
6409 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */
6410 #define METER_PREFIX_ACTION 7
6411 act_size = (sizeof(struct rte_flow_action) *
6412 (actions_n + METER_PREFIX_ACTION)) +
6413 sizeof(struct mlx5_rte_flow_action_set_tag);
6414 /* Suffix items: tag, vlan, port id, end. */
6415 #define METER_SUFFIX_ITEM 4
6416 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
6417 sizeof(struct mlx5_rte_flow_item_tag) * 2;
6418 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
6421 return rte_flow_error_set(error, ENOMEM,
6422 RTE_FLOW_ERROR_TYPE_ACTION,
6423 NULL, "no memory to split "
6425 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
6427 /* There's no suffix flow for meter of non-default policy. */
6428 if (!fm->def_policy)
6429 pre_actions = sfx_actions + 1;
6431 pre_actions = sfx_actions + actions_n;
6432 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr,
6433 items, sfx_items, actions,
6434 sfx_actions, pre_actions,
6435 (set_mtr_reg ? &mtr_flow_id : NULL),
6441 /* Add the prefix subflow. */
6442 skip_scale_restore = flow_split_info->skip_scale;
6443 flow_split_info->skip_scale |=
6444 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
6445 ret = flow_create_split_inner(dev, flow, &dev_flow,
6446 attr, items, pre_actions,
6447 flow_split_info, error);
6448 flow_split_info->skip_scale = skip_scale_restore;
6451 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id);
6456 dev_flow->handle->split_flow_id = mtr_flow_id;
6457 dev_flow->handle->is_meter_flow_id = 1;
6459 if (!fm->def_policy) {
6460 if (!set_mtr_reg && fm->drop_cnt)
6462 flow_meter_create_drop_flow_with_org_pattern(dev, flow,
6468 /* Setting the sfx group atrr. */
6469 sfx_attr.group = sfx_attr.transfer ?
6470 (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
6471 MLX5_FLOW_TABLE_LEVEL_METER;
6472 flow_split_info->prefix_layers =
6473 flow_get_prefix_layer_flags(dev_flow);
6474 flow_split_info->prefix_mark |= wks->mark;
6475 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX;
6477 /* Add the prefix subflow. */
6478 ret = flow_create_split_metadata(dev, flow,
6479 &sfx_attr, sfx_items ?
6481 sfx_actions ? sfx_actions : actions,
6482 flow_split_info, error);
6485 mlx5_free(sfx_actions);
6490 * The splitting for sample feature.
6492 * Once Sample action is detected in the action list, the flow actions should
6493 * be split into prefix sub flow and suffix sub flow.
6495 * The original items remain in the prefix sub flow, all actions preceding the
6496 * sample action and the sample action itself will be copied to the prefix
6497 * sub flow, the actions following the sample action will be copied to the
6498 * suffix sub flow, Queue action always be located in the suffix sub flow.
6500 * In order to make the packet from prefix sub flow matches with suffix sub
6501 * flow, an extra tag action be added into prefix sub flow, and the suffix sub
6502 * flow uses tag item with the unique flow id.
6505 * Pointer to Ethernet device.
6507 * Parent flow structure pointer.
6509 * Flow rule attributes.
6511 * Pattern specification (list terminated by the END pattern item).
6512 * @param[in] actions
6513 * Associated actions (list terminated by the END action).
6514 * @param[in] flow_split_info
6515 * Pointer to flow split info structure.
6517 * Perform verbose error reporting if not NULL.
6519 * 0 on success, negative value otherwise
6522 flow_create_split_sample(struct rte_eth_dev *dev,
6523 struct rte_flow *flow,
6524 const struct rte_flow_attr *attr,
6525 const struct rte_flow_item items[],
6526 const struct rte_flow_action actions[],
6527 struct mlx5_flow_split_info *flow_split_info,
6528 struct rte_flow_error *error)
6530 struct mlx5_priv *priv = dev->data->dev_private;
6531 struct rte_flow_action *sfx_actions = NULL;
6532 struct rte_flow_action *pre_actions = NULL;
6533 struct rte_flow_item *sfx_items = NULL;
6534 struct mlx5_flow *dev_flow = NULL;
6535 struct rte_flow_attr sfx_attr = *attr;
6536 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
6537 struct mlx5_flow_dv_sample_resource *sample_res;
6538 struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
6539 struct mlx5_flow_tbl_resource *sfx_tbl;
6540 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
6544 uint32_t fdb_tx = 0;
6547 int sample_action_pos;
6548 int qrss_action_pos;
6550 int modify_after_mirror = 0;
6551 uint16_t jump_table = 0;
6552 const uint32_t next_ft_step = 1;
6555 if (priv->sampler_en)
6556 actions_n = flow_check_match_action(actions, attr,
6557 RTE_FLOW_ACTION_TYPE_SAMPLE,
6558 &sample_action_pos, &qrss_action_pos,
6559 &modify_after_mirror);
6561 /* The prefix actions must includes sample, tag, end. */
6562 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
6563 + sizeof(struct mlx5_rte_flow_action_set_tag);
6564 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
6565 sizeof(struct mlx5_rte_flow_item_tag) * 2;
6566 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
6567 item_size), 0, SOCKET_ID_ANY);
6569 return rte_flow_error_set(error, ENOMEM,
6570 RTE_FLOW_ERROR_TYPE_ACTION,
6571 NULL, "no memory to split "
6573 /* The representor_id is UINT16_MAX for uplink. */
6574 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX);
6576 * When reg_c_preserve is set, metadata registers Cx preserve
6577 * their value even through packet duplication.
6579 add_tag = (!fdb_tx ||
6580 priv->sh->cdev->config.hca_attr.reg_c_preserve);
6582 sfx_items = (struct rte_flow_item *)((char *)sfx_actions
6584 if (modify_after_mirror)
6585 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR +
6587 pre_actions = sfx_actions + actions_n;
6588 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items,
6589 actions, sfx_actions,
6590 pre_actions, actions_n,
6592 qrss_action_pos, jump_table,
6594 if (tag_id < 0 || (add_tag && !tag_id)) {
6598 if (modify_after_mirror)
6599 flow_split_info->skip_scale =
6600 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
6601 /* Add the prefix subflow. */
6602 ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
6604 flow_split_info, error);
6609 dev_flow->handle->split_flow_id = tag_id;
6610 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
6611 if (!modify_after_mirror) {
6612 /* Set the sfx group attr. */
6613 sample_res = (struct mlx5_flow_dv_sample_resource *)
6614 dev_flow->dv.sample_res;
6615 sfx_tbl = (struct mlx5_flow_tbl_resource *)
6616 sample_res->normal_path_tbl;
6617 sfx_tbl_data = container_of(sfx_tbl,
6618 struct mlx5_flow_tbl_data_entry,
6620 sfx_attr.group = sfx_attr.transfer ?
6621 (sfx_tbl_data->level - 1) : sfx_tbl_data->level;
6623 MLX5_ASSERT(attr->transfer);
6624 sfx_attr.group = jump_table;
6626 flow_split_info->prefix_layers =
6627 flow_get_prefix_layer_flags(dev_flow);
6629 flow_split_info->prefix_mark |= wks->mark;
6630 /* Suffix group level already be scaled with factor, set
6631 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale
6632 * again in translation.
6634 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT;
6637 /* Add the suffix subflow. */
6638 ret = flow_create_split_meter(dev, flow, &sfx_attr,
6639 sfx_items ? sfx_items : items,
6640 sfx_actions ? sfx_actions : actions,
6641 flow_split_info, error);
6644 mlx5_free(sfx_actions);
6649 * Split the flow to subflow set. The splitters might be linked
6650 * in the chain, like this:
6651 * flow_create_split_outer() calls:
6652 * flow_create_split_meter() calls:
6653 * flow_create_split_metadata(meter_subflow_0) calls:
6654 * flow_create_split_inner(metadata_subflow_0)
6655 * flow_create_split_inner(metadata_subflow_1)
6656 * flow_create_split_inner(metadata_subflow_2)
6657 * flow_create_split_metadata(meter_subflow_1) calls:
6658 * flow_create_split_inner(metadata_subflow_0)
6659 * flow_create_split_inner(metadata_subflow_1)
6660 * flow_create_split_inner(metadata_subflow_2)
6662 * This provide flexible way to add new levels of flow splitting.
6663 * The all of successfully created subflows are included to the
6664 * parent flow dev_flow list.
6667 * Pointer to Ethernet device.
6669 * Parent flow structure pointer.
6671 * Flow rule attributes.
6673 * Pattern specification (list terminated by the END pattern item).
6674 * @param[in] actions
6675 * Associated actions (list terminated by the END action).
6676 * @param[in] flow_split_info
6677 * Pointer to flow split info structure.
6679 * Perform verbose error reporting if not NULL.
6681 * 0 on success, negative value otherwise
6684 flow_create_split_outer(struct rte_eth_dev *dev,
6685 struct rte_flow *flow,
6686 const struct rte_flow_attr *attr,
6687 const struct rte_flow_item items[],
6688 const struct rte_flow_action actions[],
6689 struct mlx5_flow_split_info *flow_split_info,
6690 struct rte_flow_error *error)
6694 ret = flow_create_split_sample(dev, flow, attr, items,
6695 actions, flow_split_info, error);
6696 MLX5_ASSERT(ret <= 0);
6700 static inline struct mlx5_flow_tunnel *
6701 flow_tunnel_from_rule(const struct mlx5_flow *flow)
6703 struct mlx5_flow_tunnel *tunnel;
6705 #pragma GCC diagnostic push
6706 #pragma GCC diagnostic ignored "-Wcast-qual"
6707 tunnel = (typeof(tunnel))flow->tunnel;
6708 #pragma GCC diagnostic pop
6714 * Adjust flow RSS workspace if needed.
6717 * Pointer to thread flow work space.
6719 * Pointer to RSS descriptor.
6720 * @param[in] nrssq_num
6721 * New RSS queue number.
6724 * 0 on success, -1 otherwise and rte_errno is set.
6727 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
6728 struct mlx5_flow_rss_desc *rss_desc,
6731 if (likely(nrssq_num <= wks->rssq_num))
6733 rss_desc->queue = realloc(rss_desc->queue,
6734 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2));
6735 if (!rss_desc->queue) {
6739 wks->rssq_num = RTE_ALIGN(nrssq_num, 2);
6744 * Create a flow and add it to @p list.
6747 * Pointer to Ethernet device.
6749 * Pointer to a TAILQ flow list. If this parameter NULL,
6750 * no list insertion occurred, flow is just created,
6751 * this is caller's responsibility to track the
6754 * Flow rule attributes.
6756 * Pattern specification (list terminated by the END pattern item).
6757 * @param[in] actions
6758 * Associated actions (list terminated by the END action).
6759 * @param[in] external
6760 * This flow rule is created by request external to PMD.
6762 * Perform verbose error reporting if not NULL.
6765 * A flow index on success, 0 otherwise and rte_errno is set.
6768 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
6769 const struct rte_flow_attr *attr,
6770 const struct rte_flow_item items[],
6771 const struct rte_flow_action original_actions[],
6772 bool external, struct rte_flow_error *error)
6774 struct mlx5_priv *priv = dev->data->dev_private;
6775 struct rte_flow *flow = NULL;
6776 struct mlx5_flow *dev_flow;
6777 const struct rte_flow_action_rss *rss = NULL;
6778 struct mlx5_translated_action_handle
6779 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
6780 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
6782 struct mlx5_flow_expand_rss buf;
6783 uint8_t buffer[4096];
6786 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6787 uint8_t buffer[2048];
6790 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6791 uint8_t buffer[2048];
6792 } actions_hairpin_tx;
6794 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
6795 uint8_t buffer[2048];
6797 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
6798 struct mlx5_flow_rss_desc *rss_desc;
6799 const struct rte_flow_action *p_actions_rx;
6803 struct rte_flow_attr attr_tx = { .priority = 0 };
6804 const struct rte_flow_action *actions;
6805 struct rte_flow_action *translated_actions = NULL;
6806 struct mlx5_flow_tunnel *tunnel;
6807 struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
6808 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace();
6809 struct mlx5_flow_split_info flow_split_info = {
6810 .external = !!external,
6820 rss_desc = &wks->rss_desc;
6821 ret = flow_action_handles_translate(dev, original_actions,
6824 &translated_actions, error);
6826 MLX5_ASSERT(translated_actions == NULL);
6829 actions = translated_actions ? translated_actions : original_actions;
6830 p_actions_rx = actions;
6831 hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
6832 ret = flow_drv_validate(dev, attr, items, p_actions_rx,
6833 external, hairpin_flow, error);
6835 goto error_before_hairpin_split;
6836 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx);
6839 goto error_before_hairpin_split;
6841 if (hairpin_flow > 0) {
6842 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
6844 goto error_before_hairpin_split;
6846 flow_hairpin_split(dev, actions, actions_rx.actions,
6847 actions_hairpin_tx.actions, items_tx.items,
6849 p_actions_rx = actions_rx.actions;
6851 flow_split_info.flow_idx = idx;
6852 flow->drv_type = flow_get_drv_type(dev, attr);
6853 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
6854 flow->drv_type < MLX5_FLOW_TYPE_MAX);
6855 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
6856 /* RSS Action only works on NIC RX domain */
6857 if (attr->ingress && !attr->transfer)
6858 rss = flow_get_rss_action(dev, p_actions_rx);
6860 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num))
6863 * The following information is required by
6864 * mlx5_flow_hashfields_adjust() in advance.
6866 rss_desc->level = rss->level;
6867 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */
6868 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types;
6870 flow->dev_handles = 0;
6871 if (rss && rss->types) {
6872 unsigned int graph_root;
6874 graph_root = find_graph_root(rss->level);
6875 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
6877 mlx5_support_expansion, graph_root);
6878 MLX5_ASSERT(ret > 0 &&
6879 (unsigned int)ret < sizeof(expand_buffer.buffer));
6880 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) {
6881 for (i = 0; i < buf->entries; ++i)
6882 mlx5_dbg__print_pattern(buf->entry[i].pattern);
6886 buf->entry[0].pattern = (void *)(uintptr_t)items;
6888 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions,
6890 for (i = 0; i < buf->entries; ++i) {
6891 /* Initialize flow split data. */
6892 flow_split_info.prefix_layers = 0;
6893 flow_split_info.prefix_mark = 0;
6894 flow_split_info.skip_scale = 0;
6896 * The splitter may create multiple dev_flows,
6897 * depending on configuration. In the simplest
6898 * case it just creates unmodified original flow.
6900 ret = flow_create_split_outer(dev, flow, attr,
6901 buf->entry[i].pattern,
6902 p_actions_rx, &flow_split_info,
6906 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) {
6907 ret = flow_tunnel_add_default_miss(dev, flow, attr,
6910 wks->flows[0].tunnel,
6914 mlx5_free(default_miss_ctx.queue);
6919 /* Create the tx flow. */
6921 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
6922 attr_tx.ingress = 0;
6924 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
6925 actions_hairpin_tx.actions,
6929 dev_flow->flow = flow;
6930 dev_flow->external = 0;
6931 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
6932 dev_flow->handle, next);
6933 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
6935 actions_hairpin_tx.actions, error);
6940 * Update the metadata register copy table. If extensive
6941 * metadata feature is enabled and registers are supported
6942 * we might create the extra rte_flow for each unique
6943 * MARK/FLAG action ID.
6945 * The table is updated for ingress Flows only, because
6946 * the egress Flows belong to the different device and
6947 * copy table should be updated in peer NIC Rx domain.
6949 if (attr->ingress &&
6950 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
6951 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
6956 * If the flow is external (from application) OR device is started,
6957 * OR mreg discover, then apply immediately.
6959 if (external || dev->data->dev_started ||
6960 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP &&
6961 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) {
6962 ret = flow_drv_apply(dev, flow, error);
6967 flow_rxq_flags_set(dev, flow);
6968 rte_free(translated_actions);
6969 tunnel = flow_tunnel_from_rule(wks->flows);
6972 flow->tunnel_id = tunnel->tunnel_id;
6973 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED);
6974 mlx5_free(default_miss_ctx.queue);
6976 mlx5_flow_pop_thread_workspace();
6980 ret = rte_errno; /* Save rte_errno before cleanup. */
6981 flow_mreg_del_copy_action(dev, flow);
6982 flow_drv_destroy(dev, flow);
6983 if (rss_desc->shared_rss)
6984 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *)
6986 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
6987 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
6988 mlx5_ipool_free(priv->flows[type], idx);
6989 rte_errno = ret; /* Restore rte_errno. */
6992 mlx5_flow_pop_thread_workspace();
6993 error_before_hairpin_split:
6994 rte_free(translated_actions);
6999 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
7000 * incoming packets to table 1.
7002 * Other flow rules, requested for group n, will be created in
7003 * e-switch table n+1.
7004 * Jump action to e-switch group n will be created to group n+1.
7006 * Used when working in switchdev mode, to utilise advantages of table 1
7010 * Pointer to Ethernet device.
7013 * Pointer to flow on success, NULL otherwise and rte_errno is set.
7016 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
7018 const struct rte_flow_attr attr = {
7025 const struct rte_flow_item pattern = {
7026 .type = RTE_FLOW_ITEM_TYPE_END,
7028 struct rte_flow_action_jump jump = {
7031 const struct rte_flow_action actions[] = {
7033 .type = RTE_FLOW_ACTION_TYPE_JUMP,
7037 .type = RTE_FLOW_ACTION_TYPE_END,
7040 struct rte_flow_error error;
7042 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7044 actions, false, &error);
7048 * Create a dedicated flow rule on e-switch table 1, matches ESW manager
7049 * and sq number, directs all packets to peer vport.
7052 * Pointer to Ethernet device.
7057 * Flow ID on success, 0 otherwise and rte_errno is set.
7060 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq)
7062 struct rte_flow_attr attr = {
7064 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
7069 struct rte_flow_item_port_id port_spec = {
7070 .id = MLX5_PORT_ESW_MGR,
7072 struct mlx5_rte_flow_item_tx_queue txq_spec = {
7075 struct rte_flow_item pattern[] = {
7077 .type = RTE_FLOW_ITEM_TYPE_PORT_ID,
7081 .type = (enum rte_flow_item_type)
7082 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
7086 .type = RTE_FLOW_ITEM_TYPE_END,
7089 struct rte_flow_action_jump jump = {
7092 struct rte_flow_action_port_id port = {
7093 .id = dev->data->port_id,
7095 struct rte_flow_action actions[] = {
7097 .type = RTE_FLOW_ACTION_TYPE_JUMP,
7101 .type = RTE_FLOW_ACTION_TYPE_END,
7104 struct rte_flow_error error;
7107 * Creates group 0, highest priority jump flow.
7108 * Matches txq to bypass kernel packets.
7110 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions,
7111 false, &error) == 0)
7113 /* Create group 1, lowest priority redirect flow for txq. */
7115 actions[0].conf = &port;
7116 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID;
7117 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern,
7118 actions, false, &error);
7122 * Validate a flow supported by the NIC.
7124 * @see rte_flow_validate()
7128 mlx5_flow_validate(struct rte_eth_dev *dev,
7129 const struct rte_flow_attr *attr,
7130 const struct rte_flow_item items[],
7131 const struct rte_flow_action original_actions[],
7132 struct rte_flow_error *error)
7135 struct mlx5_translated_action_handle
7136 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
7137 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
7138 const struct rte_flow_action *actions;
7139 struct rte_flow_action *translated_actions = NULL;
7140 int ret = flow_action_handles_translate(dev, original_actions,
7143 &translated_actions, error);
7147 actions = translated_actions ? translated_actions : original_actions;
7148 hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
7149 ret = flow_drv_validate(dev, attr, items, actions,
7150 true, hairpin_flow, error);
7151 rte_free(translated_actions);
7158 * @see rte_flow_create()
7162 mlx5_flow_create(struct rte_eth_dev *dev,
7163 const struct rte_flow_attr *attr,
7164 const struct rte_flow_item items[],
7165 const struct rte_flow_action actions[],
7166 struct rte_flow_error *error)
7168 struct mlx5_priv *priv = dev->data->dev_private;
7170 if (priv->sh->config.dv_flow_en == 2) {
7171 rte_flow_error_set(error, ENOTSUP,
7172 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7174 "Flow non-Q creation not supported");
7178 * If the device is not started yet, it is not allowed to created a
7179 * flow from application. PMD default flows and traffic control flows
7182 if (unlikely(!dev->data->dev_started)) {
7183 DRV_LOG(DEBUG, "port %u is not started when "
7184 "inserting a flow", dev->data->port_id);
7185 rte_flow_error_set(error, ENODEV,
7186 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7188 "port not started");
7192 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN,
7193 attr, items, actions,
7198 * Destroy a flow in a list.
7201 * Pointer to Ethernet device.
7202 * @param[in] flow_idx
7203 * Index of flow to destroy.
7206 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
7209 struct mlx5_priv *priv = dev->data->dev_private;
7210 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx);
7214 MLX5_ASSERT(flow->type == type);
7216 * Update RX queue flags only if port is started, otherwise it is
7219 if (dev->data->dev_started)
7220 flow_rxq_flags_trim(dev, flow);
7221 flow_drv_destroy(dev, flow);
7223 struct mlx5_flow_tunnel *tunnel;
7225 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
7227 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
7228 mlx5_flow_tunnel_free(dev, tunnel);
7230 flow_mreg_del_copy_action(dev, flow);
7231 mlx5_ipool_free(priv->flows[type], flow_idx);
7235 * Destroy all flows.
7238 * Pointer to Ethernet device.
7240 * Flow type to be flushed.
7242 * If flushing is called actively.
7245 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
7248 struct mlx5_priv *priv = dev->data->dev_private;
7249 uint32_t num_flushed = 0, fidx = 1;
7250 struct rte_flow *flow;
7252 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
7253 if (priv->sh->config.dv_flow_en == 2 &&
7254 type == MLX5_FLOW_TYPE_GEN) {
7255 flow_hw_q_flow_flush(dev, NULL);
7260 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) {
7261 flow_list_destroy(dev, type, fidx);
7265 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
7266 dev->data->port_id, num_flushed);
7271 * Stop all default actions for flows.
7274 * Pointer to Ethernet device.
7277 mlx5_flow_stop_default(struct rte_eth_dev *dev)
7279 flow_mreg_del_default_copy_action(dev);
7280 flow_rxq_flags_clear(dev);
7284 * Start all default actions for flows.
7287 * Pointer to Ethernet device.
7289 * 0 on success, a negative errno value otherwise and rte_errno is set.
7292 mlx5_flow_start_default(struct rte_eth_dev *dev)
7294 struct rte_flow_error error;
7296 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
7297 return flow_mreg_add_default_copy_action(dev, &error);
7301 * Release key of thread specific flow workspace data.
7304 flow_release_workspace(void *data)
7306 struct mlx5_flow_workspace *wks = data;
7307 struct mlx5_flow_workspace *next;
7311 free(wks->rss_desc.queue);
7318 * Get thread specific current flow workspace.
7320 * @return pointer to thread specific flow workspace data, NULL on error.
7322 struct mlx5_flow_workspace*
7323 mlx5_flow_get_thread_workspace(void)
7325 struct mlx5_flow_workspace *data;
7327 data = mlx5_flow_os_get_specific_workspace();
7328 MLX5_ASSERT(data && data->inuse);
7329 if (!data || !data->inuse)
7330 DRV_LOG(ERR, "flow workspace not initialized.");
7335 * Allocate and init new flow workspace.
7337 * @return pointer to flow workspace data, NULL on error.
7339 static struct mlx5_flow_workspace*
7340 flow_alloc_thread_workspace(void)
7342 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data));
7345 DRV_LOG(ERR, "Failed to allocate flow workspace "
7349 data->rss_desc.queue = calloc(1,
7350 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM);
7351 if (!data->rss_desc.queue)
7353 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM;
7356 free(data->rss_desc.queue);
7362 * Get new thread specific flow workspace.
7364 * If current workspace inuse, create new one and set as current.
7366 * @return pointer to thread specific flow workspace data, NULL on error.
7368 static struct mlx5_flow_workspace*
7369 mlx5_flow_push_thread_workspace(void)
7371 struct mlx5_flow_workspace *curr;
7372 struct mlx5_flow_workspace *data;
7374 curr = mlx5_flow_os_get_specific_workspace();
7376 data = flow_alloc_thread_workspace();
7379 } else if (!curr->inuse) {
7381 } else if (curr->next) {
7384 data = flow_alloc_thread_workspace();
7392 /* Set as current workspace */
7393 if (mlx5_flow_os_set_specific_workspace(data))
7394 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
7399 * Close current thread specific flow workspace.
7401 * If previous workspace available, set it as current.
7403 * @return pointer to thread specific flow workspace data, NULL on error.
7406 mlx5_flow_pop_thread_workspace(void)
7408 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace();
7413 DRV_LOG(ERR, "Failed to close unused flow workspace.");
7419 if (mlx5_flow_os_set_specific_workspace(data->prev))
7420 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
7424 * Verify the flow list is empty
7427 * Pointer to Ethernet device.
7429 * @return the number of flows not released.
7432 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused)
7434 struct mlx5_priv *priv = dev->data->dev_private;
7435 struct rte_flow *flow;
7439 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
7440 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) {
7441 DRV_LOG(DEBUG, "port %u flow %p still referenced",
7442 dev->data->port_id, (void *)flow);
7450 * Enable default hairpin egress flow.
7453 * Pointer to Ethernet device.
7458 * 0 on success, a negative errno value otherwise and rte_errno is set.
7461 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
7464 const struct rte_flow_attr attr = {
7468 struct mlx5_rte_flow_item_tx_queue queue_spec = {
7471 struct mlx5_rte_flow_item_tx_queue queue_mask = {
7472 .queue = UINT32_MAX,
7474 struct rte_flow_item items[] = {
7476 .type = (enum rte_flow_item_type)
7477 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
7478 .spec = &queue_spec,
7480 .mask = &queue_mask,
7483 .type = RTE_FLOW_ITEM_TYPE_END,
7486 struct rte_flow_action_jump jump = {
7487 .group = MLX5_HAIRPIN_TX_TABLE,
7489 struct rte_flow_action actions[2];
7491 struct rte_flow_error error;
7493 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
7494 actions[0].conf = &jump;
7495 actions[1].type = RTE_FLOW_ACTION_TYPE_END;
7496 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7497 &attr, items, actions, false, &error);
7500 "Failed to create ctrl flow: rte_errno(%d),"
7501 " type(%d), message(%s)",
7502 rte_errno, error.type,
7503 error.message ? error.message : " (no stated reason)");
7510 * Enable a control flow configured from the control plane.
7513 * Pointer to Ethernet device.
7515 * An Ethernet flow spec to apply.
7517 * An Ethernet flow mask to apply.
7519 * A VLAN flow spec to apply.
7521 * A VLAN flow mask to apply.
7524 * 0 on success, a negative errno value otherwise and rte_errno is set.
7527 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
7528 struct rte_flow_item_eth *eth_spec,
7529 struct rte_flow_item_eth *eth_mask,
7530 struct rte_flow_item_vlan *vlan_spec,
7531 struct rte_flow_item_vlan *vlan_mask)
7533 struct mlx5_priv *priv = dev->data->dev_private;
7534 const struct rte_flow_attr attr = {
7536 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
7538 struct rte_flow_item items[] = {
7540 .type = RTE_FLOW_ITEM_TYPE_ETH,
7546 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
7547 RTE_FLOW_ITEM_TYPE_END,
7553 .type = RTE_FLOW_ITEM_TYPE_END,
7556 uint16_t queue[priv->reta_idx_n];
7557 struct rte_flow_action_rss action_rss = {
7558 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
7560 .types = priv->rss_conf.rss_hf,
7561 .key_len = priv->rss_conf.rss_key_len,
7562 .queue_num = priv->reta_idx_n,
7563 .key = priv->rss_conf.rss_key,
7566 struct rte_flow_action actions[] = {
7568 .type = RTE_FLOW_ACTION_TYPE_RSS,
7569 .conf = &action_rss,
7572 .type = RTE_FLOW_ACTION_TYPE_END,
7576 struct rte_flow_error error;
7579 if (!priv->reta_idx_n || !priv->rxqs_n) {
7582 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG))
7583 action_rss.types = 0;
7584 for (i = 0; i != priv->reta_idx_n; ++i)
7585 queue[i] = (*priv->reta_idx)[i];
7586 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7587 &attr, items, actions, false, &error);
7594 * Enable a flow control configured from the control plane.
7597 * Pointer to Ethernet device.
7599 * An Ethernet flow spec to apply.
7601 * An Ethernet flow mask to apply.
7604 * 0 on success, a negative errno value otherwise and rte_errno is set.
7607 mlx5_ctrl_flow(struct rte_eth_dev *dev,
7608 struct rte_flow_item_eth *eth_spec,
7609 struct rte_flow_item_eth *eth_mask)
7611 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
7615 * Create default miss flow rule matching lacp traffic
7618 * Pointer to Ethernet device.
7620 * An Ethernet flow spec to apply.
7623 * 0 on success, a negative errno value otherwise and rte_errno is set.
7626 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
7629 * The LACP matching is done by only using ether type since using
7630 * a multicast dst mac causes kernel to give low priority to this flow.
7632 static const struct rte_flow_item_eth lacp_spec = {
7633 .type = RTE_BE16(0x8809),
7635 static const struct rte_flow_item_eth lacp_mask = {
7638 const struct rte_flow_attr attr = {
7641 struct rte_flow_item items[] = {
7643 .type = RTE_FLOW_ITEM_TYPE_ETH,
7648 .type = RTE_FLOW_ITEM_TYPE_END,
7651 struct rte_flow_action actions[] = {
7653 .type = (enum rte_flow_action_type)
7654 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
7657 .type = RTE_FLOW_ACTION_TYPE_END,
7660 struct rte_flow_error error;
7661 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7662 &attr, items, actions,
7673 * @see rte_flow_destroy()
7677 mlx5_flow_destroy(struct rte_eth_dev *dev,
7678 struct rte_flow *flow,
7679 struct rte_flow_error *error __rte_unused)
7681 struct mlx5_priv *priv = dev->data->dev_private;
7683 if (priv->sh->config.dv_flow_en == 2)
7684 return rte_flow_error_set(error, ENOTSUP,
7685 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7687 "Flow non-Q destruction not supported");
7688 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
7689 (uintptr_t)(void *)flow);
7694 * Destroy all flows.
7696 * @see rte_flow_flush()
7700 mlx5_flow_flush(struct rte_eth_dev *dev,
7701 struct rte_flow_error *error __rte_unused)
7703 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false);
7710 * @see rte_flow_isolate()
7714 mlx5_flow_isolate(struct rte_eth_dev *dev,
7716 struct rte_flow_error *error)
7718 struct mlx5_priv *priv = dev->data->dev_private;
7720 if (dev->data->dev_started) {
7721 rte_flow_error_set(error, EBUSY,
7722 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7724 "port must be stopped first");
7727 priv->isolated = !!enable;
7729 dev->dev_ops = &mlx5_dev_ops_isolate;
7731 dev->dev_ops = &mlx5_dev_ops;
7733 dev->rx_descriptor_status = mlx5_rx_descriptor_status;
7734 dev->tx_descriptor_status = mlx5_tx_descriptor_status;
7742 * @see rte_flow_query()
7746 flow_drv_query(struct rte_eth_dev *dev,
7748 const struct rte_flow_action *actions,
7750 struct rte_flow_error *error)
7752 struct mlx5_priv *priv = dev->data->dev_private;
7753 const struct mlx5_flow_driver_ops *fops;
7754 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
7756 enum mlx5_flow_drv_type ftype;
7759 return rte_flow_error_set(error, ENOENT,
7760 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7762 "invalid flow handle");
7764 ftype = flow->drv_type;
7765 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
7766 fops = flow_get_drv_ops(ftype);
7768 return fops->query(dev, flow, actions, data, error);
7774 * @see rte_flow_query()
7778 mlx5_flow_query(struct rte_eth_dev *dev,
7779 struct rte_flow *flow,
7780 const struct rte_flow_action *actions,
7782 struct rte_flow_error *error)
7785 struct mlx5_priv *priv = dev->data->dev_private;
7787 if (priv->sh->config.dv_flow_en == 2)
7788 return rte_flow_error_set(error, ENOTSUP,
7789 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7791 "Flow non-Q query not supported");
7792 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
7800 * Get rte_flow callbacks.
7803 * Pointer to Ethernet device structure.
7805 * Pointer to operation-specific structure.
7810 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused,
7811 const struct rte_flow_ops **ops)
7813 *ops = &mlx5_flow_ops;
7818 * Validate meter policy actions.
7819 * Dispatcher for action type specific validation.
7822 * Pointer to the Ethernet device structure.
7824 * The meter policy action object to validate.
7826 * Attributes of flow to determine steering domain.
7827 * @param[out] is_rss
7829 * @param[out] domain_bitmap
7831 * @param[out] is_def_policy
7832 * Is default policy or not.
7834 * Perform verbose error reporting if not NULL. Initialized in case of
7838 * 0 on success, otherwise negative errno value.
7841 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev,
7842 const struct rte_flow_action *actions[RTE_COLORS],
7843 struct rte_flow_attr *attr,
7845 uint8_t *domain_bitmap,
7846 uint8_t *policy_mode,
7847 struct rte_mtr_error *error)
7849 const struct mlx5_flow_driver_ops *fops;
7851 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7852 return fops->validate_mtr_acts(dev, actions, attr, is_rss,
7853 domain_bitmap, policy_mode, error);
7857 * Destroy the meter table set.
7860 * Pointer to Ethernet device.
7861 * @param[in] mtr_policy
7862 * Meter policy struct.
7865 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev,
7866 struct mlx5_flow_meter_policy *mtr_policy)
7868 const struct mlx5_flow_driver_ops *fops;
7870 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7871 fops->destroy_mtr_acts(dev, mtr_policy);
7875 * Create policy action, lock free,
7876 * (mutex should be acquired by caller).
7877 * Dispatcher for action type specific call.
7880 * Pointer to the Ethernet device structure.
7881 * @param[in] mtr_policy
7882 * Meter policy struct.
7884 * Action specification used to create meter actions.
7886 * Perform verbose error reporting if not NULL. Initialized in case of
7890 * 0 on success, otherwise negative errno value.
7893 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev,
7894 struct mlx5_flow_meter_policy *mtr_policy,
7895 const struct rte_flow_action *actions[RTE_COLORS],
7896 struct rte_mtr_error *error)
7898 const struct mlx5_flow_driver_ops *fops;
7900 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7901 return fops->create_mtr_acts(dev, mtr_policy, actions, error);
7905 * Create policy rules, lock free,
7906 * (mutex should be acquired by caller).
7907 * Dispatcher for action type specific call.
7910 * Pointer to the Ethernet device structure.
7911 * @param[in] mtr_policy
7912 * Meter policy struct.
7915 * 0 on success, -1 otherwise.
7918 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev,
7919 struct mlx5_flow_meter_policy *mtr_policy)
7921 const struct mlx5_flow_driver_ops *fops;
7923 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7924 return fops->create_policy_rules(dev, mtr_policy);
7928 * Destroy policy rules, lock free,
7929 * (mutex should be acquired by caller).
7930 * Dispatcher for action type specific call.
7933 * Pointer to the Ethernet device structure.
7934 * @param[in] mtr_policy
7935 * Meter policy struct.
7938 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev,
7939 struct mlx5_flow_meter_policy *mtr_policy)
7941 const struct mlx5_flow_driver_ops *fops;
7943 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7944 fops->destroy_policy_rules(dev, mtr_policy);
7948 * Destroy the default policy table set.
7951 * Pointer to Ethernet device.
7954 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev)
7956 const struct mlx5_flow_driver_ops *fops;
7958 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7959 fops->destroy_def_policy(dev);
7963 * Destroy the default policy table set.
7966 * Pointer to Ethernet device.
7969 * 0 on success, -1 otherwise.
7972 mlx5_flow_create_def_policy(struct rte_eth_dev *dev)
7974 const struct mlx5_flow_driver_ops *fops;
7976 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7977 return fops->create_def_policy(dev);
7981 * Create the needed meter and suffix tables.
7984 * Pointer to Ethernet device.
7987 * 0 on success, -1 otherwise.
7990 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
7991 struct mlx5_flow_meter_info *fm,
7993 uint8_t domain_bitmap)
7995 const struct mlx5_flow_driver_ops *fops;
7997 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7998 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap);
8002 * Destroy the meter table set.
8005 * Pointer to Ethernet device.
8007 * Pointer to the meter table set.
8010 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
8011 struct mlx5_flow_meter_info *fm)
8013 const struct mlx5_flow_driver_ops *fops;
8015 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8016 fops->destroy_mtr_tbls(dev, fm);
8020 * Destroy the global meter drop table.
8023 * Pointer to Ethernet device.
8026 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
8028 const struct mlx5_flow_driver_ops *fops;
8030 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8031 fops->destroy_mtr_drop_tbls(dev);
8035 * Destroy the sub policy table with RX queue.
8038 * Pointer to Ethernet device.
8039 * @param[in] mtr_policy
8040 * Pointer to meter policy table.
8043 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
8044 struct mlx5_flow_meter_policy *mtr_policy)
8046 const struct mlx5_flow_driver_ops *fops;
8048 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8049 fops->destroy_sub_policy_with_rxq(dev, mtr_policy);
8053 * Allocate the needed aso flow meter id.
8056 * Pointer to Ethernet device.
8059 * Index to aso flow meter on success, NULL otherwise.
8062 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev)
8064 const struct mlx5_flow_driver_ops *fops;
8066 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8067 return fops->create_meter(dev);
8071 * Free the aso flow meter id.
8074 * Pointer to Ethernet device.
8075 * @param[in] mtr_idx
8076 * Index to aso flow meter to be free.
8082 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx)
8084 const struct mlx5_flow_driver_ops *fops;
8086 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8087 fops->free_meter(dev, mtr_idx);
8091 * Allocate a counter.
8094 * Pointer to Ethernet device structure.
8097 * Index to allocated counter on success, 0 otherwise.
8100 mlx5_counter_alloc(struct rte_eth_dev *dev)
8102 const struct mlx5_flow_driver_ops *fops;
8103 struct rte_flow_attr attr = { .transfer = 0 };
8105 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
8106 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8107 return fops->counter_alloc(dev);
8110 "port %u counter allocate is not supported.",
8111 dev->data->port_id);
8119 * Pointer to Ethernet device structure.
8121 * Index to counter to be free.
8124 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
8126 const struct mlx5_flow_driver_ops *fops;
8127 struct rte_flow_attr attr = { .transfer = 0 };
8129 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
8130 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8131 fops->counter_free(dev, cnt);
8135 "port %u counter free is not supported.",
8136 dev->data->port_id);
8140 * Query counter statistics.
8143 * Pointer to Ethernet device structure.
8145 * Index to counter to query.
8147 * Set to clear counter statistics.
8149 * The counter hits packets number to save.
8151 * The counter hits bytes number to save.
8154 * 0 on success, a negative errno value otherwise.
8157 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
8158 bool clear, uint64_t *pkts, uint64_t *bytes, void **action)
8160 const struct mlx5_flow_driver_ops *fops;
8161 struct rte_flow_attr attr = { .transfer = 0 };
8163 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
8164 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8165 return fops->counter_query(dev, cnt, clear, pkts,
8169 "port %u counter query is not supported.",
8170 dev->data->port_id);
8175 * Get information about HWS pre-configurable resources.
8178 * Pointer to the rte_eth_dev structure.
8179 * @param[out] port_info
8180 * Pointer to port information.
8181 * @param[out] queue_info
8182 * Pointer to queue information.
8184 * Pointer to error structure.
8187 * 0 on success, a negative errno value otherwise and rte_errno is set.
8190 mlx5_flow_info_get(struct rte_eth_dev *dev,
8191 struct rte_flow_port_info *port_info,
8192 struct rte_flow_queue_info *queue_info,
8193 struct rte_flow_error *error)
8195 const struct mlx5_flow_driver_ops *fops;
8197 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8198 return rte_flow_error_set(error, ENOTSUP,
8199 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8201 "info get with incorrect steering mode");
8202 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8203 return fops->info_get(dev, port_info, queue_info, error);
8207 * Configure port HWS resources.
8210 * Pointer to the rte_eth_dev structure.
8211 * @param[in] port_attr
8212 * Port configuration attributes.
8213 * @param[in] nb_queue
8215 * @param[in] queue_attr
8216 * Array that holds attributes for each flow queue.
8218 * Pointer to error structure.
8221 * 0 on success, a negative errno value otherwise and rte_errno is set.
8224 mlx5_flow_port_configure(struct rte_eth_dev *dev,
8225 const struct rte_flow_port_attr *port_attr,
8227 const struct rte_flow_queue_attr *queue_attr[],
8228 struct rte_flow_error *error)
8230 const struct mlx5_flow_driver_ops *fops;
8232 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8233 return rte_flow_error_set(error, ENOTSUP,
8234 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8236 "port configure with incorrect steering mode");
8237 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8238 return fops->configure(dev, port_attr, nb_queue, queue_attr, error);
8242 * Create flow item template.
8245 * Pointer to the rte_eth_dev structure.
8247 * Pointer to the item template attributes.
8249 * The template item pattern.
8251 * Pointer to error structure.
8254 * 0 on success, a negative errno value otherwise and rte_errno is set.
8256 static struct rte_flow_pattern_template *
8257 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev,
8258 const struct rte_flow_pattern_template_attr *attr,
8259 const struct rte_flow_item items[],
8260 struct rte_flow_error *error)
8262 const struct mlx5_flow_driver_ops *fops;
8264 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) {
8265 rte_flow_error_set(error, ENOTSUP,
8266 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8268 "pattern create with incorrect steering mode");
8271 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8272 return fops->pattern_template_create(dev, attr, items, error);
8276 * Destroy flow item template.
8279 * Pointer to the rte_eth_dev structure.
8280 * @param[in] template
8281 * Pointer to the item template to be destroyed.
8283 * Pointer to error structure.
8286 * 0 on success, a negative errno value otherwise and rte_errno is set.
8289 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev,
8290 struct rte_flow_pattern_template *template,
8291 struct rte_flow_error *error)
8293 const struct mlx5_flow_driver_ops *fops;
8295 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8296 return rte_flow_error_set(error, ENOTSUP,
8297 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8299 "pattern destroy with incorrect steering mode");
8300 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8301 return fops->pattern_template_destroy(dev, template, error);
8305 * Create flow item template.
8308 * Pointer to the rte_eth_dev structure.
8310 * Pointer to the action template attributes.
8311 * @param[in] actions
8312 * Associated actions (list terminated by the END action).
8314 * List of actions that marks which of the action's member is constant.
8316 * Pointer to error structure.
8319 * 0 on success, a negative errno value otherwise and rte_errno is set.
8321 static struct rte_flow_actions_template *
8322 mlx5_flow_actions_template_create(struct rte_eth_dev *dev,
8323 const struct rte_flow_actions_template_attr *attr,
8324 const struct rte_flow_action actions[],
8325 const struct rte_flow_action masks[],
8326 struct rte_flow_error *error)
8328 const struct mlx5_flow_driver_ops *fops;
8330 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) {
8331 rte_flow_error_set(error, ENOTSUP,
8332 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8334 "action create with incorrect steering mode");
8337 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8338 return fops->actions_template_create(dev, attr, actions, masks, error);
8342 * Destroy flow action template.
8345 * Pointer to the rte_eth_dev structure.
8346 * @param[in] template
8347 * Pointer to the action template to be destroyed.
8349 * Pointer to error structure.
8352 * 0 on success, a negative errno value otherwise and rte_errno is set.
8355 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev,
8356 struct rte_flow_actions_template *template,
8357 struct rte_flow_error *error)
8359 const struct mlx5_flow_driver_ops *fops;
8361 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8362 return rte_flow_error_set(error, ENOTSUP,
8363 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8365 "action destroy with incorrect steering mode");
8366 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8367 return fops->actions_template_destroy(dev, template, error);
8371 * Create flow table.
8374 * Pointer to the rte_eth_dev structure.
8376 * Pointer to the table attributes.
8377 * @param[in] item_templates
8378 * Item template array to be binded to the table.
8379 * @param[in] nb_item_templates
8380 * Number of item template.
8381 * @param[in] action_templates
8382 * Action template array to be binded to the table.
8383 * @param[in] nb_action_templates
8384 * Number of action template.
8386 * Pointer to error structure.
8389 * Table on success, NULL otherwise and rte_errno is set.
8391 static struct rte_flow_template_table *
8392 mlx5_flow_table_create(struct rte_eth_dev *dev,
8393 const struct rte_flow_template_table_attr *attr,
8394 struct rte_flow_pattern_template *item_templates[],
8395 uint8_t nb_item_templates,
8396 struct rte_flow_actions_template *action_templates[],
8397 uint8_t nb_action_templates,
8398 struct rte_flow_error *error)
8400 const struct mlx5_flow_driver_ops *fops;
8402 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) {
8403 rte_flow_error_set(error, ENOTSUP,
8404 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8406 "table create with incorrect steering mode");
8409 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8410 return fops->template_table_create(dev,
8415 nb_action_templates,
8420 * PMD destroy flow table.
8423 * Pointer to the rte_eth_dev structure.
8425 * Pointer to the table to be destroyed.
8427 * Pointer to error structure.
8430 * 0 on success, a negative errno value otherwise and rte_errno is set.
8433 mlx5_flow_table_destroy(struct rte_eth_dev *dev,
8434 struct rte_flow_template_table *table,
8435 struct rte_flow_error *error)
8437 const struct mlx5_flow_driver_ops *fops;
8439 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8440 return rte_flow_error_set(error, ENOTSUP,
8441 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8443 "table destroy with incorrect steering mode");
8444 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8445 return fops->template_table_destroy(dev, table, error);
8449 * Enqueue flow creation.
8452 * Pointer to the rte_eth_dev structure.
8453 * @param[in] queue_id
8454 * The queue to create the flow.
8456 * Pointer to the flow operation attributes.
8458 * Items with flow spec value.
8459 * @param[in] pattern_template_index
8460 * The item pattern flow follows from the table.
8461 * @param[in] actions
8462 * Action with flow spec value.
8463 * @param[in] action_template_index
8464 * The action pattern flow follows from the table.
8465 * @param[in] user_data
8466 * Pointer to the user_data.
8468 * Pointer to error structure.
8471 * Flow pointer on success, NULL otherwise and rte_errno is set.
8473 static struct rte_flow *
8474 mlx5_flow_async_flow_create(struct rte_eth_dev *dev,
8476 const struct rte_flow_op_attr *attr,
8477 struct rte_flow_template_table *table,
8478 const struct rte_flow_item items[],
8479 uint8_t pattern_template_index,
8480 const struct rte_flow_action actions[],
8481 uint8_t action_template_index,
8483 struct rte_flow_error *error)
8485 const struct mlx5_flow_driver_ops *fops;
8487 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) {
8488 rte_flow_error_set(error, ENOTSUP,
8489 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8491 "flow_q create with incorrect steering mode");
8494 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8495 return fops->async_flow_create(dev, queue_id, attr, table,
8496 items, pattern_template_index,
8497 actions, action_template_index,
8502 * Enqueue flow destruction.
8505 * Pointer to the rte_eth_dev structure.
8507 * The queue to destroy the flow.
8509 * Pointer to the flow operation attributes.
8511 * Pointer to the flow to be destroyed.
8512 * @param[in] user_data
8513 * Pointer to the user_data.
8515 * Pointer to error structure.
8518 * 0 on success, negative value otherwise and rte_errno is set.
8521 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev,
8523 const struct rte_flow_op_attr *attr,
8524 struct rte_flow *flow,
8526 struct rte_flow_error *error)
8528 const struct mlx5_flow_driver_ops *fops;
8530 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8531 return rte_flow_error_set(error, ENOTSUP,
8532 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8534 "flow_q destroy with incorrect steering mode");
8535 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8536 return fops->async_flow_destroy(dev, queue, attr, flow,
8541 * Pull the enqueued flows.
8544 * Pointer to the rte_eth_dev structure.
8546 * The queue to pull the result.
8547 * @param[in/out] res
8548 * Array to save the results.
8550 * Available result with the array.
8552 * Pointer to error structure.
8555 * Result number on success, negative value otherwise and rte_errno is set.
8558 mlx5_flow_pull(struct rte_eth_dev *dev,
8560 struct rte_flow_op_result res[],
8562 struct rte_flow_error *error)
8564 const struct mlx5_flow_driver_ops *fops;
8566 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8567 return rte_flow_error_set(error, ENOTSUP,
8568 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8570 "flow_q pull with incorrect steering mode");
8571 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8572 return fops->pull(dev, queue, res, n_res, error);
8576 * Push the enqueued flows.
8579 * Pointer to the rte_eth_dev structure.
8581 * The queue to push the flows.
8583 * Pointer to error structure.
8586 * 0 on success, negative value otherwise and rte_errno is set.
8589 mlx5_flow_push(struct rte_eth_dev *dev,
8591 struct rte_flow_error *error)
8593 const struct mlx5_flow_driver_ops *fops;
8595 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8596 return rte_flow_error_set(error, ENOTSUP,
8597 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8599 "flow_q push with incorrect steering mode");
8600 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8601 return fops->push(dev, queue, error);
8605 * Create shared action.
8608 * Pointer to the rte_eth_dev structure.
8610 * Which queue to be used..
8612 * Operation attribute.
8614 * Indirect action configuration.
8616 * rte_flow action detail.
8617 * @param[in] user_data
8618 * Pointer to the user_data.
8620 * Pointer to error structure.
8623 * Action handle on success, NULL otherwise and rte_errno is set.
8625 static struct rte_flow_action_handle *
8626 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue,
8627 const struct rte_flow_op_attr *attr,
8628 const struct rte_flow_indir_action_conf *conf,
8629 const struct rte_flow_action *action,
8631 struct rte_flow_error *error)
8633 const struct mlx5_flow_driver_ops *fops =
8634 flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8636 return fops->async_action_create(dev, queue, attr, conf, action,
8641 * Update shared action.
8644 * Pointer to the rte_eth_dev structure.
8646 * Which queue to be used..
8648 * Operation attribute.
8650 * Action handle to be updated.
8653 * @param[in] user_data
8654 * Pointer to the user_data.
8656 * Pointer to error structure.
8659 * 0 on success, negative value otherwise and rte_errno is set.
8662 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue,
8663 const struct rte_flow_op_attr *attr,
8664 struct rte_flow_action_handle *handle,
8667 struct rte_flow_error *error)
8669 const struct mlx5_flow_driver_ops *fops =
8670 flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8672 return fops->async_action_update(dev, queue, attr, handle,
8673 update, user_data, error);
8677 * Destroy shared action.
8680 * Pointer to the rte_eth_dev structure.
8682 * Which queue to be used..
8684 * Operation attribute.
8686 * Action handle to be destroyed.
8687 * @param[in] user_data
8688 * Pointer to the user_data.
8690 * Pointer to error structure.
8693 * 0 on success, negative value otherwise and rte_errno is set.
8696 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue,
8697 const struct rte_flow_op_attr *attr,
8698 struct rte_flow_action_handle *handle,
8700 struct rte_flow_error *error)
8702 const struct mlx5_flow_driver_ops *fops =
8703 flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8705 return fops->async_action_destroy(dev, queue, attr, handle,
8710 * Allocate a new memory for the counter values wrapped by all the needed
8714 * Pointer to mlx5_dev_ctx_shared object.
8717 * 0 on success, a negative errno value otherwise.
8720 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
8722 struct mlx5_counter_stats_mem_mng *mem_mng;
8723 volatile struct flow_counter_stats *raw_data;
8724 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
8725 int size = (sizeof(struct flow_counter_stats) *
8726 MLX5_COUNTERS_PER_POOL +
8727 sizeof(struct mlx5_counter_stats_raw)) * raws_n +
8728 sizeof(struct mlx5_counter_stats_mem_mng);
8729 size_t pgsize = rte_mem_page_size();
8734 if (pgsize == (size_t)-1) {
8735 DRV_LOG(ERR, "Failed to get mem page size");
8739 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
8744 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
8745 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
8746 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd,
8747 sh->cdev->pdn, mem, size,
8754 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
8755 raw_data = (volatile struct flow_counter_stats *)mem;
8756 for (i = 0; i < raws_n; ++i) {
8757 mem_mng->raws[i].mem_mng = mem_mng;
8758 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
8760 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
8761 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
8762 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
8764 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
8765 sh->cmng.mem_mng = mem_mng;
8770 * Set the statistic memory to the new counter pool.
8773 * Pointer to mlx5_dev_ctx_shared object.
8775 * Pointer to the pool to set the statistic memory.
8778 * 0 on success, a negative errno value otherwise.
8781 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
8782 struct mlx5_flow_counter_pool *pool)
8784 struct mlx5_flow_counter_mng *cmng = &sh->cmng;
8785 /* Resize statistic memory once used out. */
8786 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
8787 mlx5_flow_create_counter_stat_mem_mng(sh)) {
8788 DRV_LOG(ERR, "Cannot resize counter stat mem.");
8791 rte_spinlock_lock(&pool->sl);
8792 pool->raw = cmng->mem_mng->raws + pool->index %
8793 MLX5_CNT_CONTAINER_RESIZE;
8794 rte_spinlock_unlock(&pool->sl);
8795 pool->raw_hw = NULL;
8799 #define MLX5_POOL_QUERY_FREQ_US 1000000
8802 * Set the periodic procedure for triggering asynchronous batch queries for all
8803 * the counter pools.
8806 * Pointer to mlx5_dev_ctx_shared object.
8809 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
8811 uint32_t pools_n, us;
8813 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED);
8814 us = MLX5_POOL_QUERY_FREQ_US / pools_n;
8815 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
8816 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
8817 sh->cmng.query_thread_on = 0;
8818 DRV_LOG(ERR, "Cannot reinitialize query alarm");
8820 sh->cmng.query_thread_on = 1;
8825 * The periodic procedure for triggering asynchronous batch queries for all the
8826 * counter pools. This function is probably called by the host thread.
8829 * The parameter for the alarm process.
8832 mlx5_flow_query_alarm(void *arg)
8834 struct mlx5_dev_ctx_shared *sh = arg;
8836 uint16_t pool_index = sh->cmng.pool_index;
8837 struct mlx5_flow_counter_mng *cmng = &sh->cmng;
8838 struct mlx5_flow_counter_pool *pool;
8841 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
8843 rte_spinlock_lock(&cmng->pool_update_sl);
8844 pool = cmng->pools[pool_index];
8845 n_valid = cmng->n_valid;
8846 rte_spinlock_unlock(&cmng->pool_update_sl);
8847 /* Set the statistic memory to the new created pool. */
8848 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
8851 /* There is a pool query in progress. */
8854 LIST_FIRST(&sh->cmng.free_stat_raws);
8856 /* No free counter statistics raw memory. */
8859 * Identify the counters released between query trigger and query
8860 * handle more efficiently. The counter released in this gap period
8861 * should wait for a new round of query as the new arrived packets
8862 * will not be taken into account.
8865 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
8866 MLX5_COUNTERS_PER_POOL,
8868 pool->raw_hw->mem_mng->wm.lkey,
8872 (uint64_t)(uintptr_t)pool);
8874 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
8875 " %d", pool->min_dcs->id);
8876 pool->raw_hw = NULL;
8879 LIST_REMOVE(pool->raw_hw, next);
8880 sh->cmng.pending_queries++;
8882 if (pool_index >= n_valid)
8885 sh->cmng.pool_index = pool_index;
8886 mlx5_set_query_alarm(sh);
8890 * Check and callback event for new aged flow in the counter pool
8893 * Pointer to mlx5_dev_ctx_shared object.
8895 * Pointer to Current counter pool.
8898 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
8899 struct mlx5_flow_counter_pool *pool)
8901 struct mlx5_priv *priv;
8902 struct mlx5_flow_counter *cnt;
8903 struct mlx5_age_info *age_info;
8904 struct mlx5_age_param *age_param;
8905 struct mlx5_counter_stats_raw *cur = pool->raw_hw;
8906 struct mlx5_counter_stats_raw *prev = pool->raw;
8907 const uint64_t curr_time = MLX5_CURR_TIME_SEC;
8908 const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
8909 uint16_t expected = AGE_CANDIDATE;
8912 pool->time_of_last_age_check = curr_time;
8913 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
8914 cnt = MLX5_POOL_GET_CNT(pool, i);
8915 age_param = MLX5_CNT_TO_AGE(cnt);
8916 if (__atomic_load_n(&age_param->state,
8917 __ATOMIC_RELAXED) != AGE_CANDIDATE)
8919 if (cur->data[i].hits != prev->data[i].hits) {
8920 __atomic_store_n(&age_param->sec_since_last_hit, 0,
8924 if (__atomic_add_fetch(&age_param->sec_since_last_hit,
8926 __ATOMIC_RELAXED) <= age_param->timeout)
8929 * Hold the lock first, or if between the
8930 * state AGE_TMOUT and tailq operation the
8931 * release happened, the release procedure
8932 * may delete a non-existent tailq node.
8934 priv = rte_eth_devices[age_param->port_id].data->dev_private;
8935 age_info = GET_PORT_AGE_INFO(priv);
8936 rte_spinlock_lock(&age_info->aged_sl);
8937 if (__atomic_compare_exchange_n(&age_param->state, &expected,
8940 __ATOMIC_RELAXED)) {
8941 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
8942 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
8944 rte_spinlock_unlock(&age_info->aged_sl);
8946 mlx5_age_event_prepare(sh);
8950 * Handler for the HW respond about ready values from an asynchronous batch
8951 * query. This function is probably called by the host thread.
8954 * The pointer to the shared device context.
8955 * @param[in] async_id
8956 * The Devx async ID.
8958 * The status of the completion.
8961 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
8962 uint64_t async_id, int status)
8964 struct mlx5_flow_counter_pool *pool =
8965 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
8966 struct mlx5_counter_stats_raw *raw_to_free;
8967 uint8_t query_gen = pool->query_gen ^ 1;
8968 struct mlx5_flow_counter_mng *cmng = &sh->cmng;
8969 enum mlx5_counter_type cnt_type =
8970 pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
8971 MLX5_COUNTER_TYPE_ORIGIN;
8973 if (unlikely(status)) {
8974 raw_to_free = pool->raw_hw;
8976 raw_to_free = pool->raw;
8978 mlx5_flow_aging_check(sh, pool);
8979 rte_spinlock_lock(&pool->sl);
8980 pool->raw = pool->raw_hw;
8981 rte_spinlock_unlock(&pool->sl);
8982 /* Be sure the new raw counters data is updated in memory. */
8984 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
8985 rte_spinlock_lock(&cmng->csl[cnt_type]);
8986 TAILQ_CONCAT(&cmng->counters[cnt_type],
8987 &pool->counters[query_gen], next);
8988 rte_spinlock_unlock(&cmng->csl[cnt_type]);
8991 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
8992 pool->raw_hw = NULL;
8993 sh->cmng.pending_queries--;
8997 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
8998 const struct flow_grp_info *grp_info,
8999 struct rte_flow_error *error)
9001 if (grp_info->transfer && grp_info->external &&
9002 grp_info->fdb_def_rule) {
9003 if (group == UINT32_MAX)
9004 return rte_flow_error_set
9006 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
9008 "group index not supported");
9013 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
9018 * Translate the rte_flow group index to HW table value.
9020 * If tunnel offload is disabled, all group ids converted to flow table
9021 * id using the standard method.
9022 * If tunnel offload is enabled, group id can be converted using the
9023 * standard or tunnel conversion method. Group conversion method
9024 * selection depends on flags in `grp_info` parameter:
9025 * - Internal (grp_info.external == 0) groups conversion uses the
9027 * - Group ids in JUMP action converted with the tunnel conversion.
9028 * - Group id in rule attribute conversion depends on a rule type and
9030 * ** non zero group attributes converted with the tunnel method
9031 * ** zero group attribute in non-tunnel rule is converted using the
9032 * standard method - there's only one root table
9033 * ** zero group attribute in steer tunnel rule is converted with the
9034 * standard method - single root table
9035 * ** zero group attribute in match tunnel rule is a special OvS
9036 * case: that value is used for portability reasons. That group
9037 * id is converted with the tunnel conversion method.
9042 * PMD tunnel offload object
9044 * rte_flow group index value.
9047 * @param[in] grp_info
9048 * flags used for conversion
9050 * Pointer to error structure.
9053 * 0 on success, a negative errno value otherwise and rte_errno is set.
9056 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
9057 const struct mlx5_flow_tunnel *tunnel,
9058 uint32_t group, uint32_t *table,
9059 const struct flow_grp_info *grp_info,
9060 struct rte_flow_error *error)
9063 bool standard_translation;
9065 if (!grp_info->skip_scale && grp_info->external &&
9066 group < MLX5_MAX_TABLES_EXTERNAL)
9067 group *= MLX5_FLOW_TABLE_FACTOR;
9068 if (is_tunnel_offload_active(dev)) {
9069 standard_translation = !grp_info->external ||
9070 grp_info->std_tbl_fix;
9072 standard_translation = true;
9075 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s",
9076 dev->data->port_id, group, grp_info->transfer,
9077 grp_info->external, grp_info->fdb_def_rule,
9078 standard_translation ? "STANDARD" : "TUNNEL");
9079 if (standard_translation)
9080 ret = flow_group_to_table(dev->data->port_id, group, table,
9083 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
9090 * Discover availability of metadata reg_c's.
9092 * Iteratively use test flows to check availability.
9095 * Pointer to the Ethernet device structure.
9098 * 0 on success, a negative errno value otherwise and rte_errno is set.
9101 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
9103 struct mlx5_priv *priv = dev->data->dev_private;
9104 enum modify_reg idx;
9107 /* reg_c[0] and reg_c[1] are reserved. */
9108 priv->sh->flow_mreg_c[n++] = REG_C_0;
9109 priv->sh->flow_mreg_c[n++] = REG_C_1;
9110 /* Discover availability of other reg_c's. */
9111 for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
9112 struct rte_flow_attr attr = {
9113 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
9114 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
9117 struct rte_flow_item items[] = {
9119 .type = RTE_FLOW_ITEM_TYPE_END,
9122 struct rte_flow_action actions[] = {
9124 .type = (enum rte_flow_action_type)
9125 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
9126 .conf = &(struct mlx5_flow_action_copy_mreg){
9132 .type = RTE_FLOW_ACTION_TYPE_JUMP,
9133 .conf = &(struct rte_flow_action_jump){
9134 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
9138 .type = RTE_FLOW_ACTION_TYPE_END,
9142 struct rte_flow *flow;
9143 struct rte_flow_error error;
9145 if (!priv->sh->config.dv_flow_en)
9147 /* Create internal flow, validation skips copy action. */
9148 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
9149 items, actions, false, &error);
9150 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
9154 priv->sh->flow_mreg_c[n++] = idx;
9155 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
9157 for (; n < MLX5_MREG_C_NUM; ++n)
9158 priv->sh->flow_mreg_c[n] = REG_NON;
9159 priv->sh->metadata_regc_check_flag = 1;
9164 save_dump_file(const uint8_t *data, uint32_t size,
9165 uint32_t type, uint64_t id, void *arg, FILE *file)
9167 char line[BUF_SIZE];
9170 uint32_t actions_num;
9171 struct rte_flow_query_count *count;
9173 memset(line, 0, BUF_SIZE);
9175 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR:
9176 actions_num = *(uint32_t *)(arg);
9177 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,",
9178 type, id, actions_num);
9180 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT:
9181 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",",
9184 case DR_DUMP_REC_TYPE_PMD_COUNTER:
9185 count = (struct rte_flow_query_count *)arg;
9187 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n",
9188 type, id, count->hits, count->bytes);
9194 for (k = 0; k < size; k++) {
9195 /* Make sure we do not overrun the line buffer length. */
9196 if (out >= BUF_SIZE - 4) {
9200 out += snprintf(line + out, BUF_SIZE - out, "%02x",
9203 fprintf(file, "%s\n", line);
9208 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow,
9209 struct rte_flow_query_count *count, struct rte_flow_error *error)
9211 struct rte_flow_action action[2];
9212 enum mlx5_flow_drv_type ftype;
9213 const struct mlx5_flow_driver_ops *fops;
9216 return rte_flow_error_set(error, ENOENT,
9217 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9219 "invalid flow handle");
9221 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT;
9222 action[1].type = RTE_FLOW_ACTION_TYPE_END;
9223 if (flow->counter) {
9224 memset(count, 0, sizeof(struct rte_flow_query_count));
9225 ftype = (enum mlx5_flow_drv_type)(flow->drv_type);
9226 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN &&
9227 ftype < MLX5_FLOW_TYPE_MAX);
9228 fops = flow_get_drv_ops(ftype);
9229 return fops->query(dev, flow, action, count, error);
9234 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
9236 * Dump flow ipool data to file
9239 * The pointer to Ethernet device.
9241 * A pointer to a file for output.
9243 * Perform verbose error reporting if not NULL. PMDs initialize this
9244 * structure in case of error only.
9246 * 0 on success, a negative value otherwise.
9249 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev,
9250 struct rte_flow *flow, FILE *file,
9251 struct rte_flow_error *error)
9253 struct mlx5_priv *priv = dev->data->dev_private;
9254 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr;
9255 struct mlx5_flow_dv_encap_decap_resource *encap_decap;
9256 uint32_t handle_idx;
9257 struct mlx5_flow_handle *dh;
9258 struct rte_flow_query_count count;
9259 uint32_t actions_num;
9260 const uint8_t *data;
9264 void *action = NULL;
9267 return rte_flow_error_set(error, ENOENT,
9268 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9270 "invalid flow handle");
9272 handle_idx = flow->dev_handles;
9274 if (flow->counter &&
9275 (!mlx5_counter_query(dev, flow->counter, false,
9276 &count.hits, &count.bytes, &action)) && action) {
9277 id = (uint64_t)(uintptr_t)action;
9278 type = DR_DUMP_REC_TYPE_PMD_COUNTER;
9279 save_dump_file(NULL, 0, type,
9280 id, (void *)&count, file);
9283 while (handle_idx) {
9284 dh = mlx5_ipool_get(priv->sh->ipool
9285 [MLX5_IPOOL_MLX5_FLOW], handle_idx);
9288 handle_idx = dh->next.next;
9290 /* Get modify_hdr and encap_decap buf from ipools. */
9292 modify_hdr = dh->dvh.modify_hdr;
9294 if (dh->dvh.rix_encap_decap) {
9295 encap_decap = mlx5_ipool_get(priv->sh->ipool
9296 [MLX5_IPOOL_DECAP_ENCAP],
9297 dh->dvh.rix_encap_decap);
9300 data = (const uint8_t *)modify_hdr->actions;
9301 size = (size_t)(modify_hdr->actions_num) * 8;
9302 id = (uint64_t)(uintptr_t)modify_hdr->action;
9303 actions_num = modify_hdr->actions_num;
9304 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR;
9305 save_dump_file(data, size, type, id,
9306 (void *)(&actions_num), file);
9309 data = encap_decap->buf;
9310 size = encap_decap->size;
9311 id = (uint64_t)(uintptr_t)encap_decap->action;
9312 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT;
9313 save_dump_file(data, size, type,
9321 * Dump all flow's encap_decap/modify_hdr/counter data to file
9324 * The pointer to Ethernet device.
9326 * A pointer to a file for output.
9328 * Perform verbose error reporting if not NULL. PMDs initialize this
9329 * structure in case of error only.
9331 * 0 on success, a negative value otherwise.
9334 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev,
9335 FILE *file, struct rte_flow_error *error __rte_unused)
9337 struct mlx5_priv *priv = dev->data->dev_private;
9338 struct mlx5_dev_ctx_shared *sh = priv->sh;
9339 struct mlx5_hlist *h;
9340 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr;
9341 struct mlx5_flow_dv_encap_decap_resource *encap_decap;
9342 struct rte_flow_query_count count;
9343 uint32_t actions_num;
9344 const uint8_t *data;
9350 struct mlx5_list_inconst *l_inconst;
9351 struct mlx5_list_entry *e;
9353 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
9357 /* encap_decap hlist is lcore_share, get global core cache. */
9358 i = MLX5_LIST_GLOBAL;
9359 h = sh->encaps_decaps;
9361 for (j = 0; j <= h->mask; j++) {
9362 l_inconst = &h->buckets[j].l;
9363 if (!l_inconst || !l_inconst->cache[i])
9366 e = LIST_FIRST(&l_inconst->cache[i]->h);
9369 (struct mlx5_flow_dv_encap_decap_resource *)e;
9370 data = encap_decap->buf;
9371 size = encap_decap->size;
9372 id = (uint64_t)(uintptr_t)encap_decap->action;
9373 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT;
9374 save_dump_file(data, size, type,
9376 e = LIST_NEXT(e, next);
9381 /* get modify_hdr */
9382 h = sh->modify_cmds;
9384 lcore_index = rte_lcore_index(rte_lcore_id());
9385 if (unlikely(lcore_index == -1)) {
9386 lcore_index = MLX5_LIST_NLCORE;
9387 rte_spinlock_lock(&h->l_const.lcore_lock);
9391 for (j = 0; j <= h->mask; j++) {
9392 l_inconst = &h->buckets[j].l;
9393 if (!l_inconst || !l_inconst->cache[i])
9396 e = LIST_FIRST(&l_inconst->cache[i]->h);
9399 (struct mlx5_flow_dv_modify_hdr_resource *)e;
9400 data = (const uint8_t *)modify_hdr->actions;
9401 size = (size_t)(modify_hdr->actions_num) * 8;
9402 actions_num = modify_hdr->actions_num;
9403 id = (uint64_t)(uintptr_t)modify_hdr->action;
9404 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR;
9405 save_dump_file(data, size, type, id,
9406 (void *)(&actions_num), file);
9407 e = LIST_NEXT(e, next);
9411 if (unlikely(lcore_index == MLX5_LIST_NLCORE))
9412 rte_spinlock_unlock(&h->l_const.lcore_lock);
9416 MLX5_ASSERT(cmng->n_valid <= cmng->n);
9417 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid;
9418 for (j = 1; j <= max; j++) {
9420 if ((!mlx5_counter_query(dev, j, false, &count.hits,
9421 &count.bytes, &action)) && action) {
9422 id = (uint64_t)(uintptr_t)action;
9423 type = DR_DUMP_REC_TYPE_PMD_COUNTER;
9424 save_dump_file(NULL, 0, type,
9425 id, (void *)&count, file);
9433 * Dump flow raw hw data to file
9436 * The pointer to Ethernet device.
9438 * A pointer to a file for output.
9440 * Perform verbose error reporting if not NULL. PMDs initialize this
9441 * structure in case of error only.
9443 * 0 on success, a negative value otherwise.
9446 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
9448 struct rte_flow_error *error __rte_unused)
9450 struct mlx5_priv *priv = dev->data->dev_private;
9451 struct mlx5_dev_ctx_shared *sh = priv->sh;
9452 uint32_t handle_idx;
9454 struct mlx5_flow_handle *dh;
9455 struct rte_flow *flow;
9457 if (!sh->config.dv_flow_en) {
9458 if (fputs("device dv flow disabled\n", file) <= 0)
9465 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
9466 if (mlx5_flow_dev_dump_sh_all(dev, file, error))
9469 return mlx5_devx_cmd_flow_dump(sh->fdb_domain,
9471 sh->tx_domain, file);
9474 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
9475 (uintptr_t)(void *)flow_idx);
9479 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
9480 mlx5_flow_dev_dump_ipool(dev, flow, file, error);
9482 handle_idx = flow->dev_handles;
9483 while (handle_idx) {
9484 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
9489 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow,
9494 handle_idx = dh->next.next;
9500 * Get aged-out flows.
9503 * Pointer to the Ethernet device structure.
9504 * @param[in] context
9505 * The address of an array of pointers to the aged-out flows contexts.
9506 * @param[in] nb_countexts
9507 * The length of context array pointers.
9509 * Perform verbose error reporting if not NULL. Initialized in case of
9513 * how many contexts get in success, otherwise negative errno value.
9514 * if nb_contexts is 0, return the amount of all aged contexts.
9515 * if nb_contexts is not 0 , return the amount of aged flows reported
9516 * in the context array.
9519 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
9520 uint32_t nb_contexts, struct rte_flow_error *error)
9522 const struct mlx5_flow_driver_ops *fops;
9523 struct rte_flow_attr attr = { .transfer = 0 };
9525 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
9526 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
9527 return fops->get_aged_flows(dev, contexts, nb_contexts,
9531 "port %u get aged flows is not supported.",
9532 dev->data->port_id);
9536 /* Wrapper for driver action_validate op callback */
9538 flow_drv_action_validate(struct rte_eth_dev *dev,
9539 const struct rte_flow_indir_action_conf *conf,
9540 const struct rte_flow_action *action,
9541 const struct mlx5_flow_driver_ops *fops,
9542 struct rte_flow_error *error)
9544 static const char err_msg[] = "indirect action validation unsupported";
9546 if (!fops->action_validate) {
9547 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
9548 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
9552 return fops->action_validate(dev, conf, action, error);
9556 * Destroys the shared action by handle.
9559 * Pointer to Ethernet device structure.
9561 * Handle for the indirect action object to be destroyed.
9563 * Perform verbose error reporting if not NULL. PMDs initialize this
9564 * structure in case of error only.
9567 * 0 on success, a negative errno value otherwise and rte_errno is set.
9569 * @note: wrapper for driver action_create op callback.
9572 mlx5_action_handle_destroy(struct rte_eth_dev *dev,
9573 struct rte_flow_action_handle *handle,
9574 struct rte_flow_error *error)
9576 static const char err_msg[] = "indirect action destruction unsupported";
9577 struct rte_flow_attr attr = { .transfer = 0 };
9578 const struct mlx5_flow_driver_ops *fops =
9579 flow_get_drv_ops(flow_get_drv_type(dev, &attr));
9581 if (!fops->action_destroy) {
9582 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
9583 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
9587 return fops->action_destroy(dev, handle, error);
9590 /* Wrapper for driver action_destroy op callback */
9592 flow_drv_action_update(struct rte_eth_dev *dev,
9593 struct rte_flow_action_handle *handle,
9595 const struct mlx5_flow_driver_ops *fops,
9596 struct rte_flow_error *error)
9598 static const char err_msg[] = "indirect action update unsupported";
9600 if (!fops->action_update) {
9601 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
9602 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
9606 return fops->action_update(dev, handle, update, error);
9609 /* Wrapper for driver action_destroy op callback */
9611 flow_drv_action_query(struct rte_eth_dev *dev,
9612 const struct rte_flow_action_handle *handle,
9614 const struct mlx5_flow_driver_ops *fops,
9615 struct rte_flow_error *error)
9617 static const char err_msg[] = "indirect action query unsupported";
9619 if (!fops->action_query) {
9620 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
9621 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
9625 return fops->action_query(dev, handle, data, error);
9629 * Create indirect action for reuse in multiple flow rules.
9632 * Pointer to Ethernet device structure.
9634 * Pointer to indirect action object configuration.
9636 * Action configuration for indirect action object creation.
9638 * Perform verbose error reporting if not NULL. PMDs initialize this
9639 * structure in case of error only.
9641 * A valid handle in case of success, NULL otherwise and rte_errno is set.
9643 static struct rte_flow_action_handle *
9644 mlx5_action_handle_create(struct rte_eth_dev *dev,
9645 const struct rte_flow_indir_action_conf *conf,
9646 const struct rte_flow_action *action,
9647 struct rte_flow_error *error)
9649 static const char err_msg[] = "indirect action creation unsupported";
9650 struct rte_flow_attr attr = { .transfer = 0 };
9651 const struct mlx5_flow_driver_ops *fops =
9652 flow_get_drv_ops(flow_get_drv_type(dev, &attr));
9654 if (flow_drv_action_validate(dev, conf, action, fops, error))
9656 if (!fops->action_create) {
9657 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
9658 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
9662 return fops->action_create(dev, conf, action, error);
9666 * Updates inplace the indirect action configuration pointed by *handle*
9667 * with the configuration provided as *update* argument.
9668 * The update of the indirect action configuration effects all flow rules
9669 * reusing the action via handle.
9672 * Pointer to Ethernet device structure.
9674 * Handle for the indirect action to be updated.
9676 * Action specification used to modify the action pointed by handle.
9677 * *update* could be of same type with the action pointed by the *handle*
9678 * handle argument, or some other structures like a wrapper, depending on
9679 * the indirect action type.
9681 * Perform verbose error reporting if not NULL. PMDs initialize this
9682 * structure in case of error only.
9685 * 0 on success, a negative errno value otherwise and rte_errno is set.
9688 mlx5_action_handle_update(struct rte_eth_dev *dev,
9689 struct rte_flow_action_handle *handle,
9691 struct rte_flow_error *error)
9693 struct rte_flow_attr attr = { .transfer = 0 };
9694 const struct mlx5_flow_driver_ops *fops =
9695 flow_get_drv_ops(flow_get_drv_type(dev, &attr));
9698 ret = flow_drv_action_validate(dev, NULL,
9699 (const struct rte_flow_action *)update, fops, error);
9702 return flow_drv_action_update(dev, handle, update, fops,
9707 * Query the indirect action by handle.
9709 * This function allows retrieving action-specific data such as counters.
9710 * Data is gathered by special action which may be present/referenced in
9711 * more than one flow rule definition.
9713 * see @RTE_FLOW_ACTION_TYPE_COUNT
9716 * Pointer to Ethernet device structure.
9718 * Handle for the indirect action to query.
9719 * @param[in, out] data
9720 * Pointer to storage for the associated query data type.
9722 * Perform verbose error reporting if not NULL. PMDs initialize this
9723 * structure in case of error only.
9726 * 0 on success, a negative errno value otherwise and rte_errno is set.
9729 mlx5_action_handle_query(struct rte_eth_dev *dev,
9730 const struct rte_flow_action_handle *handle,
9732 struct rte_flow_error *error)
9734 struct rte_flow_attr attr = { .transfer = 0 };
9735 const struct mlx5_flow_driver_ops *fops =
9736 flow_get_drv_ops(flow_get_drv_type(dev, &attr));
9738 return flow_drv_action_query(dev, handle, data, fops, error);
9742 * Destroy all indirect actions (shared RSS).
9745 * Pointer to Ethernet device.
9748 * 0 on success, a negative errno value otherwise and rte_errno is set.
9751 mlx5_action_handle_flush(struct rte_eth_dev *dev)
9753 struct rte_flow_error error;
9754 struct mlx5_priv *priv = dev->data->dev_private;
9755 struct mlx5_shared_action_rss *shared_rss;
9759 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
9760 priv->rss_shared_actions, idx, shared_rss, next) {
9761 ret |= mlx5_action_handle_destroy(dev,
9762 (struct rte_flow_action_handle *)(uintptr_t)idx, &error);
9768 * Validate existing indirect actions against current device configuration
9769 * and attach them to device resources.
9772 * Pointer to Ethernet device.
9775 * 0 on success, a negative errno value otherwise and rte_errno is set.
9778 mlx5_action_handle_attach(struct rte_eth_dev *dev)
9780 struct mlx5_priv *priv = dev->data->dev_private;
9782 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last;
9784 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
9785 const char *message;
9788 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues,
9790 &message, &queue_idx);
9792 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s",
9793 dev->data->port_id, ind_tbl->queues[queue_idx],
9800 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
9801 ret = mlx5_ind_table_obj_attach(dev, ind_tbl);
9803 DRV_LOG(ERR, "Port %u could not attach "
9804 "indirection table obj %p",
9805 dev->data->port_id, (void *)ind_tbl);
9812 ind_tbl_last = ind_tbl;
9813 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
9814 if (ind_tbl == ind_tbl_last)
9816 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0)
9817 DRV_LOG(CRIT, "Port %u could not detach "
9818 "indirection table obj %p on rollback",
9819 dev->data->port_id, (void *)ind_tbl);
9825 * Detach indirect actions of the device from its resources.
9828 * Pointer to Ethernet device.
9831 * 0 on success, a negative errno value otherwise and rte_errno is set.
9834 mlx5_action_handle_detach(struct rte_eth_dev *dev)
9836 struct mlx5_priv *priv = dev->data->dev_private;
9838 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last;
9840 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
9841 ret = mlx5_ind_table_obj_detach(dev, ind_tbl);
9843 DRV_LOG(ERR, "Port %u could not detach "
9844 "indirection table obj %p",
9845 dev->data->port_id, (void *)ind_tbl);
9851 ind_tbl_last = ind_tbl;
9852 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
9853 if (ind_tbl == ind_tbl_last)
9855 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0)
9856 DRV_LOG(CRIT, "Port %u could not attach "
9857 "indirection table obj %p on rollback",
9858 dev->data->port_id, (void *)ind_tbl);
9863 #ifndef HAVE_MLX5DV_DR
9864 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1))
9866 #define MLX5_DOMAIN_SYNC_FLOW \
9867 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW)
9870 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
9872 struct rte_eth_dev *dev = &rte_eth_devices[port_id];
9873 const struct mlx5_flow_driver_ops *fops;
9875 struct rte_flow_attr attr = { .transfer = 0 };
9877 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr));
9878 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW);
9884 const struct mlx5_flow_tunnel *
9885 mlx5_get_tof(const struct rte_flow_item *item,
9886 const struct rte_flow_action *action,
9887 enum mlx5_tof_rule_type *rule_type)
9889 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
9890 if (item->type == (typeof(item->type))
9891 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) {
9892 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE;
9893 return flow_items_to_tunnel(item);
9896 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) {
9897 if (action->type == (typeof(action->type))
9898 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) {
9899 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE;
9900 return flow_actions_to_tunnel(action);
9907 * tunnel offload functionality is defined for DV environment only
9909 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
9911 union tunnel_offload_mark {
9914 uint32_t app_reserve:8;
9915 uint32_t table_id:15;
9916 uint32_t transfer:1;
9917 uint32_t _unused_:8;
9922 mlx5_access_tunnel_offload_db
9923 (struct rte_eth_dev *dev,
9924 bool (*match)(struct rte_eth_dev *,
9925 struct mlx5_flow_tunnel *, const void *),
9926 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
9927 void (*miss)(struct rte_eth_dev *, void *),
9928 void *ctx, bool lock_op);
9931 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
9932 struct rte_flow *flow,
9933 const struct rte_flow_attr *attr,
9934 const struct rte_flow_action *app_actions,
9936 const struct mlx5_flow_tunnel *tunnel,
9937 struct tunnel_default_miss_ctx *ctx,
9938 struct rte_flow_error *error)
9940 struct mlx5_priv *priv = dev->data->dev_private;
9941 struct mlx5_flow *dev_flow;
9942 struct rte_flow_attr miss_attr = *attr;
9943 const struct rte_flow_item miss_items[2] = {
9945 .type = RTE_FLOW_ITEM_TYPE_ETH,
9951 .type = RTE_FLOW_ITEM_TYPE_END,
9957 union tunnel_offload_mark mark_id;
9958 struct rte_flow_action_mark miss_mark;
9959 struct rte_flow_action miss_actions[3] = {
9960 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
9961 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL }
9963 const struct rte_flow_action_jump *jump_data;
9964 uint32_t i, flow_table = 0; /* prevent compilation warning */
9965 struct flow_grp_info grp_info = {
9967 .transfer = attr->transfer,
9968 .fdb_def_rule = !!priv->fdb_def_rule,
9973 if (!attr->transfer) {
9976 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
9977 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
9978 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
9981 return rte_flow_error_set
9983 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
9984 NULL, "invalid default miss RSS");
9985 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
9986 ctx->action_rss.level = 0,
9987 ctx->action_rss.types = priv->rss_conf.rss_hf,
9988 ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
9989 ctx->action_rss.queue_num = priv->reta_idx_n,
9990 ctx->action_rss.key = priv->rss_conf.rss_key,
9991 ctx->action_rss.queue = ctx->queue;
9992 if (!priv->reta_idx_n || !priv->rxqs_n)
9993 return rte_flow_error_set
9995 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
9996 NULL, "invalid port configuration");
9997 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG))
9998 ctx->action_rss.types = 0;
9999 for (i = 0; i != priv->reta_idx_n; ++i)
10000 ctx->queue[i] = (*priv->reta_idx)[i];
10002 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
10003 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
10005 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
10006 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
10007 jump_data = app_actions->conf;
10008 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
10009 miss_attr.group = jump_data->group;
10010 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
10011 &flow_table, &grp_info, error);
10013 return rte_flow_error_set(error, EINVAL,
10014 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
10015 NULL, "invalid tunnel id");
10016 mark_id.app_reserve = 0;
10017 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
10018 mark_id.transfer = !!attr->transfer;
10019 mark_id._unused_ = 0;
10020 miss_mark.id = mark_id.val;
10021 dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
10022 miss_items, miss_actions, flow_idx, error);
10025 dev_flow->flow = flow;
10026 dev_flow->external = true;
10027 dev_flow->tunnel = tunnel;
10028 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE;
10029 /* Subflow object was created, we must include one in the list. */
10030 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
10031 dev_flow->handle, next);
10033 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
10034 dev->data->port_id, tunnel->app_tunnel.type,
10035 tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
10036 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
10037 miss_actions, error);
10039 ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
10045 static const struct mlx5_flow_tbl_data_entry *
10046 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
10048 struct mlx5_priv *priv = dev->data->dev_private;
10049 struct mlx5_dev_ctx_shared *sh = priv->sh;
10050 struct mlx5_list_entry *he;
10051 union tunnel_offload_mark mbits = { .val = mark };
10052 union mlx5_flow_tbl_key table_key = {
10054 .level = tunnel_id_to_flow_tbl(mbits.table_id),
10058 .is_fdb = !!mbits.transfer,
10062 struct mlx5_flow_cb_ctx ctx = {
10063 .data = &table_key.v64,
10066 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx);
10068 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
10072 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx,
10073 struct mlx5_list_entry *entry)
10075 struct mlx5_dev_ctx_shared *sh = tool_ctx;
10076 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
10078 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
10079 tunnel_flow_tbl_to_id(tte->flow_table));
10084 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused,
10085 struct mlx5_list_entry *entry, void *cb_ctx)
10087 struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10088 union tunnel_tbl_key tbl = {
10089 .val = *(uint64_t *)(ctx->data),
10091 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
10093 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
10096 static struct mlx5_list_entry *
10097 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx)
10099 struct mlx5_dev_ctx_shared *sh = tool_ctx;
10100 struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10101 struct tunnel_tbl_entry *tte;
10102 union tunnel_tbl_key tbl = {
10103 .val = *(uint64_t *)(ctx->data),
10106 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
10111 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
10113 if (tte->flow_table >= MLX5_MAX_TABLES) {
10114 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
10116 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
10119 } else if (!tte->flow_table) {
10122 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
10123 tte->tunnel_id = tbl.tunnel_id;
10124 tte->group = tbl.group;
10132 static struct mlx5_list_entry *
10133 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused,
10134 struct mlx5_list_entry *oentry,
10135 void *cb_ctx __rte_unused)
10137 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte),
10142 memcpy(tte, oentry, sizeof(*tte));
10147 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused,
10148 struct mlx5_list_entry *entry)
10150 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
10156 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
10157 const struct mlx5_flow_tunnel *tunnel,
10158 uint32_t group, uint32_t *table,
10159 struct rte_flow_error *error)
10161 struct mlx5_list_entry *he;
10162 struct tunnel_tbl_entry *tte;
10163 union tunnel_tbl_key key = {
10164 .tunnel_id = tunnel ? tunnel->tunnel_id : 0,
10167 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
10168 struct mlx5_hlist *group_hash;
10169 struct mlx5_flow_cb_ctx ctx = {
10173 group_hash = tunnel ? tunnel->groups : thub->groups;
10174 he = mlx5_hlist_register(group_hash, key.val, &ctx);
10176 return rte_flow_error_set(error, EINVAL,
10177 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
10179 "tunnel group index not supported");
10180 tte = container_of(he, typeof(*tte), hash);
10181 *table = tte->flow_table;
10182 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
10183 dev->data->port_id, key.tunnel_id, group, *table);
10188 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
10189 struct mlx5_flow_tunnel *tunnel)
10191 struct mlx5_priv *priv = dev->data->dev_private;
10192 struct mlx5_indexed_pool *ipool;
10194 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
10195 dev->data->port_id, tunnel->tunnel_id);
10196 LIST_REMOVE(tunnel, chain);
10197 mlx5_hlist_destroy(tunnel->groups);
10198 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
10199 mlx5_ipool_free(ipool, tunnel->tunnel_id);
10203 mlx5_access_tunnel_offload_db
10204 (struct rte_eth_dev *dev,
10205 bool (*match)(struct rte_eth_dev *,
10206 struct mlx5_flow_tunnel *, const void *),
10207 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
10208 void (*miss)(struct rte_eth_dev *, void *),
10209 void *ctx, bool lock_op)
10211 bool verdict = false;
10212 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
10213 struct mlx5_flow_tunnel *tunnel;
10215 rte_spinlock_lock(&thub->sl);
10216 LIST_FOREACH(tunnel, &thub->tunnels, chain) {
10217 verdict = match(dev, tunnel, (const void *)ctx);
10222 rte_spinlock_unlock(&thub->sl);
10223 if (verdict && hit)
10224 hit(dev, tunnel, ctx);
10225 if (!verdict && miss)
10228 rte_spinlock_unlock(&thub->sl);
10233 struct tunnel_db_find_tunnel_id_ctx {
10234 uint32_t tunnel_id;
10235 struct mlx5_flow_tunnel *tunnel;
10239 find_tunnel_id_match(struct rte_eth_dev *dev,
10240 struct mlx5_flow_tunnel *tunnel, const void *x)
10242 const struct tunnel_db_find_tunnel_id_ctx *ctx = x;
10245 return tunnel->tunnel_id == ctx->tunnel_id;
10249 find_tunnel_id_hit(struct rte_eth_dev *dev,
10250 struct mlx5_flow_tunnel *tunnel, void *x)
10252 struct tunnel_db_find_tunnel_id_ctx *ctx = x;
10254 ctx->tunnel = tunnel;
10257 static struct mlx5_flow_tunnel *
10258 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
10260 struct tunnel_db_find_tunnel_id_ctx ctx = {
10264 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match,
10265 find_tunnel_id_hit, NULL, &ctx, true);
10270 static struct mlx5_flow_tunnel *
10271 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
10272 const struct rte_flow_tunnel *app_tunnel)
10274 struct mlx5_priv *priv = dev->data->dev_private;
10275 struct mlx5_indexed_pool *ipool;
10276 struct mlx5_flow_tunnel *tunnel;
10279 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
10280 tunnel = mlx5_ipool_zmalloc(ipool, &id);
10283 if (id >= MLX5_MAX_TUNNELS) {
10284 mlx5_ipool_free(ipool, id);
10285 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
10288 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true,
10290 mlx5_flow_tunnel_grp2tbl_create_cb,
10291 mlx5_flow_tunnel_grp2tbl_match_cb,
10292 mlx5_flow_tunnel_grp2tbl_remove_cb,
10293 mlx5_flow_tunnel_grp2tbl_clone_cb,
10294 mlx5_flow_tunnel_grp2tbl_clone_free_cb);
10295 if (!tunnel->groups) {
10296 mlx5_ipool_free(ipool, id);
10299 /* initiate new PMD tunnel */
10300 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
10301 tunnel->tunnel_id = id;
10302 tunnel->action.type = (typeof(tunnel->action.type))
10303 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
10304 tunnel->action.conf = tunnel;
10305 tunnel->item.type = (typeof(tunnel->item.type))
10306 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
10307 tunnel->item.spec = tunnel;
10308 tunnel->item.last = NULL;
10309 tunnel->item.mask = NULL;
10311 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
10312 dev->data->port_id, tunnel->tunnel_id);
10317 struct tunnel_db_get_tunnel_ctx {
10318 const struct rte_flow_tunnel *app_tunnel;
10319 struct mlx5_flow_tunnel *tunnel;
10322 static bool get_tunnel_match(struct rte_eth_dev *dev,
10323 struct mlx5_flow_tunnel *tunnel, const void *x)
10325 const struct tunnel_db_get_tunnel_ctx *ctx = x;
10328 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel,
10329 sizeof(*ctx->app_tunnel));
10332 static void get_tunnel_hit(struct rte_eth_dev *dev,
10333 struct mlx5_flow_tunnel *tunnel, void *x)
10335 /* called under tunnel spinlock protection */
10336 struct tunnel_db_get_tunnel_ctx *ctx = x;
10340 ctx->tunnel = tunnel;
10343 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x)
10345 /* called under tunnel spinlock protection */
10346 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
10347 struct tunnel_db_get_tunnel_ctx *ctx = x;
10349 rte_spinlock_unlock(&thub->sl);
10350 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel);
10351 rte_spinlock_lock(&thub->sl);
10353 ctx->tunnel->refctn = 1;
10354 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain);
10360 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
10361 const struct rte_flow_tunnel *app_tunnel,
10362 struct mlx5_flow_tunnel **tunnel)
10364 struct tunnel_db_get_tunnel_ctx ctx = {
10365 .app_tunnel = app_tunnel,
10368 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit,
10369 get_tunnel_miss, &ctx, true);
10370 *tunnel = ctx.tunnel;
10371 return ctx.tunnel ? 0 : -ENOMEM;
10374 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
10376 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
10380 if (!LIST_EMPTY(&thub->tunnels))
10381 DRV_LOG(WARNING, "port %u tunnels present", port_id);
10382 mlx5_hlist_destroy(thub->groups);
10386 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
10389 struct mlx5_flow_tunnel_hub *thub;
10391 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
10395 LIST_INIT(&thub->tunnels);
10396 rte_spinlock_init(&thub->sl);
10397 thub->groups = mlx5_hlist_create("flow groups", 64,
10399 mlx5_flow_tunnel_grp2tbl_create_cb,
10400 mlx5_flow_tunnel_grp2tbl_match_cb,
10401 mlx5_flow_tunnel_grp2tbl_remove_cb,
10402 mlx5_flow_tunnel_grp2tbl_clone_cb,
10403 mlx5_flow_tunnel_grp2tbl_clone_free_cb);
10404 if (!thub->groups) {
10408 sh->tunnel_hub = thub;
10414 mlx5_hlist_destroy(thub->groups);
10421 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
10422 struct rte_flow_tunnel *tunnel,
10423 struct rte_flow_error *error)
10425 struct mlx5_priv *priv = dev->data->dev_private;
10427 if (!priv->sh->config.dv_flow_en)
10428 return rte_flow_error_set(error, ENOTSUP,
10429 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
10430 "flow DV interface is off");
10431 if (!is_tunnel_offload_active(dev))
10432 return rte_flow_error_set(error, ENOTSUP,
10433 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
10434 "tunnel offload was not activated");
10436 return rte_flow_error_set(error, EINVAL,
10437 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
10438 "no application tunnel");
10439 switch (tunnel->type) {
10441 return rte_flow_error_set(error, EINVAL,
10442 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
10443 "unsupported tunnel type");
10444 case RTE_FLOW_ITEM_TYPE_VXLAN:
10445 case RTE_FLOW_ITEM_TYPE_GRE:
10446 case RTE_FLOW_ITEM_TYPE_NVGRE:
10447 case RTE_FLOW_ITEM_TYPE_GENEVE:
10454 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
10455 struct rte_flow_tunnel *app_tunnel,
10456 struct rte_flow_action **actions,
10457 uint32_t *num_of_actions,
10458 struct rte_flow_error *error)
10460 struct mlx5_flow_tunnel *tunnel;
10461 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error);
10465 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
10467 return rte_flow_error_set(error, ret,
10468 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
10469 "failed to initialize pmd tunnel");
10471 *actions = &tunnel->action;
10472 *num_of_actions = 1;
10477 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
10478 struct rte_flow_tunnel *app_tunnel,
10479 struct rte_flow_item **items,
10480 uint32_t *num_of_items,
10481 struct rte_flow_error *error)
10483 struct mlx5_flow_tunnel *tunnel;
10484 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error);
10488 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
10490 return rte_flow_error_set(error, ret,
10491 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
10492 "failed to initialize pmd tunnel");
10494 *items = &tunnel->item;
10499 struct tunnel_db_element_release_ctx {
10500 struct rte_flow_item *items;
10501 struct rte_flow_action *actions;
10502 uint32_t num_elements;
10503 struct rte_flow_error *error;
10508 tunnel_element_release_match(struct rte_eth_dev *dev,
10509 struct mlx5_flow_tunnel *tunnel, const void *x)
10511 const struct tunnel_db_element_release_ctx *ctx = x;
10514 if (ctx->num_elements != 1)
10516 else if (ctx->items)
10517 return ctx->items == &tunnel->item;
10518 else if (ctx->actions)
10519 return ctx->actions == &tunnel->action;
10525 tunnel_element_release_hit(struct rte_eth_dev *dev,
10526 struct mlx5_flow_tunnel *tunnel, void *x)
10528 struct tunnel_db_element_release_ctx *ctx = x;
10530 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
10531 mlx5_flow_tunnel_free(dev, tunnel);
10535 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x)
10537 struct tunnel_db_element_release_ctx *ctx = x;
10539 ctx->ret = rte_flow_error_set(ctx->error, EINVAL,
10540 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
10541 "invalid argument");
10545 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
10546 struct rte_flow_item *pmd_items,
10547 uint32_t num_items, struct rte_flow_error *err)
10549 struct tunnel_db_element_release_ctx ctx = {
10550 .items = pmd_items,
10552 .num_elements = num_items,
10556 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
10557 tunnel_element_release_hit,
10558 tunnel_element_release_miss, &ctx, false);
10564 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
10565 struct rte_flow_action *pmd_actions,
10566 uint32_t num_actions, struct rte_flow_error *err)
10568 struct tunnel_db_element_release_ctx ctx = {
10570 .actions = pmd_actions,
10571 .num_elements = num_actions,
10575 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
10576 tunnel_element_release_hit,
10577 tunnel_element_release_miss, &ctx, false);
10583 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
10584 struct rte_mbuf *m,
10585 struct rte_flow_restore_info *info,
10586 struct rte_flow_error *err)
10588 uint64_t ol_flags = m->ol_flags;
10589 const struct mlx5_flow_tbl_data_entry *tble;
10590 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID;
10592 if (!is_tunnel_offload_active(dev)) {
10597 if ((ol_flags & mask) != mask)
10599 tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
10601 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
10602 dev->data->port_id, m->hash.fdir.hi);
10605 MLX5_ASSERT(tble->tunnel);
10606 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
10607 info->group_id = tble->group_id;
10608 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
10609 RTE_FLOW_RESTORE_INFO_GROUP_ID |
10610 RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
10615 return rte_flow_error_set(err, EINVAL,
10616 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10617 "failed to get restore info");
10620 #else /* HAVE_IBV_FLOW_DV_SUPPORT */
10622 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev,
10623 __rte_unused struct rte_flow_tunnel *app_tunnel,
10624 __rte_unused struct rte_flow_action **actions,
10625 __rte_unused uint32_t *num_of_actions,
10626 __rte_unused struct rte_flow_error *error)
10632 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev,
10633 __rte_unused struct rte_flow_tunnel *app_tunnel,
10634 __rte_unused struct rte_flow_item **items,
10635 __rte_unused uint32_t *num_of_items,
10636 __rte_unused struct rte_flow_error *error)
10642 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev,
10643 __rte_unused struct rte_flow_item *pmd_items,
10644 __rte_unused uint32_t num_items,
10645 __rte_unused struct rte_flow_error *err)
10651 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev,
10652 __rte_unused struct rte_flow_action *pmd_action,
10653 __rte_unused uint32_t num_actions,
10654 __rte_unused struct rte_flow_error *err)
10660 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev,
10661 __rte_unused struct rte_mbuf *m,
10662 __rte_unused struct rte_flow_restore_info *i,
10663 __rte_unused struct rte_flow_error *err)
10669 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev,
10670 __rte_unused struct rte_flow *flow,
10671 __rte_unused const struct rte_flow_attr *attr,
10672 __rte_unused const struct rte_flow_action *actions,
10673 __rte_unused uint32_t flow_idx,
10674 __rte_unused const struct mlx5_flow_tunnel *tunnel,
10675 __rte_unused struct tunnel_default_miss_ctx *ctx,
10676 __rte_unused struct rte_flow_error *error)
10681 static struct mlx5_flow_tunnel *
10682 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev,
10683 __rte_unused uint32_t id)
10689 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev,
10690 __rte_unused struct mlx5_flow_tunnel *tunnel)
10695 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev,
10696 __rte_unused const struct mlx5_flow_tunnel *t,
10697 __rte_unused uint32_t group,
10698 __rte_unused uint32_t *table,
10699 struct rte_flow_error *error)
10701 return rte_flow_error_set(error, ENOTSUP,
10702 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10703 "tunnel offload requires DV support");
10707 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh,
10708 __rte_unused uint16_t port_id)
10711 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
10713 /* Flex flow item API */
10714 static struct rte_flow_item_flex_handle *
10715 mlx5_flow_flex_item_create(struct rte_eth_dev *dev,
10716 const struct rte_flow_item_flex_conf *conf,
10717 struct rte_flow_error *error)
10719 static const char err_msg[] = "flex item creation unsupported";
10720 struct mlx5_priv *priv = dev->data->dev_private;
10721 struct rte_flow_attr attr = { .transfer = 0 };
10722 const struct mlx5_flow_driver_ops *fops =
10723 flow_get_drv_ops(flow_get_drv_type(dev, &attr));
10725 if (!priv->pci_dev) {
10726 rte_flow_error_set(error, ENOTSUP,
10727 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10728 "create flex item on PF only");
10731 switch (priv->pci_dev->id.device_id) {
10732 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF:
10733 case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF:
10736 rte_flow_error_set(error, ENOTSUP,
10737 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10738 "flex item available on BlueField ports only");
10741 if (!fops->item_create) {
10742 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
10743 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
10747 return fops->item_create(dev, conf, error);
10751 mlx5_flow_flex_item_release(struct rte_eth_dev *dev,
10752 const struct rte_flow_item_flex_handle *handle,
10753 struct rte_flow_error *error)
10755 static const char err_msg[] = "flex item release unsupported";
10756 struct rte_flow_attr attr = { .transfer = 0 };
10757 const struct mlx5_flow_driver_ops *fops =
10758 flow_get_drv_ops(flow_get_drv_type(dev, &attr));
10760 if (!fops->item_release) {
10761 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
10762 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
10766 return fops->item_release(dev, handle, error);
10770 mlx5_dbg__print_pattern(const struct rte_flow_item *item)
10773 struct rte_flow_error error;
10775 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
10777 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name,
10779 (void *)(uintptr_t)item->type, &error);
10781 printf("%s ", item_name);
10783 printf("%d\n", (int)item->type);
10789 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item)
10791 const struct rte_flow_item_udp *spec = udp_item->spec;
10792 const struct rte_flow_item_udp *mask = udp_item->mask;
10793 uint16_t udp_dport = 0;
10795 if (spec != NULL) {
10797 mask = &rte_flow_item_udp_mask;
10798 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port &
10799 mask->hdr.dst_port);
10801 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN);
10804 static const struct mlx5_flow_expand_node *
10805 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern,
10806 unsigned int item_idx,
10807 const struct mlx5_flow_expand_node graph[],
10808 const struct mlx5_flow_expand_node *node)
10810 const struct rte_flow_item *item = pattern + item_idx, *prev_item;
10812 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN &&
10814 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
10816 * The expansion node is VXLAN and it is also the last
10817 * expandable item in the pattern, so need to continue
10818 * expansion of the inner tunnel.
10820 MLX5_ASSERT(item_idx > 0);
10821 prev_item = pattern + item_idx - 1;
10822 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP);
10823 if (mlx5_flow_is_std_vxlan_port(prev_item))
10824 return &graph[MLX5_EXPANSION_STD_VXLAN];
10825 return &graph[MLX5_EXPANSION_L3_VXLAN];
10830 /* Map of Verbs to Flow priority with 8 Verbs priorities. */
10831 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
10832 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
10835 /* Map of Verbs to Flow priority with 16 Verbs priorities. */
10836 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
10837 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
10838 { 9, 10, 11 }, { 12, 13, 14 },
10842 * Discover the number of available flow priorities.
10848 * On success, number of available flow priorities.
10849 * On failure, a negative errno-style code and rte_errno is set.
10852 mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
10854 static const uint16_t vprio[] = {8, 16};
10855 const struct mlx5_priv *priv = dev->data->dev_private;
10856 const struct mlx5_flow_driver_ops *fops;
10857 enum mlx5_flow_drv_type type;
10860 type = mlx5_flow_os_get_type();
10861 if (type == MLX5_FLOW_TYPE_MAX) {
10862 type = MLX5_FLOW_TYPE_VERBS;
10863 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en)
10864 type = MLX5_FLOW_TYPE_DV;
10866 fops = flow_get_drv_ops(type);
10867 if (fops->discover_priorities == NULL) {
10868 DRV_LOG(ERR, "Priority discovery not supported");
10869 rte_errno = ENOTSUP;
10872 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio));
10877 ret = RTE_DIM(priority_map_3);
10880 ret = RTE_DIM(priority_map_5);
10883 rte_errno = ENOTSUP;
10885 "port %u maximum priority: %d expected 8/16",
10886 dev->data->port_id, ret);
10889 DRV_LOG(INFO, "port %u supported flow priorities:"
10890 " 0-%d for ingress or egress root table,"
10891 " 0-%d for non-root table or transfer root table.",
10892 dev->data->port_id, ret - 2,
10893 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1);
10898 * Adjust flow priority based on the highest layer and the request priority.
10901 * Pointer to the Ethernet device structure.
10902 * @param[in] priority
10903 * The rule base priority.
10904 * @param[in] subpriority
10905 * The priority based on the items.
10908 * The new priority.
10911 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
10912 uint32_t subpriority)
10915 struct mlx5_priv *priv = dev->data->dev_private;
10917 switch (priv->sh->flow_max_priority) {
10918 case RTE_DIM(priority_map_3):
10919 res = priority_map_3[priority][subpriority];
10921 case RTE_DIM(priority_map_5):
10922 res = priority_map_5[priority][subpriority];