drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <stdbool.h>
  10 #include <sys/queue.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_ether.h>
  14 #include <ethdev_driver.h>
  15 #include <rte_eal_paging.h>
  16 #include <rte_flow.h>
  17 #include <rte_cycles.h>
  18 #include <rte_flow_driver.h>
  19 #include <rte_malloc.h>
  20 #include <rte_ip.h>
  21
  22 #include <mlx5_glue.h>
  23 #include <mlx5_devx_cmds.h>
  24 #include <mlx5_prm.h>
  25 #include <mlx5_malloc.h>
  26
  27 #include "mlx5_defs.h"
  28 #include "mlx5.h"
  29 #include "mlx5_flow.h"
  30 #include "mlx5_flow_os.h"
  31 #include "mlx5_rx.h"
  32 #include "mlx5_tx.h"
  33 #include "mlx5_common_os.h"
  34 #include "rte_pmd_mlx5.h"
  35
  36 struct tunnel_default_miss_ctx {
  37         uint16_t *queue;
  38         __extension__
  39         union {
  40                 struct rte_flow_action_rss action_rss;
  41                 struct rte_flow_action_queue miss_queue;
  42                 struct rte_flow_action_jump miss_jump;
  43                 uint8_t raw[0];
  44         };
  45 };
  46
  47 static int
  48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
  49                              struct rte_flow *flow,
  50                              const struct rte_flow_attr *attr,
  51                              const struct rte_flow_action *app_actions,
  52                              uint32_t flow_idx,
  53                              const struct mlx5_flow_tunnel *tunnel,
  54                              struct tunnel_default_miss_ctx *ctx,
  55                              struct rte_flow_error *error);
  56 static struct mlx5_flow_tunnel *
  57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
  58 static void
  59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
  60 static uint32_t
  61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
  62                                 const struct mlx5_flow_tunnel *tunnel,
  63                                 uint32_t group, uint32_t *table,
  64                                 struct rte_flow_error *error);
  65
  66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void);
  67 static void mlx5_flow_pop_thread_workspace(void);
  68
  69
  70 /** Device flow drivers. */
  71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  72
  73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  74
  75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  76         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  78         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  79 #endif
  80         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  81         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  82 };
  83
  84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
  85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
  86         (const int []){ \
  87                 __VA_ARGS__, 0, \
  88         }
  89
  90 /** Node object of input graph for mlx5_flow_expand_rss(). */
  91 struct mlx5_flow_expand_node {
  92         const int *const next;
  93         /**<
  94          * List of next node indexes. Index 0 is interpreted as a terminator.
  95          */
  96         const enum rte_flow_item_type type;
  97         /**< Pattern item type of current node. */
  98         uint64_t rss_types;
  99         /**<
 100          * RSS types bit-field associated with this node
 101          * (see RTE_ETH_RSS_* definitions).
 102          */
 103         uint64_t node_flags;
 104         /**<
 105          *  Bit-fields that define how the node is used in the expansion.
 106          * (see MLX5_EXPANSION_NODE_* definitions).
 107          */
 108 };
 109
 110 /* Optional expand field. The expansion alg will not go deeper. */
 111 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0)
 112
 113 /* The node is not added implicitly as expansion to the flow pattern.
 114  * If the node type does not match the flow pattern item type, the
 115  * expansion alg will go deeper to its next items.
 116  * In the current implementation, the list of next nodes indexes can
 117  * have up to one node with this flag set and it has to be the last
 118  * node index (before the list terminator).
 119  */
 120 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1)
 121
 122 /** Object returned by mlx5_flow_expand_rss(). */
 123 struct mlx5_flow_expand_rss {
 124         uint32_t entries;
 125         /**< Number of entries @p patterns and @p priorities. */
 126         struct {
 127                 struct rte_flow_item *pattern; /**< Expanded pattern array. */
 128                 uint32_t priority; /**< Priority offset for each expansion. */
 129         } entry[];
 130 };
 131
 132 static void
 133 mlx5_dbg__print_pattern(const struct rte_flow_item *item);
 134
 135 static const struct mlx5_flow_expand_node *
 136 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern,
 137                 unsigned int item_idx,
 138                 const struct mlx5_flow_expand_node graph[],
 139                 const struct mlx5_flow_expand_node *node);
 140
 141 static bool
 142 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item)
 143 {
 144         switch (item->type) {
 145         case RTE_FLOW_ITEM_TYPE_ETH:
 146         case RTE_FLOW_ITEM_TYPE_VLAN:
 147         case RTE_FLOW_ITEM_TYPE_IPV4:
 148         case RTE_FLOW_ITEM_TYPE_IPV6:
 149         case RTE_FLOW_ITEM_TYPE_UDP:
 150         case RTE_FLOW_ITEM_TYPE_TCP:
 151         case RTE_FLOW_ITEM_TYPE_VXLAN:
 152         case RTE_FLOW_ITEM_TYPE_NVGRE:
 153         case RTE_FLOW_ITEM_TYPE_GRE:
 154         case RTE_FLOW_ITEM_TYPE_GENEVE:
 155         case RTE_FLOW_ITEM_TYPE_MPLS:
 156         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 157         case RTE_FLOW_ITEM_TYPE_GRE_KEY:
 158         case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
 159         case RTE_FLOW_ITEM_TYPE_GTP:
 160                 return true;
 161         default:
 162                 break;
 163         }
 164         return false;
 165 }
 166
 167 static enum rte_flow_item_type
 168 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
 169 {
 170         enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID;
 171         uint16_t ether_type = 0;
 172         uint16_t ether_type_m;
 173         uint8_t ip_next_proto = 0;
 174         uint8_t ip_next_proto_m;
 175
 176         if (item == NULL || item->spec == NULL)
 177                 return ret;
 178         switch (item->type) {
 179         case RTE_FLOW_ITEM_TYPE_ETH:
 180                 if (item->mask)
 181                         ether_type_m = ((const struct rte_flow_item_eth *)
 182                                                 (item->mask))->type;
 183                 else
 184                         ether_type_m = rte_flow_item_eth_mask.type;
 185                 if (ether_type_m != RTE_BE16(0xFFFF))
 186                         break;
 187                 ether_type = ((const struct rte_flow_item_eth *)
 188                                 (item->spec))->type;
 189                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 190                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 191                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 192                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 193                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 194                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 195                 else
 196                         ret = RTE_FLOW_ITEM_TYPE_END;
 197                 break;
 198         case RTE_FLOW_ITEM_TYPE_VLAN:
 199                 if (item->mask)
 200                         ether_type_m = ((const struct rte_flow_item_vlan *)
 201                                                 (item->mask))->inner_type;
 202                 else
 203                         ether_type_m = rte_flow_item_vlan_mask.inner_type;
 204                 if (ether_type_m != RTE_BE16(0xFFFF))
 205                         break;
 206                 ether_type = ((const struct rte_flow_item_vlan *)
 207                                 (item->spec))->inner_type;
 208                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 209                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 210                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 211                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 212                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 213                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 214                 else
 215                         ret = RTE_FLOW_ITEM_TYPE_END;
 216                 break;
 217         case RTE_FLOW_ITEM_TYPE_IPV4:
 218                 if (item->mask)
 219                         ip_next_proto_m = ((const struct rte_flow_item_ipv4 *)
 220                                         (item->mask))->hdr.next_proto_id;
 221                 else
 222                         ip_next_proto_m =
 223                                 rte_flow_item_ipv4_mask.hdr.next_proto_id;
 224                 if (ip_next_proto_m != 0xFF)
 225                         break;
 226                 ip_next_proto = ((const struct rte_flow_item_ipv4 *)
 227                                 (item->spec))->hdr.next_proto_id;
 228                 if (ip_next_proto == IPPROTO_UDP)
 229                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 230                 else if (ip_next_proto == IPPROTO_TCP)
 231                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 232                 else if (ip_next_proto == IPPROTO_IP)
 233                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 234                 else if (ip_next_proto == IPPROTO_IPV6)
 235                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 236                 else
 237                         ret = RTE_FLOW_ITEM_TYPE_END;
 238                 break;
 239         case RTE_FLOW_ITEM_TYPE_IPV6:
 240                 if (item->mask)
 241                         ip_next_proto_m = ((const struct rte_flow_item_ipv6 *)
 242                                                 (item->mask))->hdr.proto;
 243                 else
 244                         ip_next_proto_m =
 245                                 rte_flow_item_ipv6_mask.hdr.proto;
 246                 if (ip_next_proto_m != 0xFF)
 247                         break;
 248                 ip_next_proto = ((const struct rte_flow_item_ipv6 *)
 249                                 (item->spec))->hdr.proto;
 250                 if (ip_next_proto == IPPROTO_UDP)
 251                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 252                 else if (ip_next_proto == IPPROTO_TCP)
 253                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 254                 else if (ip_next_proto == IPPROTO_IP)
 255                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 256                 else if (ip_next_proto == IPPROTO_IPV6)
 257                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 258                 else
 259                         ret = RTE_FLOW_ITEM_TYPE_END;
 260                 break;
 261         default:
 262                 ret = RTE_FLOW_ITEM_TYPE_VOID;
 263                 break;
 264         }
 265         return ret;
 266 }
 267
 268 static const int *
 269 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[],
 270                 const int *next_node)
 271 {
 272         const struct mlx5_flow_expand_node *node = NULL;
 273         const int *next = next_node;
 274
 275         while (next && *next) {
 276                 /*
 277                  * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT
 278                  * flag set, because they were not found in the flow pattern.
 279                  */
 280                 node = &graph[*next];
 281                 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT))
 282                         break;
 283                 next = node->next;
 284         }
 285         return next;
 286 }
 287
 288 #define MLX5_RSS_EXP_ELT_N 16
 289
 290 /**
 291  * Expand RSS flows into several possible flows according to the RSS hash
 292  * fields requested and the driver capabilities.
 293  *
 294  * @param[out] buf
 295  *   Buffer to store the result expansion.
 296  * @param[in] size
 297  *   Buffer size in bytes. If 0, @p buf can be NULL.
 298  * @param[in] pattern
 299  *   User flow pattern.
 300  * @param[in] types
 301  *   RSS types to expand (see RTE_ETH_RSS_* definitions).
 302  * @param[in] graph
 303  *   Input graph to expand @p pattern according to @p types.
 304  * @param[in] graph_root_index
 305  *   Index of root node in @p graph, typically 0.
 306  *
 307  * @return
 308  *   A positive value representing the size of @p buf in bytes regardless of
 309  *   @p size on success, a negative errno value otherwise and rte_errno is
 310  *   set, the following errors are defined:
 311  *
 312  *   -E2BIG: graph-depth @p graph is too deep.
 313  *   -EINVAL: @p size has not enough space for expanded pattern.
 314  */
 315 static int
 316 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 317                      const struct rte_flow_item *pattern, uint64_t types,
 318                      const struct mlx5_flow_expand_node graph[],
 319                      int graph_root_index)
 320 {
 321         const struct rte_flow_item *item;
 322         const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
 323         const int *next_node;
 324         const int *stack[MLX5_RSS_EXP_ELT_N];
 325         int stack_pos = 0;
 326         struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N];
 327         unsigned int i, item_idx, last_expand_item_idx = 0;
 328         size_t lsize;
 329         size_t user_pattern_size = 0;
 330         void *addr = NULL;
 331         const struct mlx5_flow_expand_node *next = NULL;
 332         struct rte_flow_item missed_item;
 333         int missed = 0;
 334         int elt = 0;
 335         const struct rte_flow_item *last_expand_item = NULL;
 336
 337         memset(&missed_item, 0, sizeof(missed_item));
 338         lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
 339                 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]);
 340         if (lsize > size)
 341                 return -EINVAL;
 342         buf->entry[0].priority = 0;
 343         buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N];
 344         buf->entries = 0;
 345         addr = buf->entry[0].pattern;
 346         for (item = pattern, item_idx = 0;
 347                         item->type != RTE_FLOW_ITEM_TYPE_END;
 348                         item++, item_idx++) {
 349                 if (!mlx5_flow_is_rss_expandable_item(item)) {
 350                         user_pattern_size += sizeof(*item);
 351                         continue;
 352                 }
 353                 last_expand_item = item;
 354                 last_expand_item_idx = item_idx;
 355                 i = 0;
 356                 while (node->next && node->next[i]) {
 357                         next = &graph[node->next[i]];
 358                         if (next->type == item->type)
 359                                 break;
 360                         if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) {
 361                                 node = next;
 362                                 i = 0;
 363                         } else {
 364                                 ++i;
 365                         }
 366                 }
 367                 if (next)
 368                         node = next;
 369                 user_pattern_size += sizeof(*item);
 370         }
 371         user_pattern_size += sizeof(*item); /* Handle END item. */
 372         lsize += user_pattern_size;
 373         if (lsize > size)
 374                 return -EINVAL;
 375         /* Copy the user pattern in the first entry of the buffer. */
 376         rte_memcpy(addr, pattern, user_pattern_size);
 377         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 378         buf->entries = 1;
 379         /* Start expanding. */
 380         memset(flow_items, 0, sizeof(flow_items));
 381         user_pattern_size -= sizeof(*item);
 382         /*
 383          * Check if the last valid item has spec set, need complete pattern,
 384          * and the pattern can be used for expansion.
 385          */
 386         missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item);
 387         if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 388                 /* Item type END indicates expansion is not required. */
 389                 return lsize;
 390         }
 391         if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 392                 next = NULL;
 393                 missed = 1;
 394                 for (i = 0; node->next && node->next[i]; ++i) {
 395                         next = &graph[node->next[i]];
 396                         if (next->type == missed_item.type) {
 397                                 flow_items[0].type = missed_item.type;
 398                                 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 399                                 break;
 400                         }
 401                         next = NULL;
 402                 }
 403         }
 404         if (next && missed) {
 405                 elt = 2; /* missed item + item end. */
 406                 node = next;
 407                 lsize += elt * sizeof(*item) + user_pattern_size;
 408                 if (lsize > size)
 409                         return -EINVAL;
 410                 if (node->rss_types & types) {
 411                         buf->entry[buf->entries].priority = 1;
 412                         buf->entry[buf->entries].pattern = addr;
 413                         buf->entries++;
 414                         rte_memcpy(addr, buf->entry[0].pattern,
 415                                    user_pattern_size);
 416                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 417                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 418                         addr = (void *)(((uintptr_t)addr) +
 419                                         elt * sizeof(*item));
 420                 }
 421         } else if (last_expand_item != NULL) {
 422                 node = mlx5_flow_expand_rss_adjust_node(pattern,
 423                                 last_expand_item_idx, graph, node);
 424         }
 425         memset(flow_items, 0, sizeof(flow_items));
 426         next_node = mlx5_flow_expand_rss_skip_explicit(graph,
 427                         node->next);
 428         stack[stack_pos] = next_node;
 429         node = next_node ? &graph[*next_node] : NULL;
 430         while (node) {
 431                 flow_items[stack_pos].type = node->type;
 432                 if (node->rss_types & types) {
 433                         size_t n;
 434                         /*
 435                          * compute the number of items to copy from the
 436                          * expansion and copy it.
 437                          * When the stack_pos is 0, there are 1 element in it,
 438                          * plus the addition END item.
 439                          */
 440                         elt = stack_pos + 2;
 441                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 442                         lsize += elt * sizeof(*item) + user_pattern_size;
 443                         if (lsize > size)
 444                                 return -EINVAL;
 445                         n = elt * sizeof(*item);
 446                         buf->entry[buf->entries].priority =
 447                                 stack_pos + 1 + missed;
 448                         buf->entry[buf->entries].pattern = addr;
 449                         buf->entries++;
 450                         rte_memcpy(addr, buf->entry[0].pattern,
 451                                    user_pattern_size);
 452                         addr = (void *)(((uintptr_t)addr) +
 453                                         user_pattern_size);
 454                         rte_memcpy(addr, &missed_item,
 455                                    missed * sizeof(*item));
 456                         addr = (void *)(((uintptr_t)addr) +
 457                                 missed * sizeof(*item));
 458                         rte_memcpy(addr, flow_items, n);
 459                         addr = (void *)(((uintptr_t)addr) + n);
 460                 }
 461                 /* Go deeper. */
 462                 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) &&
 463                                 node->next) {
 464                         next_node = mlx5_flow_expand_rss_skip_explicit(graph,
 465                                         node->next);
 466                         if (stack_pos++ == MLX5_RSS_EXP_ELT_N) {
 467                                 rte_errno = E2BIG;
 468                                 return -rte_errno;
 469                         }
 470                         stack[stack_pos] = next_node;
 471                 } else if (*(next_node + 1)) {
 472                         /* Follow up with the next possibility. */
 473                         next_node = mlx5_flow_expand_rss_skip_explicit(graph,
 474                                         ++next_node);
 475                 } else if (!stack_pos) {
 476                         /*
 477                          * Completing the traverse over the different paths.
 478                          * The next_node is advanced to the terminator.
 479                          */
 480                         ++next_node;
 481                 } else {
 482                         /* Move to the next path. */
 483                         while (stack_pos) {
 484                                 next_node = stack[--stack_pos];
 485                                 next_node++;
 486                                 if (*next_node)
 487                                         break;
 488                         }
 489                         next_node = mlx5_flow_expand_rss_skip_explicit(graph,
 490                                         next_node);
 491                         stack[stack_pos] = next_node;
 492                 }
 493                 node = next_node && *next_node ? &graph[*next_node] : NULL;
 494         };
 495         return lsize;
 496 }
 497
 498 enum mlx5_expansion {
 499         MLX5_EXPANSION_ROOT,
 500         MLX5_EXPANSION_ROOT_OUTER,
 501         MLX5_EXPANSION_OUTER_ETH,
 502         MLX5_EXPANSION_OUTER_VLAN,
 503         MLX5_EXPANSION_OUTER_IPV4,
 504         MLX5_EXPANSION_OUTER_IPV4_UDP,
 505         MLX5_EXPANSION_OUTER_IPV4_TCP,
 506         MLX5_EXPANSION_OUTER_IPV6,
 507         MLX5_EXPANSION_OUTER_IPV6_UDP,
 508         MLX5_EXPANSION_OUTER_IPV6_TCP,
 509         MLX5_EXPANSION_VXLAN,
 510         MLX5_EXPANSION_STD_VXLAN,
 511         MLX5_EXPANSION_L3_VXLAN,
 512         MLX5_EXPANSION_VXLAN_GPE,
 513         MLX5_EXPANSION_GRE,
 514         MLX5_EXPANSION_NVGRE,
 515         MLX5_EXPANSION_GRE_KEY,
 516         MLX5_EXPANSION_MPLS,
 517         MLX5_EXPANSION_ETH,
 518         MLX5_EXPANSION_VLAN,
 519         MLX5_EXPANSION_IPV4,
 520         MLX5_EXPANSION_IPV4_UDP,
 521         MLX5_EXPANSION_IPV4_TCP,
 522         MLX5_EXPANSION_IPV6,
 523         MLX5_EXPANSION_IPV6_UDP,
 524         MLX5_EXPANSION_IPV6_TCP,
 525         MLX5_EXPANSION_IPV6_FRAG_EXT,
 526         MLX5_EXPANSION_GTP
 527 };
 528
 529 /** Supported expansion of items. */
 530 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 531         [MLX5_EXPANSION_ROOT] = {
 532                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 533                                                   MLX5_EXPANSION_IPV4,
 534                                                   MLX5_EXPANSION_IPV6),
 535                 .type = RTE_FLOW_ITEM_TYPE_END,
 536         },
 537         [MLX5_EXPANSION_ROOT_OUTER] = {
 538                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 539                                                   MLX5_EXPANSION_OUTER_IPV4,
 540                                                   MLX5_EXPANSION_OUTER_IPV6),
 541                 .type = RTE_FLOW_ITEM_TYPE_END,
 542         },
 543         [MLX5_EXPANSION_OUTER_ETH] = {
 544                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 545                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 546                 .rss_types = 0,
 547         },
 548         [MLX5_EXPANSION_OUTER_VLAN] = {
 549                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 550                                                   MLX5_EXPANSION_OUTER_IPV6),
 551                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 552                 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
 553         },
 554         [MLX5_EXPANSION_OUTER_IPV4] = {
 555                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 556                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 557                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 558                          MLX5_EXPANSION_GRE,
 559                          MLX5_EXPANSION_NVGRE,
 560                          MLX5_EXPANSION_IPV4,
 561                          MLX5_EXPANSION_IPV6),
 562                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 563                 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 |
 564                         RTE_ETH_RSS_NONFRAG_IPV4_OTHER,
 565         },
 566         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 567                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 568                                                   MLX5_EXPANSION_VXLAN_GPE,
 569                                                   MLX5_EXPANSION_MPLS,
 570                                                   MLX5_EXPANSION_GTP),
 571                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 572                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP,
 573         },
 574         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 575                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 576                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP,
 577         },
 578         [MLX5_EXPANSION_OUTER_IPV6] = {
 579                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 580                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 581                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 582                          MLX5_EXPANSION_IPV4,
 583                          MLX5_EXPANSION_IPV6,
 584                          MLX5_EXPANSION_GRE,
 585                          MLX5_EXPANSION_NVGRE),
 586                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 587                 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 |
 588                         RTE_ETH_RSS_NONFRAG_IPV6_OTHER,
 589         },
 590         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 591                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 592                                                   MLX5_EXPANSION_VXLAN_GPE,
 593                                                   MLX5_EXPANSION_MPLS,
 594                                                   MLX5_EXPANSION_GTP),
 595                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 596                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP,
 597         },
 598         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 599                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 600                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP,
 601         },
 602         [MLX5_EXPANSION_VXLAN] = {
 603                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 604                                                   MLX5_EXPANSION_IPV4,
 605                                                   MLX5_EXPANSION_IPV6),
 606                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 607         },
 608         [MLX5_EXPANSION_STD_VXLAN] = {
 609                         .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
 610                                         .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 611         },
 612         [MLX5_EXPANSION_L3_VXLAN] = {
 613                         .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 614                                         MLX5_EXPANSION_IPV6),
 615                                         .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 616         },
 617         [MLX5_EXPANSION_VXLAN_GPE] = {
 618                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 619                                                   MLX5_EXPANSION_IPV4,
 620                                                   MLX5_EXPANSION_IPV6),
 621                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 622         },
 623         [MLX5_EXPANSION_GRE] = {
 624                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 625                                                   MLX5_EXPANSION_IPV6,
 626                                                   MLX5_EXPANSION_GRE_KEY,
 627                                                   MLX5_EXPANSION_MPLS),
 628                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 629         },
 630         [MLX5_EXPANSION_GRE_KEY] = {
 631                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 632                                                   MLX5_EXPANSION_IPV6,
 633                                                   MLX5_EXPANSION_MPLS),
 634                 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY,
 635                 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
 636         },
 637         [MLX5_EXPANSION_NVGRE] = {
 638                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
 639                 .type = RTE_FLOW_ITEM_TYPE_NVGRE,
 640         },
 641         [MLX5_EXPANSION_MPLS] = {
 642                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 643                                                   MLX5_EXPANSION_IPV6,
 644                                                   MLX5_EXPANSION_ETH),
 645                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 646                 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
 647         },
 648         [MLX5_EXPANSION_ETH] = {
 649                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 650                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 651         },
 652         [MLX5_EXPANSION_VLAN] = {
 653                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 654                                                   MLX5_EXPANSION_IPV6),
 655                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 656                 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
 657         },
 658         [MLX5_EXPANSION_IPV4] = {
 659                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 660                                                   MLX5_EXPANSION_IPV4_TCP),
 661                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 662                 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 |
 663                         RTE_ETH_RSS_NONFRAG_IPV4_OTHER,
 664         },
 665         [MLX5_EXPANSION_IPV4_UDP] = {
 666                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 667                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP,
 668         },
 669         [MLX5_EXPANSION_IPV4_TCP] = {
 670                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 671                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP,
 672         },
 673         [MLX5_EXPANSION_IPV6] = {
 674                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 675                                                   MLX5_EXPANSION_IPV6_TCP,
 676                                                   MLX5_EXPANSION_IPV6_FRAG_EXT),
 677                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 678                 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 |
 679                         RTE_ETH_RSS_NONFRAG_IPV6_OTHER,
 680         },
 681         [MLX5_EXPANSION_IPV6_UDP] = {
 682                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 683                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP,
 684         },
 685         [MLX5_EXPANSION_IPV6_TCP] = {
 686                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 687                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP,
 688         },
 689         [MLX5_EXPANSION_IPV6_FRAG_EXT] = {
 690                 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT,
 691         },
 692         [MLX5_EXPANSION_GTP] = {
 693                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 694                                                   MLX5_EXPANSION_IPV6),
 695                 .type = RTE_FLOW_ITEM_TYPE_GTP,
 696         },
 697 };
 698
 699 static struct rte_flow_action_handle *
 700 mlx5_action_handle_create(struct rte_eth_dev *dev,
 701                           const struct rte_flow_indir_action_conf *conf,
 702                           const struct rte_flow_action *action,
 703                           struct rte_flow_error *error);
 704 static int mlx5_action_handle_destroy
 705                                 (struct rte_eth_dev *dev,
 706                                  struct rte_flow_action_handle *handle,
 707                                  struct rte_flow_error *error);
 708 static int mlx5_action_handle_update
 709                                 (struct rte_eth_dev *dev,
 710                                  struct rte_flow_action_handle *handle,
 711                                  const void *update,
 712                                  struct rte_flow_error *error);
 713 static int mlx5_action_handle_query
 714                                 (struct rte_eth_dev *dev,
 715                                  const struct rte_flow_action_handle *handle,
 716                                  void *data,
 717                                  struct rte_flow_error *error);
 718 static int
 719 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
 720                     struct rte_flow_tunnel *app_tunnel,
 721                     struct rte_flow_action **actions,
 722                     uint32_t *num_of_actions,
 723                     struct rte_flow_error *error);
 724 static int
 725 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
 726                        struct rte_flow_tunnel *app_tunnel,
 727                        struct rte_flow_item **items,
 728                        uint32_t *num_of_items,
 729                        struct rte_flow_error *error);
 730 static int
 731 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
 732                               struct rte_flow_item *pmd_items,
 733                               uint32_t num_items, struct rte_flow_error *err);
 734 static int
 735 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
 736                                 struct rte_flow_action *pmd_actions,
 737                                 uint32_t num_actions,
 738                                 struct rte_flow_error *err);
 739 static int
 740 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
 741                                   struct rte_mbuf *m,
 742                                   struct rte_flow_restore_info *info,
 743                                   struct rte_flow_error *err);
 744
 745 static const struct rte_flow_ops mlx5_flow_ops = {
 746         .validate = mlx5_flow_validate,
 747         .create = mlx5_flow_create,
 748         .destroy = mlx5_flow_destroy,
 749         .flush = mlx5_flow_flush,
 750         .isolate = mlx5_flow_isolate,
 751         .query = mlx5_flow_query,
 752         .dev_dump = mlx5_flow_dev_dump,
 753         .get_aged_flows = mlx5_flow_get_aged_flows,
 754         .action_handle_create = mlx5_action_handle_create,
 755         .action_handle_destroy = mlx5_action_handle_destroy,
 756         .action_handle_update = mlx5_action_handle_update,
 757         .action_handle_query = mlx5_action_handle_query,
 758         .tunnel_decap_set = mlx5_flow_tunnel_decap_set,
 759         .tunnel_match = mlx5_flow_tunnel_match,
 760         .tunnel_action_decap_release = mlx5_flow_tunnel_action_release,
 761         .tunnel_item_release = mlx5_flow_tunnel_item_release,
 762         .get_restore_info = mlx5_flow_tunnel_get_restore_info,
 763 };
 764
 765 /* Tunnel information. */
 766 struct mlx5_flow_tunnel_info {
 767         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 768         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 769 };
 770
 771 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 772         {
 773                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 774                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 775         },
 776         {
 777                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 778                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 779         },
 780         {
 781                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 782                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 783         },
 784         {
 785                 .tunnel = MLX5_FLOW_LAYER_GRE,
 786                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 787         },
 788         {
 789                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 790                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 791         },
 792         {
 793                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 794                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 795         },
 796         {
 797                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
 798                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
 799         },
 800         {
 801                 .tunnel = MLX5_FLOW_LAYER_IPIP,
 802                 .ptype = RTE_PTYPE_TUNNEL_IP,
 803         },
 804         {
 805                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
 806                 .ptype = RTE_PTYPE_TUNNEL_IP,
 807         },
 808         {
 809                 .tunnel = MLX5_FLOW_LAYER_GTP,
 810                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
 811         },
 812 };
 813
 814
 815
 816 /**
 817  * Translate tag ID to register.
 818  *
 819  * @param[in] dev
 820  *   Pointer to the Ethernet device structure.
 821  * @param[in] feature
 822  *   The feature that request the register.
 823  * @param[in] id
 824  *   The request register ID.
 825  * @param[out] error
 826  *   Error description in case of any.
 827  *
 828  * @return
 829  *   The request register on success, a negative errno
 830  *   value otherwise and rte_errno is set.
 831  */
 832 int
 833 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 834                      enum mlx5_feature_name feature,
 835                      uint32_t id,
 836                      struct rte_flow_error *error)
 837 {
 838         struct mlx5_priv *priv = dev->data->dev_private;
 839         struct mlx5_dev_config *config = &priv->config;
 840         enum modify_reg start_reg;
 841         bool skip_mtr_reg = false;
 842
 843         switch (feature) {
 844         case MLX5_HAIRPIN_RX:
 845                 return REG_B;
 846         case MLX5_HAIRPIN_TX:
 847                 return REG_A;
 848         case MLX5_METADATA_RX:
 849                 switch (config->dv_xmeta_en) {
 850                 case MLX5_XMETA_MODE_LEGACY:
 851                         return REG_B;
 852                 case MLX5_XMETA_MODE_META16:
 853                         return REG_C_0;
 854                 case MLX5_XMETA_MODE_META32:
 855                         return REG_C_1;
 856                 }
 857                 break;
 858         case MLX5_METADATA_TX:
 859                 return REG_A;
 860         case MLX5_METADATA_FDB:
 861                 switch (config->dv_xmeta_en) {
 862                 case MLX5_XMETA_MODE_LEGACY:
 863                         return REG_NON;
 864                 case MLX5_XMETA_MODE_META16:
 865                         return REG_C_0;
 866                 case MLX5_XMETA_MODE_META32:
 867                         return REG_C_1;
 868                 }
 869                 break;
 870         case MLX5_FLOW_MARK:
 871                 switch (config->dv_xmeta_en) {
 872                 case MLX5_XMETA_MODE_LEGACY:
 873                         return REG_NON;
 874                 case MLX5_XMETA_MODE_META16:
 875                         return REG_C_1;
 876                 case MLX5_XMETA_MODE_META32:
 877                         return REG_C_0;
 878                 }
 879                 break;
 880         case MLX5_MTR_ID:
 881                 /*
 882                  * If meter color and meter id share one register, flow match
 883                  * should use the meter color register for match.
 884                  */
 885                 if (priv->mtr_reg_share)
 886                         return priv->mtr_color_reg;
 887                 else
 888                         return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 889                                REG_C_3;
 890         case MLX5_MTR_COLOR:
 891         case MLX5_ASO_FLOW_HIT:
 892         case MLX5_ASO_CONNTRACK:
 893                 /* All features use the same REG_C. */
 894                 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
 895                 return priv->mtr_color_reg;
 896         case MLX5_COPY_MARK:
 897                 /*
 898                  * Metadata COPY_MARK register using is in meter suffix sub
 899                  * flow while with meter. It's safe to share the same register.
 900                  */
 901                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
 902         case MLX5_APP_TAG:
 903                 /*
 904                  * If meter is enable, it will engage the register for color
 905                  * match and flow match. If meter color match is not using the
 906                  * REG_C_2, need to skip the REG_C_x be used by meter color
 907                  * match.
 908                  * If meter is disable, free to use all available registers.
 909                  */
 910                 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 911                             (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
 912                 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
 913                 if (id > (uint32_t)(REG_C_7 - start_reg))
 914                         return rte_flow_error_set(error, EINVAL,
 915                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 916                                                   NULL, "invalid tag id");
 917                 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 918                         return rte_flow_error_set(error, ENOTSUP,
 919                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 920                                                   NULL, "unsupported tag id");
 921                 /*
 922                  * This case means meter is using the REG_C_x great than 2.
 923                  * Take care not to conflict with meter color REG_C_x.
 924                  * If the available index REG_C_y >= REG_C_x, skip the
 925                  * color register.
 926                  */
 927                 if (skip_mtr_reg && priv->sh->flow_mreg_c
 928                     [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 929                         if (id >= (uint32_t)(REG_C_7 - start_reg))
 930                                 return rte_flow_error_set(error, EINVAL,
 931                                                        RTE_FLOW_ERROR_TYPE_ITEM,
 932                                                         NULL, "invalid tag id");
 933                         if (priv->sh->flow_mreg_c
 934                             [id + 1 + start_reg - REG_C_0] != REG_NON)
 935                                 return priv->sh->flow_mreg_c
 936                                                [id + 1 + start_reg - REG_C_0];
 937                         return rte_flow_error_set(error, ENOTSUP,
 938                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 939                                                   NULL, "unsupported tag id");
 940                 }
 941                 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0];
 942         }
 943         MLX5_ASSERT(false);
 944         return rte_flow_error_set(error, EINVAL,
 945                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 946                                   NULL, "invalid feature name");
 947 }
 948
 949 /**
 950  * Check extensive flow metadata register support.
 951  *
 952  * @param dev
 953  *   Pointer to rte_eth_dev structure.
 954  *
 955  * @return
 956  *   True if device supports extensive flow metadata register, otherwise false.
 957  */
 958 bool
 959 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 960 {
 961         struct mlx5_priv *priv = dev->data->dev_private;
 962
 963         /*
 964          * Having available reg_c can be regarded inclusively as supporting
 965          * extensive flow metadata register, which could mean,
 966          * - metadata register copy action by modify header.
 967          * - 16 modify header actions is supported.
 968          * - reg_c's are preserved across different domain (FDB and NIC) on
 969          *   packet loopback by flow lookup miss.
 970          */
 971         return priv->sh->flow_mreg_c[2] != REG_NON;
 972 }
 973
 974 /**
 975  * Get the lowest priority.
 976  *
 977  * @param[in] dev
 978  *   Pointer to the Ethernet device structure.
 979  * @param[in] attributes
 980  *   Pointer to device flow rule attributes.
 981  *
 982  * @return
 983  *   The value of lowest priority of flow.
 984  */
 985 uint32_t
 986 mlx5_get_lowest_priority(struct rte_eth_dev *dev,
 987                           const struct rte_flow_attr *attr)
 988 {
 989         struct mlx5_priv *priv = dev->data->dev_private;
 990
 991         if (!attr->group && !attr->transfer)
 992                 return priv->sh->flow_max_priority - 2;
 993         return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1;
 994 }
 995
 996 /**
 997  * Calculate matcher priority of the flow.
 998  *
 999  * @param[in] dev
1000  *   Pointer to the Ethernet device structure.
1001  * @param[in] attr
1002  *   Pointer to device flow rule attributes.
1003  * @param[in] subpriority
1004  *   The priority based on the items.
1005  * @param[in] external
1006  *   Flow is user flow.
1007  * @return
1008  *   The matcher priority of the flow.
1009  */
1010 uint16_t
1011 mlx5_get_matcher_priority(struct rte_eth_dev *dev,
1012                           const struct rte_flow_attr *attr,
1013                           uint32_t subpriority, bool external)
1014 {
1015         uint16_t priority = (uint16_t)attr->priority;
1016         struct mlx5_priv *priv = dev->data->dev_private;
1017
1018         if (!attr->group && !attr->transfer) {
1019                 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
1020                         priority = priv->sh->flow_max_priority - 1;
1021                 return mlx5_os_flow_adjust_priority(dev, priority, subpriority);
1022         } else if (!external && attr->transfer && attr->group == 0 &&
1023                    attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) {
1024                 return (priv->sh->flow_max_priority - 1) * 3;
1025         }
1026         if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
1027                 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO;
1028         return priority * 3 + subpriority;
1029 }
1030
1031 /**
1032  * Verify the @p item specifications (spec, last, mask) are compatible with the
1033  * NIC capabilities.
1034  *
1035  * @param[in] item
1036  *   Item specification.
1037  * @param[in] mask
1038  *   @p item->mask or flow default bit-masks.
1039  * @param[in] nic_mask
1040  *   Bit-masks covering supported fields by the NIC to compare with user mask.
1041  * @param[in] size
1042  *   Bit-masks size in bytes.
1043  * @param[in] range_accepted
1044  *   True if range of values is accepted for specific fields, false otherwise.
1045  * @param[out] error
1046  *   Pointer to error structure.
1047  *
1048  * @return
1049  *   0 on success, a negative errno value otherwise and rte_errno is set.
1050  */
1051 int
1052 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
1053                           const uint8_t *mask,
1054                           const uint8_t *nic_mask,
1055                           unsigned int size,
1056                           bool range_accepted,
1057                           struct rte_flow_error *error)
1058 {
1059         unsigned int i;
1060
1061         MLX5_ASSERT(nic_mask);
1062         for (i = 0; i < size; ++i)
1063                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
1064                         return rte_flow_error_set(error, ENOTSUP,
1065                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1066                                                   item,
1067                                                   "mask enables non supported"
1068                                                   " bits");
1069         if (!item->spec && (item->mask || item->last))
1070                 return rte_flow_error_set(error, EINVAL,
1071                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1072                                           "mask/last without a spec is not"
1073                                           " supported");
1074         if (item->spec && item->last && !range_accepted) {
1075                 uint8_t spec[size];
1076                 uint8_t last[size];
1077                 unsigned int i;
1078                 int ret;
1079
1080                 for (i = 0; i < size; ++i) {
1081                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
1082                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
1083                 }
1084                 ret = memcmp(spec, last, size);
1085                 if (ret != 0)
1086                         return rte_flow_error_set(error, EINVAL,
1087                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1088                                                   item,
1089                                                   "range is not valid");
1090         }
1091         return 0;
1092 }
1093
1094 /**
1095  * Adjust the hash fields according to the @p flow information.
1096  *
1097  * @param[in] dev_flow.
1098  *   Pointer to the mlx5_flow.
1099  * @param[in] tunnel
1100  *   1 when the hash field is for a tunnel item.
1101  * @param[in] layer_types
1102  *   RTE_ETH_RSS_* types.
1103  * @param[in] hash_fields
1104  *   Item hash fields.
1105  *
1106  * @return
1107  *   The hash fields that should be used.
1108  */
1109 uint64_t
1110 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
1111                             int tunnel __rte_unused, uint64_t layer_types,
1112                             uint64_t hash_fields)
1113 {
1114 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1115         int rss_request_inner = rss_desc->level >= 2;
1116
1117         /* Check RSS hash level for tunnel. */
1118         if (tunnel && rss_request_inner)
1119                 hash_fields |= IBV_RX_HASH_INNER;
1120         else if (tunnel || rss_request_inner)
1121                 return 0;
1122 #endif
1123         /* Check if requested layer matches RSS hash fields. */
1124         if (!(rss_desc->types & layer_types))
1125                 return 0;
1126         return hash_fields;
1127 }
1128
1129 /**
1130  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
1131  * if several tunnel rules are used on this queue, the tunnel ptype will be
1132  * cleared.
1133  *
1134  * @param rxq_ctrl
1135  *   Rx queue to update.
1136  */
1137 static void
1138 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
1139 {
1140         unsigned int i;
1141         uint32_t tunnel_ptype = 0;
1142
1143         /* Look up for the ptype to use. */
1144         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
1145                 if (!rxq_ctrl->flow_tunnels_n[i])
1146                         continue;
1147                 if (!tunnel_ptype) {
1148                         tunnel_ptype = tunnels_info[i].ptype;
1149                 } else {
1150                         tunnel_ptype = 0;
1151                         break;
1152                 }
1153         }
1154         rxq_ctrl->rxq.tunnel = tunnel_ptype;
1155 }
1156
1157 /**
1158  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
1159  * flow.
1160  *
1161  * @param[in] dev
1162  *   Pointer to the Ethernet device structure.
1163  * @param[in] dev_handle
1164  *   Pointer to device flow handle structure.
1165  */
1166 void
1167 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
1168                        struct mlx5_flow_handle *dev_handle)
1169 {
1170         struct mlx5_priv *priv = dev->data->dev_private;
1171         const int mark = dev_handle->mark;
1172         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1173         struct mlx5_ind_table_obj *ind_tbl = NULL;
1174         unsigned int i;
1175
1176         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1177                 struct mlx5_hrxq *hrxq;
1178
1179                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1180                               dev_handle->rix_hrxq);
1181                 if (hrxq)
1182                         ind_tbl = hrxq->ind_table;
1183         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1184                 struct mlx5_shared_action_rss *shared_rss;
1185
1186                 shared_rss = mlx5_ipool_get
1187                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1188                          dev_handle->rix_srss);
1189                 if (shared_rss)
1190                         ind_tbl = shared_rss->ind_tbl;
1191         }
1192         if (!ind_tbl)
1193                 return;
1194         for (i = 0; i != ind_tbl->queues_n; ++i) {
1195                 int idx = ind_tbl->queues[i];
1196                 struct mlx5_rxq_ctrl *rxq_ctrl =
1197                         container_of((*priv->rxqs)[idx],
1198                                      struct mlx5_rxq_ctrl, rxq);
1199
1200                 /*
1201                  * To support metadata register copy on Tx loopback,
1202                  * this must be always enabled (metadata may arive
1203                  * from other port - not from local flows only.
1204                  */
1205                 if (priv->config.dv_flow_en &&
1206                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1207                     mlx5_flow_ext_mreg_supported(dev)) {
1208                         rxq_ctrl->rxq.mark = 1;
1209                         rxq_ctrl->flow_mark_n = 1;
1210                 } else if (mark) {
1211                         rxq_ctrl->rxq.mark = 1;
1212                         rxq_ctrl->flow_mark_n++;
1213                 }
1214                 if (tunnel) {
1215                         unsigned int j;
1216
1217                         /* Increase the counter matching the flow. */
1218                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1219                                 if ((tunnels_info[j].tunnel &
1220                                      dev_handle->layers) ==
1221                                     tunnels_info[j].tunnel) {
1222                                         rxq_ctrl->flow_tunnels_n[j]++;
1223                                         break;
1224                                 }
1225                         }
1226                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1227                 }
1228         }
1229 }
1230
1231 /**
1232  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1233  *
1234  * @param[in] dev
1235  *   Pointer to the Ethernet device structure.
1236  * @param[in] flow
1237  *   Pointer to flow structure.
1238  */
1239 static void
1240 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1241 {
1242         struct mlx5_priv *priv = dev->data->dev_private;
1243         uint32_t handle_idx;
1244         struct mlx5_flow_handle *dev_handle;
1245
1246         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1247                        handle_idx, dev_handle, next)
1248                 flow_drv_rxq_flags_set(dev, dev_handle);
1249 }
1250
1251 /**
1252  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1253  * device flow if no other flow uses it with the same kind of request.
1254  *
1255  * @param dev
1256  *   Pointer to Ethernet device.
1257  * @param[in] dev_handle
1258  *   Pointer to the device flow handle structure.
1259  */
1260 static void
1261 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1262                         struct mlx5_flow_handle *dev_handle)
1263 {
1264         struct mlx5_priv *priv = dev->data->dev_private;
1265         const int mark = dev_handle->mark;
1266         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1267         struct mlx5_ind_table_obj *ind_tbl = NULL;
1268         unsigned int i;
1269
1270         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1271                 struct mlx5_hrxq *hrxq;
1272
1273                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1274                               dev_handle->rix_hrxq);
1275                 if (hrxq)
1276                         ind_tbl = hrxq->ind_table;
1277         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1278                 struct mlx5_shared_action_rss *shared_rss;
1279
1280                 shared_rss = mlx5_ipool_get
1281                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1282                          dev_handle->rix_srss);
1283                 if (shared_rss)
1284                         ind_tbl = shared_rss->ind_tbl;
1285         }
1286         if (!ind_tbl)
1287                 return;
1288         MLX5_ASSERT(dev->data->dev_started);
1289         for (i = 0; i != ind_tbl->queues_n; ++i) {
1290                 int idx = ind_tbl->queues[i];
1291                 struct mlx5_rxq_ctrl *rxq_ctrl =
1292                         container_of((*priv->rxqs)[idx],
1293                                      struct mlx5_rxq_ctrl, rxq);
1294
1295                 if (priv->config.dv_flow_en &&
1296                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1297                     mlx5_flow_ext_mreg_supported(dev)) {
1298                         rxq_ctrl->rxq.mark = 1;
1299                         rxq_ctrl->flow_mark_n = 1;
1300                 } else if (mark) {
1301                         rxq_ctrl->flow_mark_n--;
1302                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
1303                 }
1304                 if (tunnel) {
1305                         unsigned int j;
1306
1307                         /* Decrease the counter matching the flow. */
1308                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1309                                 if ((tunnels_info[j].tunnel &
1310                                      dev_handle->layers) ==
1311                                     tunnels_info[j].tunnel) {
1312                                         rxq_ctrl->flow_tunnels_n[j]--;
1313                                         break;
1314                                 }
1315                         }
1316                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1317                 }
1318         }
1319 }
1320
1321 /**
1322  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1323  * @p flow if no other flow uses it with the same kind of request.
1324  *
1325  * @param dev
1326  *   Pointer to Ethernet device.
1327  * @param[in] flow
1328  *   Pointer to the flow.
1329  */
1330 static void
1331 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1332 {
1333         struct mlx5_priv *priv = dev->data->dev_private;
1334         uint32_t handle_idx;
1335         struct mlx5_flow_handle *dev_handle;
1336
1337         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1338                        handle_idx, dev_handle, next)
1339                 flow_drv_rxq_flags_trim(dev, dev_handle);
1340 }
1341
1342 /**
1343  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1344  *
1345  * @param dev
1346  *   Pointer to Ethernet device.
1347  */
1348 static void
1349 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1350 {
1351         struct mlx5_priv *priv = dev->data->dev_private;
1352         unsigned int i;
1353
1354         for (i = 0; i != priv->rxqs_n; ++i) {
1355                 struct mlx5_rxq_ctrl *rxq_ctrl;
1356                 unsigned int j;
1357
1358                 if (!(*priv->rxqs)[i])
1359                         continue;
1360                 rxq_ctrl = container_of((*priv->rxqs)[i],
1361                                         struct mlx5_rxq_ctrl, rxq);
1362                 rxq_ctrl->flow_mark_n = 0;
1363                 rxq_ctrl->rxq.mark = 0;
1364                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1365                         rxq_ctrl->flow_tunnels_n[j] = 0;
1366                 rxq_ctrl->rxq.tunnel = 0;
1367         }
1368 }
1369
1370 /**
1371  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1372  *
1373  * @param[in] dev
1374  *   Pointer to the Ethernet device structure.
1375  */
1376 void
1377 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1378 {
1379         struct mlx5_priv *priv = dev->data->dev_private;
1380         struct mlx5_rxq_data *data;
1381         unsigned int i;
1382
1383         for (i = 0; i != priv->rxqs_n; ++i) {
1384                 if (!(*priv->rxqs)[i])
1385                         continue;
1386                 data = (*priv->rxqs)[i];
1387                 if (!rte_flow_dynf_metadata_avail()) {
1388                         data->dynf_meta = 0;
1389                         data->flow_meta_mask = 0;
1390                         data->flow_meta_offset = -1;
1391                         data->flow_meta_port_mask = 0;
1392                 } else {
1393                         data->dynf_meta = 1;
1394                         data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1395                         data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1396                         data->flow_meta_port_mask = priv->sh->dv_meta_mask;
1397                 }
1398         }
1399 }
1400
1401 /*
1402  * return a pointer to the desired action in the list of actions.
1403  *
1404  * @param[in] actions
1405  *   The list of actions to search the action in.
1406  * @param[in] action
1407  *   The action to find.
1408  *
1409  * @return
1410  *   Pointer to the action in the list, if found. NULL otherwise.
1411  */
1412 const struct rte_flow_action *
1413 mlx5_flow_find_action(const struct rte_flow_action *actions,
1414                       enum rte_flow_action_type action)
1415 {
1416         if (actions == NULL)
1417                 return NULL;
1418         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1419                 if (actions->type == action)
1420                         return actions;
1421         return NULL;
1422 }
1423
1424 /*
1425  * Validate the flag action.
1426  *
1427  * @param[in] action_flags
1428  *   Bit-fields that holds the actions detected until now.
1429  * @param[in] attr
1430  *   Attributes of flow that includes this action.
1431  * @param[out] error
1432  *   Pointer to error structure.
1433  *
1434  * @return
1435  *   0 on success, a negative errno value otherwise and rte_errno is set.
1436  */
1437 int
1438 mlx5_flow_validate_action_flag(uint64_t action_flags,
1439                                const struct rte_flow_attr *attr,
1440                                struct rte_flow_error *error)
1441 {
1442         if (action_flags & MLX5_FLOW_ACTION_MARK)
1443                 return rte_flow_error_set(error, EINVAL,
1444                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1445                                           "can't mark and flag in same flow");
1446         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1447                 return rte_flow_error_set(error, EINVAL,
1448                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1449                                           "can't have 2 flag"
1450                                           " actions in same flow");
1451         if (attr->egress)
1452                 return rte_flow_error_set(error, ENOTSUP,
1453                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1454                                           "flag action not supported for "
1455                                           "egress");
1456         return 0;
1457 }
1458
1459 /*
1460  * Validate the mark action.
1461  *
1462  * @param[in] action
1463  *   Pointer to the queue action.
1464  * @param[in] action_flags
1465  *   Bit-fields that holds the actions detected until now.
1466  * @param[in] attr
1467  *   Attributes of flow that includes this action.
1468  * @param[out] error
1469  *   Pointer to error structure.
1470  *
1471  * @return
1472  *   0 on success, a negative errno value otherwise and rte_errno is set.
1473  */
1474 int
1475 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1476                                uint64_t action_flags,
1477                                const struct rte_flow_attr *attr,
1478                                struct rte_flow_error *error)
1479 {
1480         const struct rte_flow_action_mark *mark = action->conf;
1481
1482         if (!mark)
1483                 return rte_flow_error_set(error, EINVAL,
1484                                           RTE_FLOW_ERROR_TYPE_ACTION,
1485                                           action,
1486                                           "configuration cannot be null");
1487         if (mark->id >= MLX5_FLOW_MARK_MAX)
1488                 return rte_flow_error_set(error, EINVAL,
1489                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1490                                           &mark->id,
1491                                           "mark id must in 0 <= id < "
1492                                           RTE_STR(MLX5_FLOW_MARK_MAX));
1493         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1494                 return rte_flow_error_set(error, EINVAL,
1495                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1496                                           "can't flag and mark in same flow");
1497         if (action_flags & MLX5_FLOW_ACTION_MARK)
1498                 return rte_flow_error_set(error, EINVAL,
1499                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1500                                           "can't have 2 mark actions in same"
1501                                           " flow");
1502         if (attr->egress)
1503                 return rte_flow_error_set(error, ENOTSUP,
1504                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1505                                           "mark action not supported for "
1506                                           "egress");
1507         return 0;
1508 }
1509
1510 /*
1511  * Validate the drop action.
1512  *
1513  * @param[in] action_flags
1514  *   Bit-fields that holds the actions detected until now.
1515  * @param[in] attr
1516  *   Attributes of flow that includes this action.
1517  * @param[out] error
1518  *   Pointer to error structure.
1519  *
1520  * @return
1521  *   0 on success, a negative errno value otherwise and rte_errno is set.
1522  */
1523 int
1524 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1525                                const struct rte_flow_attr *attr,
1526                                struct rte_flow_error *error)
1527 {
1528         if (attr->egress)
1529                 return rte_flow_error_set(error, ENOTSUP,
1530                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1531                                           "drop action not supported for "
1532                                           "egress");
1533         return 0;
1534 }
1535
1536 /*
1537  * Validate the queue action.
1538  *
1539  * @param[in] action
1540  *   Pointer to the queue action.
1541  * @param[in] action_flags
1542  *   Bit-fields that holds the actions detected until now.
1543  * @param[in] dev
1544  *   Pointer to the Ethernet device structure.
1545  * @param[in] attr
1546  *   Attributes of flow that includes this action.
1547  * @param[out] error
1548  *   Pointer to error structure.
1549  *
1550  * @return
1551  *   0 on success, a negative errno value otherwise and rte_errno is set.
1552  */
1553 int
1554 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1555                                 uint64_t action_flags,
1556                                 struct rte_eth_dev *dev,
1557                                 const struct rte_flow_attr *attr,
1558                                 struct rte_flow_error *error)
1559 {
1560         struct mlx5_priv *priv = dev->data->dev_private;
1561         const struct rte_flow_action_queue *queue = action->conf;
1562
1563         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1564                 return rte_flow_error_set(error, EINVAL,
1565                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1566                                           "can't have 2 fate actions in"
1567                                           " same flow");
1568         if (!priv->rxqs_n)
1569                 return rte_flow_error_set(error, EINVAL,
1570                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1571                                           NULL, "No Rx queues configured");
1572         if (queue->index >= priv->rxqs_n)
1573                 return rte_flow_error_set(error, EINVAL,
1574                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1575                                           &queue->index,
1576                                           "queue index out of range");
1577         if (!(*priv->rxqs)[queue->index])
1578                 return rte_flow_error_set(error, EINVAL,
1579                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1580                                           &queue->index,
1581                                           "queue is not configured");
1582         if (attr->egress)
1583                 return rte_flow_error_set(error, ENOTSUP,
1584                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1585                                           "queue action not supported for "
1586                                           "egress");
1587         return 0;
1588 }
1589
1590 /*
1591  * Validate the rss action.
1592  *
1593  * @param[in] dev
1594  *   Pointer to the Ethernet device structure.
1595  * @param[in] action
1596  *   Pointer to the queue action.
1597  * @param[out] error
1598  *   Pointer to error structure.
1599  *
1600  * @return
1601  *   0 on success, a negative errno value otherwise and rte_errno is set.
1602  */
1603 int
1604 mlx5_validate_action_rss(struct rte_eth_dev *dev,
1605                          const struct rte_flow_action *action,
1606                          struct rte_flow_error *error)
1607 {
1608         struct mlx5_priv *priv = dev->data->dev_private;
1609         const struct rte_flow_action_rss *rss = action->conf;
1610         enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED;
1611         unsigned int i;
1612
1613         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1614             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1615                 return rte_flow_error_set(error, ENOTSUP,
1616                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1617                                           &rss->func,
1618                                           "RSS hash function not supported");
1619 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1620         if (rss->level > 2)
1621 #else
1622         if (rss->level > 1)
1623 #endif
1624                 return rte_flow_error_set(error, ENOTSUP,
1625                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1626                                           &rss->level,
1627                                           "tunnel RSS is not supported");
1628         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1629         if (rss->key_len == 0 && rss->key != NULL)
1630                 return rte_flow_error_set(error, ENOTSUP,
1631                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1632                                           &rss->key_len,
1633                                           "RSS hash key length 0");
1634         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1635                 return rte_flow_error_set(error, ENOTSUP,
1636                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1637                                           &rss->key_len,
1638                                           "RSS hash key too small");
1639         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1640                 return rte_flow_error_set(error, ENOTSUP,
1641                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1642                                           &rss->key_len,
1643                                           "RSS hash key too large");
1644         if (rss->queue_num > priv->config.ind_table_max_size)
1645                 return rte_flow_error_set(error, ENOTSUP,
1646                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1647                                           &rss->queue_num,
1648                                           "number of queues too large");
1649         if (rss->types & MLX5_RSS_HF_MASK)
1650                 return rte_flow_error_set(error, ENOTSUP,
1651                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1652                                           &rss->types,
1653                                           "some RSS protocols are not"
1654                                           " supported");
1655         if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) &&
1656             !(rss->types & RTE_ETH_RSS_IP))
1657                 return rte_flow_error_set(error, EINVAL,
1658                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1659                                           "L3 partial RSS requested but L3 RSS"
1660                                           " type not specified");
1661         if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) &&
1662             !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP)))
1663                 return rte_flow_error_set(error, EINVAL,
1664                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1665                                           "L4 partial RSS requested but L4 RSS"
1666                                           " type not specified");
1667         if (!priv->rxqs_n)
1668                 return rte_flow_error_set(error, EINVAL,
1669                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1670                                           NULL, "No Rx queues configured");
1671         if (!rss->queue_num)
1672                 return rte_flow_error_set(error, EINVAL,
1673                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1674                                           NULL, "No queues configured");
1675         for (i = 0; i != rss->queue_num; ++i) {
1676                 struct mlx5_rxq_ctrl *rxq_ctrl;
1677
1678                 if (rss->queue[i] >= priv->rxqs_n)
1679                         return rte_flow_error_set
1680                                 (error, EINVAL,
1681                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1682                                  &rss->queue[i], "queue index out of range");
1683                 if (!(*priv->rxqs)[rss->queue[i]])
1684                         return rte_flow_error_set
1685                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1686                                  &rss->queue[i], "queue is not configured");
1687                 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]],
1688                                         struct mlx5_rxq_ctrl, rxq);
1689                 if (i == 0)
1690                         rxq_type = rxq_ctrl->type;
1691                 if (rxq_type != rxq_ctrl->type)
1692                         return rte_flow_error_set
1693                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1694                                  &rss->queue[i],
1695                                  "combining hairpin and regular RSS queues is not supported");
1696         }
1697         return 0;
1698 }
1699
1700 /*
1701  * Validate the rss action.
1702  *
1703  * @param[in] action
1704  *   Pointer to the queue action.
1705  * @param[in] action_flags
1706  *   Bit-fields that holds the actions detected until now.
1707  * @param[in] dev
1708  *   Pointer to the Ethernet device structure.
1709  * @param[in] attr
1710  *   Attributes of flow that includes this action.
1711  * @param[in] item_flags
1712  *   Items that were detected.
1713  * @param[out] error
1714  *   Pointer to error structure.
1715  *
1716  * @return
1717  *   0 on success, a negative errno value otherwise and rte_errno is set.
1718  */
1719 int
1720 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1721                               uint64_t action_flags,
1722                               struct rte_eth_dev *dev,
1723                               const struct rte_flow_attr *attr,
1724                               uint64_t item_flags,
1725                               struct rte_flow_error *error)
1726 {
1727         const struct rte_flow_action_rss *rss = action->conf;
1728         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1729         int ret;
1730
1731         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1732                 return rte_flow_error_set(error, EINVAL,
1733                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1734                                           "can't have 2 fate actions"
1735                                           " in same flow");
1736         ret = mlx5_validate_action_rss(dev, action, error);
1737         if (ret)
1738                 return ret;
1739         if (attr->egress)
1740                 return rte_flow_error_set(error, ENOTSUP,
1741                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1742                                           "rss action not supported for "
1743                                           "egress");
1744         if (rss->level > 1 && !tunnel)
1745                 return rte_flow_error_set(error, EINVAL,
1746                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1747                                           "inner RSS is not supported for "
1748                                           "non-tunnel flows");
1749         if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
1750             !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
1751                 return rte_flow_error_set(error, EINVAL,
1752                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1753                                           "RSS on eCPRI is not supported now");
1754         }
1755         if ((item_flags & MLX5_FLOW_LAYER_MPLS) &&
1756             !(item_flags &
1757               (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) &&
1758             rss->level > 1)
1759                 return rte_flow_error_set(error, EINVAL,
1760                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1761                                           "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern");
1762         return 0;
1763 }
1764
1765 /*
1766  * Validate the default miss action.
1767  *
1768  * @param[in] action_flags
1769  *   Bit-fields that holds the actions detected until now.
1770  * @param[out] error
1771  *   Pointer to error structure.
1772  *
1773  * @return
1774  *   0 on success, a negative errno value otherwise and rte_errno is set.
1775  */
1776 int
1777 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
1778                                 const struct rte_flow_attr *attr,
1779                                 struct rte_flow_error *error)
1780 {
1781         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1782                 return rte_flow_error_set(error, EINVAL,
1783                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1784                                           "can't have 2 fate actions in"
1785                                           " same flow");
1786         if (attr->egress)
1787                 return rte_flow_error_set(error, ENOTSUP,
1788                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1789                                           "default miss action not supported "
1790                                           "for egress");
1791         if (attr->group)
1792                 return rte_flow_error_set(error, ENOTSUP,
1793                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
1794                                           "only group 0 is supported");
1795         if (attr->transfer)
1796                 return rte_flow_error_set(error, ENOTSUP,
1797                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1798                                           NULL, "transfer is not supported");
1799         return 0;
1800 }
1801
1802 /*
1803  * Validate the count action.
1804  *
1805  * @param[in] dev
1806  *   Pointer to the Ethernet device structure.
1807  * @param[in] attr
1808  *   Attributes of flow that includes this action.
1809  * @param[out] error
1810  *   Pointer to error structure.
1811  *
1812  * @return
1813  *   0 on success, a negative errno value otherwise and rte_errno is set.
1814  */
1815 int
1816 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1817                                 const struct rte_flow_attr *attr,
1818                                 struct rte_flow_error *error)
1819 {
1820         if (attr->egress)
1821                 return rte_flow_error_set(error, ENOTSUP,
1822                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1823                                           "count action not supported for "
1824                                           "egress");
1825         return 0;
1826 }
1827
1828 /*
1829  * Validate the ASO CT action.
1830  *
1831  * @param[in] dev
1832  *   Pointer to the Ethernet device structure.
1833  * @param[in] conntrack
1834  *   Pointer to the CT action profile.
1835  * @param[out] error
1836  *   Pointer to error structure.
1837  *
1838  * @return
1839  *   0 on success, a negative errno value otherwise and rte_errno is set.
1840  */
1841 int
1842 mlx5_validate_action_ct(struct rte_eth_dev *dev,
1843                         const struct rte_flow_action_conntrack *conntrack,
1844                         struct rte_flow_error *error)
1845 {
1846         RTE_SET_USED(dev);
1847
1848         if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT)
1849                 return rte_flow_error_set(error, EINVAL,
1850                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1851                                           "Invalid CT state");
1852         if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST)
1853                 return rte_flow_error_set(error, EINVAL,
1854                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1855                                           "Invalid last TCP packet flag");
1856         return 0;
1857 }
1858
1859 /**
1860  * Verify the @p attributes will be correctly understood by the NIC and store
1861  * them in the @p flow if everything is correct.
1862  *
1863  * @param[in] dev
1864  *   Pointer to the Ethernet device structure.
1865  * @param[in] attributes
1866  *   Pointer to flow attributes
1867  * @param[out] error
1868  *   Pointer to error structure.
1869  *
1870  * @return
1871  *   0 on success, a negative errno value otherwise and rte_errno is set.
1872  */
1873 int
1874 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1875                               const struct rte_flow_attr *attributes,
1876                               struct rte_flow_error *error)
1877 {
1878         struct mlx5_priv *priv = dev->data->dev_private;
1879         uint32_t priority_max = priv->sh->flow_max_priority - 1;
1880
1881         if (attributes->group)
1882                 return rte_flow_error_set(error, ENOTSUP,
1883                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1884                                           NULL, "groups is not supported");
1885         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
1886             attributes->priority >= priority_max)
1887                 return rte_flow_error_set(error, ENOTSUP,
1888                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1889                                           NULL, "priority out of range");
1890         if (attributes->egress)
1891                 return rte_flow_error_set(error, ENOTSUP,
1892                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1893                                           "egress is not supported");
1894         if (attributes->transfer && !priv->config.dv_esw_en)
1895                 return rte_flow_error_set(error, ENOTSUP,
1896                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1897                                           NULL, "transfer is not supported");
1898         if (!attributes->ingress)
1899                 return rte_flow_error_set(error, EINVAL,
1900                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1901                                           NULL,
1902                                           "ingress attribute is mandatory");
1903         return 0;
1904 }
1905
1906 /**
1907  * Validate ICMP6 item.
1908  *
1909  * @param[in] item
1910  *   Item specification.
1911  * @param[in] item_flags
1912  *   Bit-fields that holds the items detected until now.
1913  * @param[in] ext_vlan_sup
1914  *   Whether extended VLAN features are supported or not.
1915  * @param[out] error
1916  *   Pointer to error structure.
1917  *
1918  * @return
1919  *   0 on success, a negative errno value otherwise and rte_errno is set.
1920  */
1921 int
1922 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1923                                uint64_t item_flags,
1924                                uint8_t target_protocol,
1925                                struct rte_flow_error *error)
1926 {
1927         const struct rte_flow_item_icmp6 *mask = item->mask;
1928         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1929         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1930                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1931         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1932                                       MLX5_FLOW_LAYER_OUTER_L4;
1933         int ret;
1934
1935         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1936                 return rte_flow_error_set(error, EINVAL,
1937                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1938                                           "protocol filtering not compatible"
1939                                           " with ICMP6 layer");
1940         if (!(item_flags & l3m))
1941                 return rte_flow_error_set(error, EINVAL,
1942                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1943                                           "IPv6 is mandatory to filter on"
1944                                           " ICMP6");
1945         if (item_flags & l4m)
1946                 return rte_flow_error_set(error, EINVAL,
1947                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1948                                           "multiple L4 layers not supported");
1949         if (!mask)
1950                 mask = &rte_flow_item_icmp6_mask;
1951         ret = mlx5_flow_item_acceptable
1952                 (item, (const uint8_t *)mask,
1953                  (const uint8_t *)&rte_flow_item_icmp6_mask,
1954                  sizeof(struct rte_flow_item_icmp6),
1955                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1956         if (ret < 0)
1957                 return ret;
1958         return 0;
1959 }
1960
1961 /**
1962  * Validate ICMP item.
1963  *
1964  * @param[in] item
1965  *   Item specification.
1966  * @param[in] item_flags
1967  *   Bit-fields that holds the items detected until now.
1968  * @param[out] error
1969  *   Pointer to error structure.
1970  *
1971  * @return
1972  *   0 on success, a negative errno value otherwise and rte_errno is set.
1973  */
1974 int
1975 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1976                              uint64_t item_flags,
1977                              uint8_t target_protocol,
1978                              struct rte_flow_error *error)
1979 {
1980         const struct rte_flow_item_icmp *mask = item->mask;
1981         const struct rte_flow_item_icmp nic_mask = {
1982                 .hdr.icmp_type = 0xff,
1983                 .hdr.icmp_code = 0xff,
1984                 .hdr.icmp_ident = RTE_BE16(0xffff),
1985                 .hdr.icmp_seq_nb = RTE_BE16(0xffff),
1986         };
1987         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1988         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1989                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1990         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1991                                       MLX5_FLOW_LAYER_OUTER_L4;
1992         int ret;
1993
1994         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1995                 return rte_flow_error_set(error, EINVAL,
1996                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1997                                           "protocol filtering not compatible"
1998                                           " with ICMP layer");
1999         if (!(item_flags & l3m))
2000                 return rte_flow_error_set(error, EINVAL,
2001                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2002                                           "IPv4 is mandatory to filter"
2003                                           " on ICMP");
2004         if (item_flags & l4m)
2005                 return rte_flow_error_set(error, EINVAL,
2006                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2007                                           "multiple L4 layers not supported");
2008         if (!mask)
2009                 mask = &nic_mask;
2010         ret = mlx5_flow_item_acceptable
2011                 (item, (const uint8_t *)mask,
2012                  (const uint8_t *)&nic_mask,
2013                  sizeof(struct rte_flow_item_icmp),
2014                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2015         if (ret < 0)
2016                 return ret;
2017         return 0;
2018 }
2019
2020 /**
2021  * Validate Ethernet item.
2022  *
2023  * @param[in] item
2024  *   Item specification.
2025  * @param[in] item_flags
2026  *   Bit-fields that holds the items detected until now.
2027  * @param[out] error
2028  *   Pointer to error structure.
2029  *
2030  * @return
2031  *   0 on success, a negative errno value otherwise and rte_errno is set.
2032  */
2033 int
2034 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
2035                             uint64_t item_flags, bool ext_vlan_sup,
2036                             struct rte_flow_error *error)
2037 {
2038         const struct rte_flow_item_eth *mask = item->mask;
2039         const struct rte_flow_item_eth nic_mask = {
2040                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2041                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2042                 .type = RTE_BE16(0xffff),
2043                 .has_vlan = ext_vlan_sup ? 1 : 0,
2044         };
2045         int ret;
2046         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2047         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
2048                                        MLX5_FLOW_LAYER_OUTER_L2;
2049
2050         if (item_flags & ethm)
2051                 return rte_flow_error_set(error, ENOTSUP,
2052                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2053                                           "multiple L2 layers not supported");
2054         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
2055             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
2056                 return rte_flow_error_set(error, EINVAL,
2057                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2058                                           "L2 layer should not follow "
2059                                           "L3 layers");
2060         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
2061             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
2062                 return rte_flow_error_set(error, EINVAL,
2063                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2064                                           "L2 layer should not follow VLAN");
2065         if (item_flags & MLX5_FLOW_LAYER_GTP)
2066                 return rte_flow_error_set(error, EINVAL,
2067                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2068                                           "L2 layer should not follow GTP");
2069         if (!mask)
2070                 mask = &rte_flow_item_eth_mask;
2071         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2072                                         (const uint8_t *)&nic_mask,
2073                                         sizeof(struct rte_flow_item_eth),
2074                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2075         return ret;
2076 }
2077
2078 /**
2079  * Validate VLAN item.
2080  *
2081  * @param[in] item
2082  *   Item specification.
2083  * @param[in] item_flags
2084  *   Bit-fields that holds the items detected until now.
2085  * @param[in] dev
2086  *   Ethernet device flow is being created on.
2087  * @param[out] error
2088  *   Pointer to error structure.
2089  *
2090  * @return
2091  *   0 on success, a negative errno value otherwise and rte_errno is set.
2092  */
2093 int
2094 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
2095                              uint64_t item_flags,
2096                              struct rte_eth_dev *dev,
2097                              struct rte_flow_error *error)
2098 {
2099         const struct rte_flow_item_vlan *spec = item->spec;
2100         const struct rte_flow_item_vlan *mask = item->mask;
2101         const struct rte_flow_item_vlan nic_mask = {
2102                 .tci = RTE_BE16(UINT16_MAX),
2103                 .inner_type = RTE_BE16(UINT16_MAX),
2104         };
2105         uint16_t vlan_tag = 0;
2106         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2107         int ret;
2108         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2109                                         MLX5_FLOW_LAYER_INNER_L4) :
2110                                        (MLX5_FLOW_LAYER_OUTER_L3 |
2111                                         MLX5_FLOW_LAYER_OUTER_L4);
2112         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2113                                         MLX5_FLOW_LAYER_OUTER_VLAN;
2114
2115         if (item_flags & vlanm)
2116                 return rte_flow_error_set(error, EINVAL,
2117                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2118                                           "multiple VLAN layers not supported");
2119         else if ((item_flags & l34m) != 0)
2120                 return rte_flow_error_set(error, EINVAL,
2121                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2122                                           "VLAN cannot follow L3/L4 layer");
2123         if (!mask)
2124                 mask = &rte_flow_item_vlan_mask;
2125         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2126                                         (const uint8_t *)&nic_mask,
2127                                         sizeof(struct rte_flow_item_vlan),
2128                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2129         if (ret)
2130                 return ret;
2131         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2132                 struct mlx5_priv *priv = dev->data->dev_private;
2133
2134                 if (priv->vmwa_context) {
2135                         /*
2136                          * Non-NULL context means we have a virtual machine
2137                          * and SR-IOV enabled, we have to create VLAN interface
2138                          * to make hypervisor to setup E-Switch vport
2139                          * context correctly. We avoid creating the multiple
2140                          * VLAN interfaces, so we cannot support VLAN tag mask.
2141                          */
2142                         return rte_flow_error_set(error, EINVAL,
2143                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2144                                                   item,
2145                                                   "VLAN tag mask is not"
2146                                                   " supported in virtual"
2147                                                   " environment");
2148                 }
2149         }
2150         if (spec) {
2151                 vlan_tag = spec->tci;
2152                 vlan_tag &= mask->tci;
2153         }
2154         /*
2155          * From verbs perspective an empty VLAN is equivalent
2156          * to a packet without VLAN layer.
2157          */
2158         if (!vlan_tag)
2159                 return rte_flow_error_set(error, EINVAL,
2160                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2161                                           item->spec,
2162                                           "VLAN cannot be empty");
2163         return 0;
2164 }
2165
2166 /**
2167  * Validate IPV4 item.
2168  *
2169  * @param[in] item
2170  *   Item specification.
2171  * @param[in] item_flags
2172  *   Bit-fields that holds the items detected until now.
2173  * @param[in] last_item
2174  *   Previous validated item in the pattern items.
2175  * @param[in] ether_type
2176  *   Type in the ethernet layer header (including dot1q).
2177  * @param[in] acc_mask
2178  *   Acceptable mask, if NULL default internal default mask
2179  *   will be used to check whether item fields are supported.
2180  * @param[in] range_accepted
2181  *   True if range of values is accepted for specific fields, false otherwise.
2182  * @param[out] error
2183  *   Pointer to error structure.
2184  *
2185  * @return
2186  *   0 on success, a negative errno value otherwise and rte_errno is set.
2187  */
2188 int
2189 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
2190                              uint64_t item_flags,
2191                              uint64_t last_item,
2192                              uint16_t ether_type,
2193                              const struct rte_flow_item_ipv4 *acc_mask,
2194                              bool range_accepted,
2195                              struct rte_flow_error *error)
2196 {
2197         const struct rte_flow_item_ipv4 *mask = item->mask;
2198         const struct rte_flow_item_ipv4 *spec = item->spec;
2199         const struct rte_flow_item_ipv4 nic_mask = {
2200                 .hdr = {
2201                         .src_addr = RTE_BE32(0xffffffff),
2202                         .dst_addr = RTE_BE32(0xffffffff),
2203                         .type_of_service = 0xff,
2204                         .next_proto_id = 0xff,
2205                 },
2206         };
2207         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2208         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2209                                       MLX5_FLOW_LAYER_OUTER_L3;
2210         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2211                                       MLX5_FLOW_LAYER_OUTER_L4;
2212         int ret;
2213         uint8_t next_proto = 0xFF;
2214         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2215                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2216                                   MLX5_FLOW_LAYER_INNER_VLAN);
2217
2218         if ((last_item & l2_vlan) && ether_type &&
2219             ether_type != RTE_ETHER_TYPE_IPV4)
2220                 return rte_flow_error_set(error, EINVAL,
2221                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2222                                           "IPv4 cannot follow L2/VLAN layer "
2223                                           "which ether type is not IPv4");
2224         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2225                 if (mask && spec)
2226                         next_proto = mask->hdr.next_proto_id &
2227                                      spec->hdr.next_proto_id;
2228                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2229                         return rte_flow_error_set(error, EINVAL,
2230                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2231                                                   item,
2232                                                   "multiple tunnel "
2233                                                   "not supported");
2234         }
2235         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
2236                 return rte_flow_error_set(error, EINVAL,
2237                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2238                                           "wrong tunnel type - IPv6 specified "
2239                                           "but IPv4 item provided");
2240         if (item_flags & l3m)
2241                 return rte_flow_error_set(error, ENOTSUP,
2242                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2243                                           "multiple L3 layers not supported");
2244         else if (item_flags & l4m)
2245                 return rte_flow_error_set(error, EINVAL,
2246                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2247                                           "L3 cannot follow an L4 layer.");
2248         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2249                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2250                 return rte_flow_error_set(error, EINVAL,
2251                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2252                                           "L3 cannot follow an NVGRE layer.");
2253         if (!mask)
2254                 mask = &rte_flow_item_ipv4_mask;
2255         else if (mask->hdr.next_proto_id != 0 &&
2256                  mask->hdr.next_proto_id != 0xff)
2257                 return rte_flow_error_set(error, EINVAL,
2258                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2259                                           "partial mask is not supported"
2260                                           " for protocol");
2261         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2262                                         acc_mask ? (const uint8_t *)acc_mask
2263                                                  : (const uint8_t *)&nic_mask,
2264                                         sizeof(struct rte_flow_item_ipv4),
2265                                         range_accepted, error);
2266         if (ret < 0)
2267                 return ret;
2268         return 0;
2269 }
2270
2271 /**
2272  * Validate IPV6 item.
2273  *
2274  * @param[in] item
2275  *   Item specification.
2276  * @param[in] item_flags
2277  *   Bit-fields that holds the items detected until now.
2278  * @param[in] last_item
2279  *   Previous validated item in the pattern items.
2280  * @param[in] ether_type
2281  *   Type in the ethernet layer header (including dot1q).
2282  * @param[in] acc_mask
2283  *   Acceptable mask, if NULL default internal default mask
2284  *   will be used to check whether item fields are supported.
2285  * @param[out] error
2286  *   Pointer to error structure.
2287  *
2288  * @return
2289  *   0 on success, a negative errno value otherwise and rte_errno is set.
2290  */
2291 int
2292 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2293                              uint64_t item_flags,
2294                              uint64_t last_item,
2295                              uint16_t ether_type,
2296                              const struct rte_flow_item_ipv6 *acc_mask,
2297                              struct rte_flow_error *error)
2298 {
2299         const struct rte_flow_item_ipv6 *mask = item->mask;
2300         const struct rte_flow_item_ipv6 *spec = item->spec;
2301         const struct rte_flow_item_ipv6 nic_mask = {
2302                 .hdr = {
2303                         .src_addr =
2304                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2305                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2306                         .dst_addr =
2307                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2308                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2309                         .vtc_flow = RTE_BE32(0xffffffff),
2310                         .proto = 0xff,
2311                 },
2312         };
2313         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2314         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2315                                       MLX5_FLOW_LAYER_OUTER_L3;
2316         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2317                                       MLX5_FLOW_LAYER_OUTER_L4;
2318         int ret;
2319         uint8_t next_proto = 0xFF;
2320         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2321                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2322                                   MLX5_FLOW_LAYER_INNER_VLAN);
2323
2324         if ((last_item & l2_vlan) && ether_type &&
2325             ether_type != RTE_ETHER_TYPE_IPV6)
2326                 return rte_flow_error_set(error, EINVAL,
2327                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2328                                           "IPv6 cannot follow L2/VLAN layer "
2329                                           "which ether type is not IPv6");
2330         if (mask && mask->hdr.proto == UINT8_MAX && spec)
2331                 next_proto = spec->hdr.proto;
2332         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2333                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2334                         return rte_flow_error_set(error, EINVAL,
2335                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2336                                                   item,
2337                                                   "multiple tunnel "
2338                                                   "not supported");
2339         }
2340         if (next_proto == IPPROTO_HOPOPTS  ||
2341             next_proto == IPPROTO_ROUTING  ||
2342             next_proto == IPPROTO_FRAGMENT ||
2343             next_proto == IPPROTO_ESP      ||
2344             next_proto == IPPROTO_AH       ||
2345             next_proto == IPPROTO_DSTOPTS)
2346                 return rte_flow_error_set(error, EINVAL,
2347                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2348                                           "IPv6 proto (next header) should "
2349                                           "not be set as extension header");
2350         if (item_flags & MLX5_FLOW_LAYER_IPIP)
2351                 return rte_flow_error_set(error, EINVAL,
2352                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2353                                           "wrong tunnel type - IPv4 specified "
2354                                           "but IPv6 item provided");
2355         if (item_flags & l3m)
2356                 return rte_flow_error_set(error, ENOTSUP,
2357                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2358                                           "multiple L3 layers not supported");
2359         else if (item_flags & l4m)
2360                 return rte_flow_error_set(error, EINVAL,
2361                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2362                                           "L3 cannot follow an L4 layer.");
2363         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2364                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2365                 return rte_flow_error_set(error, EINVAL,
2366                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2367                                           "L3 cannot follow an NVGRE layer.");
2368         if (!mask)
2369                 mask = &rte_flow_item_ipv6_mask;
2370         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2371                                         acc_mask ? (const uint8_t *)acc_mask
2372                                                  : (const uint8_t *)&nic_mask,
2373                                         sizeof(struct rte_flow_item_ipv6),
2374                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2375         if (ret < 0)
2376                 return ret;
2377         return 0;
2378 }
2379
2380 /**
2381  * Validate UDP item.
2382  *
2383  * @param[in] item
2384  *   Item specification.
2385  * @param[in] item_flags
2386  *   Bit-fields that holds the items detected until now.
2387  * @param[in] target_protocol
2388  *   The next protocol in the previous item.
2389  * @param[in] flow_mask
2390  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2391  * @param[out] error
2392  *   Pointer to error structure.
2393  *
2394  * @return
2395  *   0 on success, a negative errno value otherwise and rte_errno is set.
2396  */
2397 int
2398 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2399                             uint64_t item_flags,
2400                             uint8_t target_protocol,
2401                             struct rte_flow_error *error)
2402 {
2403         const struct rte_flow_item_udp *mask = item->mask;
2404         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2405         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2406                                       MLX5_FLOW_LAYER_OUTER_L3;
2407         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2408                                       MLX5_FLOW_LAYER_OUTER_L4;
2409         int ret;
2410
2411         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2412                 return rte_flow_error_set(error, EINVAL,
2413                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2414                                           "protocol filtering not compatible"
2415                                           " with UDP layer");
2416         if (!(item_flags & l3m))
2417                 return rte_flow_error_set(error, EINVAL,
2418                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2419                                           "L3 is mandatory to filter on L4");
2420         if (item_flags & l4m)
2421                 return rte_flow_error_set(error, EINVAL,
2422                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2423                                           "multiple L4 layers not supported");
2424         if (!mask)
2425                 mask = &rte_flow_item_udp_mask;
2426         ret = mlx5_flow_item_acceptable
2427                 (item, (const uint8_t *)mask,
2428                  (const uint8_t *)&rte_flow_item_udp_mask,
2429                  sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2430                  error);
2431         if (ret < 0)
2432                 return ret;
2433         return 0;
2434 }
2435
2436 /**
2437  * Validate TCP item.
2438  *
2439  * @param[in] item
2440  *   Item specification.
2441  * @param[in] item_flags
2442  *   Bit-fields that holds the items detected until now.
2443  * @param[in] target_protocol
2444  *   The next protocol in the previous item.
2445  * @param[out] error
2446  *   Pointer to error structure.
2447  *
2448  * @return
2449  *   0 on success, a negative errno value otherwise and rte_errno is set.
2450  */
2451 int
2452 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2453                             uint64_t item_flags,
2454                             uint8_t target_protocol,
2455                             const struct rte_flow_item_tcp *flow_mask,
2456                             struct rte_flow_error *error)
2457 {
2458         const struct rte_flow_item_tcp *mask = item->mask;
2459         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2460         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2461                                       MLX5_FLOW_LAYER_OUTER_L3;
2462         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2463                                       MLX5_FLOW_LAYER_OUTER_L4;
2464         int ret;
2465
2466         MLX5_ASSERT(flow_mask);
2467         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2468                 return rte_flow_error_set(error, EINVAL,
2469                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2470                                           "protocol filtering not compatible"
2471                                           " with TCP layer");
2472         if (!(item_flags & l3m))
2473                 return rte_flow_error_set(error, EINVAL,
2474                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2475                                           "L3 is mandatory to filter on L4");
2476         if (item_flags & l4m)
2477                 return rte_flow_error_set(error, EINVAL,
2478                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2479                                           "multiple L4 layers not supported");
2480         if (!mask)
2481                 mask = &rte_flow_item_tcp_mask;
2482         ret = mlx5_flow_item_acceptable
2483                 (item, (const uint8_t *)mask,
2484                  (const uint8_t *)flow_mask,
2485                  sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2486                  error);
2487         if (ret < 0)
2488                 return ret;
2489         return 0;
2490 }
2491
2492 /**
2493  * Validate VXLAN item.
2494  *
2495  * @param[in] dev
2496  *   Pointer to the Ethernet device structure.
2497  * @param[in] udp_dport
2498  *   UDP destination port
2499  * @param[in] item
2500  *   Item specification.
2501  * @param[in] item_flags
2502  *   Bit-fields that holds the items detected until now.
2503  * @param[in] attr
2504  *   Flow rule attributes.
2505  * @param[out] error
2506  *   Pointer to error structure.
2507  *
2508  * @return
2509  *   0 on success, a negative errno value otherwise and rte_errno is set.
2510  */
2511 int
2512 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev,
2513                               uint16_t udp_dport,
2514                               const struct rte_flow_item *item,
2515                               uint64_t item_flags,
2516                               const struct rte_flow_attr *attr,
2517                               struct rte_flow_error *error)
2518 {
2519         const struct rte_flow_item_vxlan *spec = item->spec;
2520         const struct rte_flow_item_vxlan *mask = item->mask;
2521         int ret;
2522         struct mlx5_priv *priv = dev->data->dev_private;
2523         union vni {
2524                 uint32_t vlan_id;
2525                 uint8_t vni[4];
2526         } id = { .vlan_id = 0, };
2527         const struct rte_flow_item_vxlan nic_mask = {
2528                 .vni = "\xff\xff\xff",
2529                 .rsvd1 = 0xff,
2530         };
2531         const struct rte_flow_item_vxlan *valid_mask;
2532
2533         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2534                 return rte_flow_error_set(error, ENOTSUP,
2535                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2536                                           "multiple tunnel layers not"
2537                                           " supported");
2538         valid_mask = &rte_flow_item_vxlan_mask;
2539         /*
2540          * Verify only UDPv4 is present as defined in
2541          * https://tools.ietf.org/html/rfc7348
2542          */
2543         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2544                 return rte_flow_error_set(error, EINVAL,
2545                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2546                                           "no outer UDP layer found");
2547         if (!mask)
2548                 mask = &rte_flow_item_vxlan_mask;
2549
2550         if (priv->sh->steering_format_version !=
2551             MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 ||
2552             !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) {
2553                 /* FDB domain & NIC domain non-zero group */
2554                 if ((attr->transfer || attr->group) && priv->sh->misc5_cap)
2555                         valid_mask = &nic_mask;
2556                 /* Group zero in NIC domain */
2557                 if (!attr->group && !attr->transfer &&
2558                     priv->sh->tunnel_header_0_1)
2559                         valid_mask = &nic_mask;
2560         }
2561         ret = mlx5_flow_item_acceptable
2562                 (item, (const uint8_t *)mask,
2563                  (const uint8_t *)valid_mask,
2564                  sizeof(struct rte_flow_item_vxlan),
2565                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2566         if (ret < 0)
2567                 return ret;
2568         if (spec) {
2569                 memcpy(&id.vni[1], spec->vni, 3);
2570                 memcpy(&id.vni[1], mask->vni, 3);
2571         }
2572         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2573                 return rte_flow_error_set(error, ENOTSUP,
2574                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2575                                           "VXLAN tunnel must be fully defined");
2576         return 0;
2577 }
2578
2579 /**
2580  * Validate VXLAN_GPE item.
2581  *
2582  * @param[in] item
2583  *   Item specification.
2584  * @param[in] item_flags
2585  *   Bit-fields that holds the items detected until now.
2586  * @param[in] priv
2587  *   Pointer to the private data structure.
2588  * @param[in] target_protocol
2589  *   The next protocol in the previous item.
2590  * @param[out] error
2591  *   Pointer to error structure.
2592  *
2593  * @return
2594  *   0 on success, a negative errno value otherwise and rte_errno is set.
2595  */
2596 int
2597 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2598                                   uint64_t item_flags,
2599                                   struct rte_eth_dev *dev,
2600                                   struct rte_flow_error *error)
2601 {
2602         struct mlx5_priv *priv = dev->data->dev_private;
2603         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2604         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2605         int ret;
2606         union vni {
2607                 uint32_t vlan_id;
2608                 uint8_t vni[4];
2609         } id = { .vlan_id = 0, };
2610
2611         if (!priv->config.l3_vxlan_en)
2612                 return rte_flow_error_set(error, ENOTSUP,
2613                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2614                                           "L3 VXLAN is not enabled by device"
2615                                           " parameter and/or not configured in"
2616                                           " firmware");
2617         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2618                 return rte_flow_error_set(error, ENOTSUP,
2619                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2620                                           "multiple tunnel layers not"
2621                                           " supported");
2622         /*
2623          * Verify only UDPv4 is present as defined in
2624          * https://tools.ietf.org/html/rfc7348
2625          */
2626         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2627                 return rte_flow_error_set(error, EINVAL,
2628                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2629                                           "no outer UDP layer found");
2630         if (!mask)
2631                 mask = &rte_flow_item_vxlan_gpe_mask;
2632         ret = mlx5_flow_item_acceptable
2633                 (item, (const uint8_t *)mask,
2634                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2635                  sizeof(struct rte_flow_item_vxlan_gpe),
2636                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2637         if (ret < 0)
2638                 return ret;
2639         if (spec) {
2640                 if (spec->protocol)
2641                         return rte_flow_error_set(error, ENOTSUP,
2642                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2643                                                   item,
2644                                                   "VxLAN-GPE protocol"
2645                                                   " not supported");
2646                 memcpy(&id.vni[1], spec->vni, 3);
2647                 memcpy(&id.vni[1], mask->vni, 3);
2648         }
2649         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2650                 return rte_flow_error_set(error, ENOTSUP,
2651                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2652                                           "VXLAN-GPE tunnel must be fully"
2653                                           " defined");
2654         return 0;
2655 }
2656 /**
2657  * Validate GRE Key item.
2658  *
2659  * @param[in] item
2660  *   Item specification.
2661  * @param[in] item_flags
2662  *   Bit flags to mark detected items.
2663  * @param[in] gre_item
2664  *   Pointer to gre_item
2665  * @param[out] error
2666  *   Pointer to error structure.
2667  *
2668  * @return
2669  *   0 on success, a negative errno value otherwise and rte_errno is set.
2670  */
2671 int
2672 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2673                                 uint64_t item_flags,
2674                                 const struct rte_flow_item *gre_item,
2675                                 struct rte_flow_error *error)
2676 {
2677         const rte_be32_t *mask = item->mask;
2678         int ret = 0;
2679         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2680         const struct rte_flow_item_gre *gre_spec;
2681         const struct rte_flow_item_gre *gre_mask;
2682
2683         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2684                 return rte_flow_error_set(error, ENOTSUP,
2685                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2686                                           "Multiple GRE key not support");
2687         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2688                 return rte_flow_error_set(error, ENOTSUP,
2689                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2690                                           "No preceding GRE header");
2691         if (item_flags & MLX5_FLOW_LAYER_INNER)
2692                 return rte_flow_error_set(error, ENOTSUP,
2693                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2694                                           "GRE key following a wrong item");
2695         gre_mask = gre_item->mask;
2696         if (!gre_mask)
2697                 gre_mask = &rte_flow_item_gre_mask;
2698         gre_spec = gre_item->spec;
2699         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2700                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2701                 return rte_flow_error_set(error, EINVAL,
2702                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2703                                           "Key bit must be on");
2704
2705         if (!mask)
2706                 mask = &gre_key_default_mask;
2707         ret = mlx5_flow_item_acceptable
2708                 (item, (const uint8_t *)mask,
2709                  (const uint8_t *)&gre_key_default_mask,
2710                  sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2711         return ret;
2712 }
2713
2714 /**
2715  * Validate GRE item.
2716  *
2717  * @param[in] item
2718  *   Item specification.
2719  * @param[in] item_flags
2720  *   Bit flags to mark detected items.
2721  * @param[in] target_protocol
2722  *   The next protocol in the previous item.
2723  * @param[out] error
2724  *   Pointer to error structure.
2725  *
2726  * @return
2727  *   0 on success, a negative errno value otherwise and rte_errno is set.
2728  */
2729 int
2730 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2731                             uint64_t item_flags,
2732                             uint8_t target_protocol,
2733                             struct rte_flow_error *error)
2734 {
2735         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2736         const struct rte_flow_item_gre *mask = item->mask;
2737         int ret;
2738         const struct rte_flow_item_gre nic_mask = {
2739                 .c_rsvd0_ver = RTE_BE16(0xB000),
2740                 .protocol = RTE_BE16(UINT16_MAX),
2741         };
2742
2743         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2744                 return rte_flow_error_set(error, EINVAL,
2745                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2746                                           "protocol filtering not compatible"
2747                                           " with this GRE layer");
2748         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2749                 return rte_flow_error_set(error, ENOTSUP,
2750                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2751                                           "multiple tunnel layers not"
2752                                           " supported");
2753         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2754                 return rte_flow_error_set(error, ENOTSUP,
2755                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2756                                           "L3 Layer is missing");
2757         if (!mask)
2758                 mask = &rte_flow_item_gre_mask;
2759         ret = mlx5_flow_item_acceptable
2760                 (item, (const uint8_t *)mask,
2761                  (const uint8_t *)&nic_mask,
2762                  sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2763                  error);
2764         if (ret < 0)
2765                 return ret;
2766 #ifndef HAVE_MLX5DV_DR
2767 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2768         if (spec && (spec->protocol & mask->protocol))
2769                 return rte_flow_error_set(error, ENOTSUP,
2770                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2771                                           "without MPLS support the"
2772                                           " specification cannot be used for"
2773                                           " filtering");
2774 #endif
2775 #endif
2776         return 0;
2777 }
2778
2779 /**
2780  * Validate Geneve item.
2781  *
2782  * @param[in] item
2783  *   Item specification.
2784  * @param[in] itemFlags
2785  *   Bit-fields that holds the items detected until now.
2786  * @param[in] enPriv
2787  *   Pointer to the private data structure.
2788  * @param[out] error
2789  *   Pointer to error structure.
2790  *
2791  * @return
2792  *   0 on success, a negative errno value otherwise and rte_errno is set.
2793  */
2794
2795 int
2796 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2797                                uint64_t item_flags,
2798                                struct rte_eth_dev *dev,
2799                                struct rte_flow_error *error)
2800 {
2801         struct mlx5_priv *priv = dev->data->dev_private;
2802         const struct rte_flow_item_geneve *spec = item->spec;
2803         const struct rte_flow_item_geneve *mask = item->mask;
2804         int ret;
2805         uint16_t gbhdr;
2806         uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2807                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2808         const struct rte_flow_item_geneve nic_mask = {
2809                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2810                 .vni = "\xff\xff\xff",
2811                 .protocol = RTE_BE16(UINT16_MAX),
2812         };
2813
2814         if (!priv->config.hca_attr.tunnel_stateless_geneve_rx)
2815                 return rte_flow_error_set(error, ENOTSUP,
2816                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2817                                           "L3 Geneve is not enabled by device"
2818                                           " parameter and/or not configured in"
2819                                           " firmware");
2820         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2821                 return rte_flow_error_set(error, ENOTSUP,
2822                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2823                                           "multiple tunnel layers not"
2824                                           " supported");
2825         /*
2826          * Verify only UDPv4 is present as defined in
2827          * https://tools.ietf.org/html/rfc7348
2828          */
2829         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2830                 return rte_flow_error_set(error, EINVAL,
2831                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2832                                           "no outer UDP layer found");
2833         if (!mask)
2834                 mask = &rte_flow_item_geneve_mask;
2835         ret = mlx5_flow_item_acceptable
2836                                   (item, (const uint8_t *)mask,
2837                                    (const uint8_t *)&nic_mask,
2838                                    sizeof(struct rte_flow_item_geneve),
2839                                    MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2840         if (ret)
2841                 return ret;
2842         if (spec) {
2843                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2844                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2845                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2846                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2847                         return rte_flow_error_set(error, ENOTSUP,
2848                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2849                                                   item,
2850                                                   "Geneve protocol unsupported"
2851                                                   " fields are being used");
2852                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2853                         return rte_flow_error_set
2854                                         (error, ENOTSUP,
2855                                          RTE_FLOW_ERROR_TYPE_ITEM,
2856                                          item,
2857                                          "Unsupported Geneve options length");
2858         }
2859         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2860                 return rte_flow_error_set
2861                                     (error, ENOTSUP,
2862                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
2863                                      "Geneve tunnel must be fully defined");
2864         return 0;
2865 }
2866
2867 /**
2868  * Validate Geneve TLV option item.
2869  *
2870  * @param[in] item
2871  *   Item specification.
2872  * @param[in] last_item
2873  *   Previous validated item in the pattern items.
2874  * @param[in] geneve_item
2875  *   Previous GENEVE item specification.
2876  * @param[in] dev
2877  *   Pointer to the rte_eth_dev structure.
2878  * @param[out] error
2879  *   Pointer to error structure.
2880  *
2881  * @return
2882  *   0 on success, a negative errno value otherwise and rte_errno is set.
2883  */
2884 int
2885 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item,
2886                                    uint64_t last_item,
2887                                    const struct rte_flow_item *geneve_item,
2888                                    struct rte_eth_dev *dev,
2889                                    struct rte_flow_error *error)
2890 {
2891         struct mlx5_priv *priv = dev->data->dev_private;
2892         struct mlx5_dev_ctx_shared *sh = priv->sh;
2893         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource;
2894         struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr;
2895         uint8_t data_max_supported =
2896                         hca_attr->max_geneve_tlv_option_data_len * 4;
2897         struct mlx5_dev_config *config = &priv->config;
2898         const struct rte_flow_item_geneve *geneve_spec;
2899         const struct rte_flow_item_geneve *geneve_mask;
2900         const struct rte_flow_item_geneve_opt *spec = item->spec;
2901         const struct rte_flow_item_geneve_opt *mask = item->mask;
2902         unsigned int i;
2903         unsigned int data_len;
2904         uint8_t tlv_option_len;
2905         uint16_t optlen_m, optlen_v;
2906         const struct rte_flow_item_geneve_opt full_mask = {
2907                 .option_class = RTE_BE16(0xffff),
2908                 .option_type = 0xff,
2909                 .option_len = 0x1f,
2910         };
2911
2912         if (!mask)
2913                 mask = &rte_flow_item_geneve_opt_mask;
2914         if (!spec)
2915                 return rte_flow_error_set
2916                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2917                         "Geneve TLV opt class/type/length must be specified");
2918         if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK)
2919                 return rte_flow_error_set
2920                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2921                         "Geneve TLV opt length exceeeds the limit (31)");
2922         /* Check if class type and length masks are full. */
2923         if (full_mask.option_class != mask->option_class ||
2924             full_mask.option_type != mask->option_type ||
2925             full_mask.option_len != (mask->option_len & full_mask.option_len))
2926                 return rte_flow_error_set
2927                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2928                         "Geneve TLV opt class/type/length masks must be full");
2929         /* Check if length is supported */
2930         if ((uint32_t)spec->option_len >
2931                         config->hca_attr.max_geneve_tlv_option_data_len)
2932                 return rte_flow_error_set
2933                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2934                         "Geneve TLV opt length not supported");
2935         if (config->hca_attr.max_geneve_tlv_options > 1)
2936                 DRV_LOG(DEBUG,
2937                         "max_geneve_tlv_options supports more than 1 option");
2938         /* Check GENEVE item preceding. */
2939         if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE))
2940                 return rte_flow_error_set
2941                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2942                         "Geneve opt item must be preceded with Geneve item");
2943         geneve_spec = geneve_item->spec;
2944         geneve_mask = geneve_item->mask ? geneve_item->mask :
2945                                           &rte_flow_item_geneve_mask;
2946         /* Check if GENEVE TLV option size doesn't exceed option length */
2947         if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 ||
2948                             geneve_spec->ver_opt_len_o_c_rsvd0)) {
2949                 tlv_option_len = spec->option_len & mask->option_len;
2950                 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0);
2951                 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v);
2952                 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0);
2953                 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m);
2954                 if ((optlen_v & optlen_m) <= tlv_option_len)
2955                         return rte_flow_error_set
2956                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2957                                  "GENEVE TLV option length exceeds optlen");
2958         }
2959         /* Check if length is 0 or data is 0. */
2960         if (spec->data == NULL || spec->option_len == 0)
2961                 return rte_flow_error_set
2962                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2963                         "Geneve TLV opt with zero data/length not supported");
2964         /* Check not all data & mask are 0. */
2965         data_len = spec->option_len * 4;
2966         if (mask->data == NULL) {
2967                 for (i = 0; i < data_len; i++)
2968                         if (spec->data[i])
2969                                 break;
2970                 if (i == data_len)
2971                         return rte_flow_error_set(error, ENOTSUP,
2972                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2973                                 "Can't match on Geneve option data 0");
2974         } else {
2975                 for (i = 0; i < data_len; i++)
2976                         if (spec->data[i] & mask->data[i])
2977                                 break;
2978                 if (i == data_len)
2979                         return rte_flow_error_set(error, ENOTSUP,
2980                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2981                                 "Can't match on Geneve option data and mask 0");
2982                 /* Check data mask supported. */
2983                 for (i = data_max_supported; i < data_len ; i++)
2984                         if (mask->data[i])
2985                                 return rte_flow_error_set(error, ENOTSUP,
2986                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2987                                         "Data mask is of unsupported size");
2988         }
2989         /* Check GENEVE option is supported in NIC. */
2990         if (!config->hca_attr.geneve_tlv_opt)
2991                 return rte_flow_error_set
2992                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2993                         "Geneve TLV opt not supported");
2994         /* Check if we already have geneve option with different type/class. */
2995         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
2996         geneve_opt_resource = sh->geneve_tlv_option_resource;
2997         if (geneve_opt_resource != NULL)
2998                 if (geneve_opt_resource->option_class != spec->option_class ||
2999                     geneve_opt_resource->option_type != spec->option_type ||
3000                     geneve_opt_resource->length != spec->option_len) {
3001                         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
3002                         return rte_flow_error_set(error, ENOTSUP,
3003                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
3004                                 "Only one Geneve TLV option supported");
3005                 }
3006         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
3007         return 0;
3008 }
3009
3010 /**
3011  * Validate MPLS item.
3012  *
3013  * @param[in] dev
3014  *   Pointer to the rte_eth_dev structure.
3015  * @param[in] item
3016  *   Item specification.
3017  * @param[in] item_flags
3018  *   Bit-fields that holds the items detected until now.
3019  * @param[in] prev_layer
3020  *   The protocol layer indicated in previous item.
3021  * @param[out] error
3022  *   Pointer to error structure.
3023  *
3024  * @return
3025  *   0 on success, a negative errno value otherwise and rte_errno is set.
3026  */
3027 int
3028 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
3029                              const struct rte_flow_item *item __rte_unused,
3030                              uint64_t item_flags __rte_unused,
3031                              uint64_t prev_layer __rte_unused,
3032                              struct rte_flow_error *error)
3033 {
3034 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
3035         const struct rte_flow_item_mpls *mask = item->mask;
3036         struct mlx5_priv *priv = dev->data->dev_private;
3037         int ret;
3038
3039         if (!priv->config.mpls_en)
3040                 return rte_flow_error_set(error, ENOTSUP,
3041                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3042                                           "MPLS not supported or"
3043                                           " disabled in firmware"
3044                                           " configuration.");
3045         /* MPLS over UDP, GRE is allowed */
3046         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP |
3047                             MLX5_FLOW_LAYER_GRE |
3048                             MLX5_FLOW_LAYER_GRE_KEY)))
3049                 return rte_flow_error_set(error, EINVAL,
3050                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3051                                           "protocol filtering not compatible"
3052                                           " with MPLS layer");
3053         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
3054         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
3055             !(item_flags & MLX5_FLOW_LAYER_GRE))
3056                 return rte_flow_error_set(error, ENOTSUP,
3057                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3058                                           "multiple tunnel layers not"
3059                                           " supported");
3060         if (!mask)
3061                 mask = &rte_flow_item_mpls_mask;
3062         ret = mlx5_flow_item_acceptable
3063                 (item, (const uint8_t *)mask,
3064                  (const uint8_t *)&rte_flow_item_mpls_mask,
3065                  sizeof(struct rte_flow_item_mpls),
3066                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3067         if (ret < 0)
3068                 return ret;
3069         return 0;
3070 #else
3071         return rte_flow_error_set(error, ENOTSUP,
3072                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
3073                                   "MPLS is not supported by Verbs, please"
3074                                   " update.");
3075 #endif
3076 }
3077
3078 /**
3079  * Validate NVGRE item.
3080  *
3081  * @param[in] item
3082  *   Item specification.
3083  * @param[in] item_flags
3084  *   Bit flags to mark detected items.
3085  * @param[in] target_protocol
3086  *   The next protocol in the previous item.
3087  * @param[out] error
3088  *   Pointer to error structure.
3089  *
3090  * @return
3091  *   0 on success, a negative errno value otherwise and rte_errno is set.
3092  */
3093 int
3094 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
3095                               uint64_t item_flags,
3096                               uint8_t target_protocol,
3097                               struct rte_flow_error *error)
3098 {
3099         const struct rte_flow_item_nvgre *mask = item->mask;
3100         int ret;
3101
3102         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
3103                 return rte_flow_error_set(error, EINVAL,
3104                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3105                                           "protocol filtering not compatible"
3106                                           " with this GRE layer");
3107         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3108                 return rte_flow_error_set(error, ENOTSUP,
3109                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3110                                           "multiple tunnel layers not"
3111                                           " supported");
3112         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
3113                 return rte_flow_error_set(error, ENOTSUP,
3114                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3115                                           "L3 Layer is missing");
3116         if (!mask)
3117                 mask = &rte_flow_item_nvgre_mask;
3118         ret = mlx5_flow_item_acceptable
3119                 (item, (const uint8_t *)mask,
3120                  (const uint8_t *)&rte_flow_item_nvgre_mask,
3121                  sizeof(struct rte_flow_item_nvgre),
3122                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3123         if (ret < 0)
3124                 return ret;
3125         return 0;
3126 }
3127
3128 /**
3129  * Validate eCPRI item.
3130  *
3131  * @param[in] item
3132  *   Item specification.
3133  * @param[in] item_flags
3134  *   Bit-fields that holds the items detected until now.
3135  * @param[in] last_item
3136  *   Previous validated item in the pattern items.
3137  * @param[in] ether_type
3138  *   Type in the ethernet layer header (including dot1q).
3139  * @param[in] acc_mask
3140  *   Acceptable mask, if NULL default internal default mask
3141  *   will be used to check whether item fields are supported.
3142  * @param[out] error
3143  *   Pointer to error structure.
3144  *
3145  * @return
3146  *   0 on success, a negative errno value otherwise and rte_errno is set.
3147  */
3148 int
3149 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
3150                               uint64_t item_flags,
3151                               uint64_t last_item,
3152                               uint16_t ether_type,
3153                               const struct rte_flow_item_ecpri *acc_mask,
3154                               struct rte_flow_error *error)
3155 {
3156         const struct rte_flow_item_ecpri *mask = item->mask;
3157         const struct rte_flow_item_ecpri nic_mask = {
3158                 .hdr = {
3159                         .common = {
3160                                 .u32 =
3161                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
3162                                         .type = 0xFF,
3163                                         }).u32),
3164                         },
3165                         .dummy[0] = 0xFFFFFFFF,
3166                 },
3167         };
3168         const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
3169                                         MLX5_FLOW_LAYER_OUTER_VLAN);
3170         struct rte_flow_item_ecpri mask_lo;
3171
3172         if (!(last_item & outer_l2_vlan) &&
3173             last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP)
3174                 return rte_flow_error_set(error, EINVAL,
3175                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3176                                           "eCPRI can only follow L2/VLAN layer or UDP layer");
3177         if ((last_item & outer_l2_vlan) && ether_type &&
3178             ether_type != RTE_ETHER_TYPE_ECPRI)
3179                 return rte_flow_error_set(error, EINVAL,
3180                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3181                                           "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE");
3182         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3183                 return rte_flow_error_set(error, EINVAL,
3184                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3185                                           "eCPRI with tunnel is not supported right now");
3186         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
3187                 return rte_flow_error_set(error, ENOTSUP,
3188                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3189                                           "multiple L3 layers not supported");
3190         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
3191                 return rte_flow_error_set(error, EINVAL,
3192                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3193                                           "eCPRI cannot coexist with a TCP layer");
3194         /* In specification, eCPRI could be over UDP layer. */
3195         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
3196                 return rte_flow_error_set(error, EINVAL,
3197                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3198                                           "eCPRI over UDP layer is not yet supported right now");
3199         /* Mask for type field in common header could be zero. */
3200         if (!mask)
3201                 mask = &rte_flow_item_ecpri_mask;
3202         mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
3203         /* Input mask is in big-endian format. */
3204         if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
3205                 return rte_flow_error_set(error, EINVAL,
3206                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3207                                           "partial mask is not supported for protocol");
3208         else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
3209                 return rte_flow_error_set(error, EINVAL,
3210                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3211                                           "message header mask must be after a type mask");
3212         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
3213                                          acc_mask ? (const uint8_t *)acc_mask
3214                                                   : (const uint8_t *)&nic_mask,
3215                                          sizeof(struct rte_flow_item_ecpri),
3216                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3217 }
3218
3219 static int
3220 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
3221                    const struct rte_flow_attr *attr __rte_unused,
3222                    const struct rte_flow_item items[] __rte_unused,
3223                    const struct rte_flow_action actions[] __rte_unused,
3224                    bool external __rte_unused,
3225                    int hairpin __rte_unused,
3226                    struct rte_flow_error *error)
3227 {
3228         return rte_flow_error_set(error, ENOTSUP,
3229                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3230 }
3231
3232 static struct mlx5_flow *
3233 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
3234                   const struct rte_flow_attr *attr __rte_unused,
3235                   const struct rte_flow_item items[] __rte_unused,
3236                   const struct rte_flow_action actions[] __rte_unused,
3237                   struct rte_flow_error *error)
3238 {
3239         rte_flow_error_set(error, ENOTSUP,
3240                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3241         return NULL;
3242 }
3243
3244 static int
3245 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
3246                     struct mlx5_flow *dev_flow __rte_unused,
3247                     const struct rte_flow_attr *attr __rte_unused,
3248                     const struct rte_flow_item items[] __rte_unused,
3249                     const struct rte_flow_action actions[] __rte_unused,
3250                     struct rte_flow_error *error)
3251 {
3252         return rte_flow_error_set(error, ENOTSUP,
3253                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3254 }
3255
3256 static int
3257 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
3258                 struct rte_flow *flow __rte_unused,
3259                 struct rte_flow_error *error)
3260 {
3261         return rte_flow_error_set(error, ENOTSUP,
3262                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3263 }
3264
3265 static void
3266 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
3267                  struct rte_flow *flow __rte_unused)
3268 {
3269 }
3270
3271 static void
3272 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
3273                   struct rte_flow *flow __rte_unused)
3274 {
3275 }
3276
3277 static int
3278 flow_null_query(struct rte_eth_dev *dev __rte_unused,
3279                 struct rte_flow *flow __rte_unused,
3280                 const struct rte_flow_action *actions __rte_unused,
3281                 void *data __rte_unused,
3282                 struct rte_flow_error *error)
3283 {
3284         return rte_flow_error_set(error, ENOTSUP,
3285                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3286 }
3287
3288 static int
3289 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused,
3290                       uint32_t domains __rte_unused,
3291                       uint32_t flags __rte_unused)
3292 {
3293         return 0;
3294 }
3295
3296 /* Void driver to protect from null pointer reference. */
3297 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
3298         .validate = flow_null_validate,
3299         .prepare = flow_null_prepare,
3300         .translate = flow_null_translate,
3301         .apply = flow_null_apply,
3302         .remove = flow_null_remove,
3303         .destroy = flow_null_destroy,
3304         .query = flow_null_query,
3305         .sync_domain = flow_null_sync_domain,
3306 };
3307
3308 /**
3309  * Select flow driver type according to flow attributes and device
3310  * configuration.
3311  *
3312  * @param[in] dev
3313  *   Pointer to the dev structure.
3314  * @param[in] attr
3315  *   Pointer to the flow attributes.
3316  *
3317  * @return
3318  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
3319  */
3320 static enum mlx5_flow_drv_type
3321 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
3322 {
3323         struct mlx5_priv *priv = dev->data->dev_private;
3324         /* The OS can determine first a specific flow type (DV, VERBS) */
3325         enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
3326
3327         if (type != MLX5_FLOW_TYPE_MAX)
3328                 return type;
3329         /* If no OS specific type - continue with DV/VERBS selection */
3330         if (attr->transfer && priv->config.dv_esw_en)
3331                 type = MLX5_FLOW_TYPE_DV;
3332         if (!attr->transfer)
3333                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
3334                                                  MLX5_FLOW_TYPE_VERBS;
3335         return type;
3336 }
3337
3338 #define flow_get_drv_ops(type) flow_drv_ops[type]
3339
3340 /**
3341  * Flow driver validation API. This abstracts calling driver specific functions.
3342  * The type of flow driver is determined according to flow attributes.
3343  *
3344  * @param[in] dev
3345  *   Pointer to the dev structure.
3346  * @param[in] attr
3347  *   Pointer to the flow attributes.
3348  * @param[in] items
3349  *   Pointer to the list of items.
3350  * @param[in] actions
3351  *   Pointer to the list of actions.
3352  * @param[in] external
3353  *   This flow rule is created by request external to PMD.
3354  * @param[in] hairpin
3355  *   Number of hairpin TX actions, 0 means classic flow.
3356  * @param[out] error
3357  *   Pointer to the error structure.
3358  *
3359  * @return
3360  *   0 on success, a negative errno value otherwise and rte_errno is set.
3361  */
3362 static inline int
3363 flow_drv_validate(struct rte_eth_dev *dev,
3364                   const struct rte_flow_attr *attr,
3365                   const struct rte_flow_item items[],
3366                   const struct rte_flow_action actions[],
3367                   bool external, int hairpin, struct rte_flow_error *error)
3368 {
3369         const struct mlx5_flow_driver_ops *fops;
3370         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
3371
3372         fops = flow_get_drv_ops(type);
3373         return fops->validate(dev, attr, items, actions, external,
3374                               hairpin, error);
3375 }
3376
3377 /**
3378  * Flow driver preparation API. This abstracts calling driver specific
3379  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3380  * calculates the size of memory required for device flow, allocates the memory,
3381  * initializes the device flow and returns the pointer.
3382  *
3383  * @note
3384  *   This function initializes device flow structure such as dv or verbs in
3385  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
3386  *   rest. For example, adding returning device flow to flow->dev_flow list and
3387  *   setting backward reference to the flow should be done out of this function.
3388  *   layers field is not filled either.
3389  *
3390  * @param[in] dev
3391  *   Pointer to the dev structure.
3392  * @param[in] attr
3393  *   Pointer to the flow attributes.
3394  * @param[in] items
3395  *   Pointer to the list of items.
3396  * @param[in] actions
3397  *   Pointer to the list of actions.
3398  * @param[in] flow_idx
3399  *   This memory pool index to the flow.
3400  * @param[out] error
3401  *   Pointer to the error structure.
3402  *
3403  * @return
3404  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
3405  */
3406 static inline struct mlx5_flow *
3407 flow_drv_prepare(struct rte_eth_dev *dev,
3408                  const struct rte_flow *flow,
3409                  const struct rte_flow_attr *attr,
3410                  const struct rte_flow_item items[],
3411                  const struct rte_flow_action actions[],
3412                  uint32_t flow_idx,
3413                  struct rte_flow_error *error)
3414 {
3415         const struct mlx5_flow_driver_ops *fops;
3416         enum mlx5_flow_drv_type type = flow->drv_type;
3417         struct mlx5_flow *mlx5_flow = NULL;
3418
3419         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3420         fops = flow_get_drv_ops(type);
3421         mlx5_flow = fops->prepare(dev, attr, items, actions, error);
3422         if (mlx5_flow)
3423                 mlx5_flow->flow_idx = flow_idx;
3424         return mlx5_flow;
3425 }
3426
3427 /**
3428  * Flow driver translation API. This abstracts calling driver specific
3429  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3430  * translates a generic flow into a driver flow. flow_drv_prepare() must
3431  * precede.
3432  *
3433  * @note
3434  *   dev_flow->layers could be filled as a result of parsing during translation
3435  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
3436  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
3437  *   flow->actions could be overwritten even though all the expanded dev_flows
3438  *   have the same actions.
3439  *
3440  * @param[in] dev
3441  *   Pointer to the rte dev structure.
3442  * @param[in, out] dev_flow
3443  *   Pointer to the mlx5 flow.
3444  * @param[in] attr
3445  *   Pointer to the flow attributes.
3446  * @param[in] items
3447  *   Pointer to the list of items.
3448  * @param[in] actions
3449  *   Pointer to the list of actions.
3450  * @param[out] error
3451  *   Pointer to the error structure.
3452  *
3453  * @return
3454  *   0 on success, a negative errno value otherwise and rte_errno is set.
3455  */
3456 static inline int
3457 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3458                    const struct rte_flow_attr *attr,
3459                    const struct rte_flow_item items[],
3460                    const struct rte_flow_action actions[],
3461                    struct rte_flow_error *error)
3462 {
3463         const struct mlx5_flow_driver_ops *fops;
3464         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
3465
3466         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3467         fops = flow_get_drv_ops(type);
3468         return fops->translate(dev, dev_flow, attr, items, actions, error);
3469 }
3470
3471 /**
3472  * Flow driver apply API. This abstracts calling driver specific functions.
3473  * Parent flow (rte_flow) should have driver type (drv_type). It applies
3474  * translated driver flows on to device. flow_drv_translate() must precede.
3475  *
3476  * @param[in] dev
3477  *   Pointer to Ethernet device structure.
3478  * @param[in, out] flow
3479  *   Pointer to flow structure.
3480  * @param[out] error
3481  *   Pointer to error structure.
3482  *
3483  * @return
3484  *   0 on success, a negative errno value otherwise and rte_errno is set.
3485  */
3486 static inline int
3487 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3488                struct rte_flow_error *error)
3489 {
3490         const struct mlx5_flow_driver_ops *fops;
3491         enum mlx5_flow_drv_type type = flow->drv_type;
3492
3493         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3494         fops = flow_get_drv_ops(type);
3495         return fops->apply(dev, flow, error);
3496 }
3497
3498 /**
3499  * Flow driver destroy API. This abstracts calling driver specific functions.
3500  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3501  * on device and releases resources of the flow.
3502  *
3503  * @param[in] dev
3504  *   Pointer to Ethernet device.
3505  * @param[in, out] flow
3506  *   Pointer to flow structure.
3507  */
3508 static inline void
3509 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3510 {
3511         const struct mlx5_flow_driver_ops *fops;
3512         enum mlx5_flow_drv_type type = flow->drv_type;
3513
3514         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3515         fops = flow_get_drv_ops(type);
3516         fops->destroy(dev, flow);
3517 }
3518
3519 /**
3520  * Flow driver find RSS policy tbl API. This abstracts calling driver
3521  * specific functions. Parent flow (rte_flow) should have driver
3522  * type (drv_type). It will find the RSS policy table that has the rss_desc.
3523  *
3524  * @param[in] dev
3525  *   Pointer to Ethernet device.
3526  * @param[in, out] flow
3527  *   Pointer to flow structure.
3528  * @param[in] policy
3529  *   Pointer to meter policy table.
3530  * @param[in] rss_desc
3531  *   Pointer to rss_desc
3532  */
3533 static struct mlx5_flow_meter_sub_policy *
3534 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
3535                 struct rte_flow *flow,
3536                 struct mlx5_flow_meter_policy *policy,
3537                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
3538 {
3539         const struct mlx5_flow_driver_ops *fops;
3540         enum mlx5_flow_drv_type type = flow->drv_type;
3541
3542         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3543         fops = flow_get_drv_ops(type);
3544         return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc);
3545 }
3546
3547 /**
3548  * Flow driver color tag rule API. This abstracts calling driver
3549  * specific functions. Parent flow (rte_flow) should have driver
3550  * type (drv_type). It will create the color tag rules in hierarchy meter.
3551  *
3552  * @param[in] dev
3553  *   Pointer to Ethernet device.
3554  * @param[in, out] flow
3555  *   Pointer to flow structure.
3556  * @param[in] fm
3557  *   Pointer to flow meter structure.
3558  * @param[in] src_port
3559  *   The src port this extra rule should use.
3560  * @param[in] item
3561  *   The src port id match item.
3562  * @param[out] error
3563  *   Pointer to error structure.
3564  */
3565 static int
3566 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev,
3567                 struct rte_flow *flow,
3568                 struct mlx5_flow_meter_info *fm,
3569                 int32_t src_port,
3570                 const struct rte_flow_item *item,
3571                 struct rte_flow_error *error)
3572 {
3573         const struct mlx5_flow_driver_ops *fops;
3574         enum mlx5_flow_drv_type type = flow->drv_type;
3575
3576         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3577         fops = flow_get_drv_ops(type);
3578         return fops->meter_hierarchy_rule_create(dev, fm,
3579                                                 src_port, item, error);
3580 }
3581
3582 /**
3583  * Get RSS action from the action list.
3584  *
3585  * @param[in] dev
3586  *   Pointer to Ethernet device.
3587  * @param[in] actions
3588  *   Pointer to the list of actions.
3589  * @param[in] flow
3590  *   Parent flow structure pointer.
3591  *
3592  * @return
3593  *   Pointer to the RSS action if exist, else return NULL.
3594  */
3595 static const struct rte_flow_action_rss*
3596 flow_get_rss_action(struct rte_eth_dev *dev,
3597                     const struct rte_flow_action actions[])
3598 {
3599         struct mlx5_priv *priv = dev->data->dev_private;
3600         const struct rte_flow_action_rss *rss = NULL;
3601         struct mlx5_meter_policy_action_container *acg;
3602         struct mlx5_meter_policy_action_container *acy;
3603
3604         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3605                 switch (actions->type) {
3606                 case RTE_FLOW_ACTION_TYPE_RSS:
3607                         rss = actions->conf;
3608                         break;
3609                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
3610                 {
3611                         const struct rte_flow_action_sample *sample =
3612                                                                 actions->conf;
3613                         const struct rte_flow_action *act = sample->actions;
3614                         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++)
3615                                 if (act->type == RTE_FLOW_ACTION_TYPE_RSS)
3616                                         rss = act->conf;
3617                         break;
3618                 }
3619                 case RTE_FLOW_ACTION_TYPE_METER:
3620                 {
3621                         uint32_t mtr_idx;
3622                         struct mlx5_flow_meter_info *fm;
3623                         struct mlx5_flow_meter_policy *policy;
3624                         const struct rte_flow_action_meter *mtr = actions->conf;
3625
3626                         fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx);
3627                         if (fm && !fm->def_policy) {
3628                                 policy = mlx5_flow_meter_policy_find(dev,
3629                                                 fm->policy_id, NULL);
3630                                 MLX5_ASSERT(policy);
3631                                 if (policy->is_hierarchy) {
3632                                         policy =
3633                                 mlx5_flow_meter_hierarchy_get_final_policy(dev,
3634                                                                         policy);
3635                                         if (!policy)
3636                                                 return NULL;
3637                                 }
3638                                 if (policy->is_rss) {
3639                                         acg =
3640                                         &policy->act_cnt[RTE_COLOR_GREEN];
3641                                         acy =
3642                                         &policy->act_cnt[RTE_COLOR_YELLOW];
3643                                         if (acg->fate_action ==
3644                                             MLX5_FLOW_FATE_SHARED_RSS)
3645                                                 rss = acg->rss->conf;
3646                                         else if (acy->fate_action ==
3647                                                  MLX5_FLOW_FATE_SHARED_RSS)
3648                                                 rss = acy->rss->conf;
3649                                 }
3650                         }
3651                         break;
3652                 }
3653                 default:
3654                         break;
3655                 }
3656         }
3657         return rss;
3658 }
3659
3660 /**
3661  * Get ASO age action by index.
3662  *
3663  * @param[in] dev
3664  *   Pointer to the Ethernet device structure.
3665  * @param[in] age_idx
3666  *   Index to the ASO age action.
3667  *
3668  * @return
3669  *   The specified ASO age action.
3670  */
3671 struct mlx5_aso_age_action*
3672 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx)
3673 {
3674         uint16_t pool_idx = age_idx & UINT16_MAX;
3675         uint16_t offset = (age_idx >> 16) & UINT16_MAX;
3676         struct mlx5_priv *priv = dev->data->dev_private;
3677         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
3678         struct mlx5_aso_age_pool *pool = mng->pools[pool_idx];
3679
3680         return &pool->actions[offset - 1];
3681 }
3682
3683 /* maps indirect action to translated direct in some actions array */
3684 struct mlx5_translated_action_handle {
3685         struct rte_flow_action_handle *action; /**< Indirect action handle. */
3686         int index; /**< Index in related array of rte_flow_action. */
3687 };
3688
3689 /**
3690  * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related
3691  * direct action if translation possible.
3692  * This functionality used to run same execution path for both direct and
3693  * indirect actions on flow create. All necessary preparations for indirect
3694  * action handling should be performed on *handle* actions list returned
3695  * from this call.
3696  *
3697  * @param[in] dev
3698  *   Pointer to Ethernet device.
3699  * @param[in] actions
3700  *   List of actions to translate.
3701  * @param[out] handle
3702  *   List to store translated indirect action object handles.
3703  * @param[in, out] indir_n
3704  *   Size of *handle* array. On return should be updated with number of
3705  *   indirect actions retrieved from the *actions* list.
3706  * @param[out] translated_actions
3707  *   List of actions where all indirect actions were translated to direct
3708  *   if possible. NULL if no translation took place.
3709  * @param[out] error
3710  *   Pointer to the error structure.
3711  *
3712  * @return
3713  *   0 on success, a negative errno value otherwise and rte_errno is set.
3714  */
3715 static int
3716 flow_action_handles_translate(struct rte_eth_dev *dev,
3717                               const struct rte_flow_action actions[],
3718                               struct mlx5_translated_action_handle *handle,
3719                               int *indir_n,
3720                               struct rte_flow_action **translated_actions,
3721                               struct rte_flow_error *error)
3722 {
3723         struct mlx5_priv *priv = dev->data->dev_private;
3724         struct rte_flow_action *translated = NULL;
3725         size_t actions_size;
3726         int n;
3727         int copied_n = 0;
3728         struct mlx5_translated_action_handle *handle_end = NULL;
3729
3730         for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
3731                 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT)
3732                         continue;
3733                 if (copied_n == *indir_n) {
3734                         return rte_flow_error_set
3735                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
3736                                  NULL, "too many shared actions");
3737                 }
3738                 rte_memcpy(&handle[copied_n].action, &actions[n].conf,
3739                            sizeof(actions[n].conf));
3740                 handle[copied_n].index = n;
3741                 copied_n++;
3742         }
3743         n++;
3744         *indir_n = copied_n;
3745         if (!copied_n)
3746                 return 0;
3747         actions_size = sizeof(struct rte_flow_action) * n;
3748         translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
3749         if (!translated) {
3750                 rte_errno = ENOMEM;
3751                 return -ENOMEM;
3752         }
3753         memcpy(translated, actions, actions_size);
3754         for (handle_end = handle + copied_n; handle < handle_end; handle++) {
3755                 struct mlx5_shared_action_rss *shared_rss;
3756                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
3757                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
3758                 uint32_t idx = act_idx &
3759                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
3760
3761                 switch (type) {
3762                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
3763                         shared_rss = mlx5_ipool_get
3764                           (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
3765                         translated[handle->index].type =
3766                                 RTE_FLOW_ACTION_TYPE_RSS;
3767                         translated[handle->index].conf =
3768                                 &shared_rss->origin;
3769                         break;
3770                 case MLX5_INDIRECT_ACTION_TYPE_COUNT:
3771                         translated[handle->index].type =
3772                                                 (enum rte_flow_action_type)
3773                                                 MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
3774                         translated[handle->index].conf = (void *)(uintptr_t)idx;
3775                         break;
3776                 case MLX5_INDIRECT_ACTION_TYPE_AGE:
3777                         if (priv->sh->flow_hit_aso_en) {
3778                                 translated[handle->index].type =
3779                                         (enum rte_flow_action_type)
3780                                         MLX5_RTE_FLOW_ACTION_TYPE_AGE;
3781                                 translated[handle->index].conf =
3782                                                          (void *)(uintptr_t)idx;
3783                                 break;
3784                         }
3785                         /* Fall-through */
3786                 case MLX5_INDIRECT_ACTION_TYPE_CT:
3787                         if (priv->sh->ct_aso_en) {
3788                                 translated[handle->index].type =
3789                                         RTE_FLOW_ACTION_TYPE_CONNTRACK;
3790                                 translated[handle->index].conf =
3791                                                          (void *)(uintptr_t)idx;
3792                                 break;
3793                         }
3794                         /* Fall-through */
3795                 default:
3796                         mlx5_free(translated);
3797                         return rte_flow_error_set
3798                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
3799                                  NULL, "invalid indirect action type");
3800                 }
3801         }
3802         *translated_actions = translated;
3803         return 0;
3804 }
3805
3806 /**
3807  * Get Shared RSS action from the action list.
3808  *
3809  * @param[in] dev
3810  *   Pointer to Ethernet device.
3811  * @param[in] shared
3812  *   Pointer to the list of actions.
3813  * @param[in] shared_n
3814  *   Actions list length.
3815  *
3816  * @return
3817  *   The MLX5 RSS action ID if exists, otherwise return 0.
3818  */
3819 static uint32_t
3820 flow_get_shared_rss_action(struct rte_eth_dev *dev,
3821                            struct mlx5_translated_action_handle *handle,
3822                            int shared_n)
3823 {
3824         struct mlx5_translated_action_handle *handle_end;
3825         struct mlx5_priv *priv = dev->data->dev_private;
3826         struct mlx5_shared_action_rss *shared_rss;
3827
3828
3829         for (handle_end = handle + shared_n; handle < handle_end; handle++) {
3830                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
3831                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
3832                 uint32_t idx = act_idx &
3833                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
3834                 switch (type) {
3835                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
3836                         shared_rss = mlx5_ipool_get
3837                                 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
3838                                                                            idx);
3839                         __atomic_add_fetch(&shared_rss->refcnt, 1,
3840                                            __ATOMIC_RELAXED);
3841                         return idx;
3842                 default:
3843                         break;
3844                 }
3845         }
3846         return 0;
3847 }
3848
3849 static unsigned int
3850 find_graph_root(uint32_t rss_level)
3851 {
3852         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
3853                                MLX5_EXPANSION_ROOT_OUTER;
3854 }
3855
3856 /**
3857  *  Get layer flags from the prefix flow.
3858  *
3859  *  Some flows may be split to several subflows, the prefix subflow gets the
3860  *  match items and the suffix sub flow gets the actions.
3861  *  Some actions need the user defined match item flags to get the detail for
3862  *  the action.
3863  *  This function helps the suffix flow to get the item layer flags from prefix
3864  *  subflow.
3865  *
3866  * @param[in] dev_flow
3867  *   Pointer the created preifx subflow.
3868  *
3869  * @return
3870  *   The layers get from prefix subflow.
3871  */
3872 static inline uint64_t
3873 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
3874 {
3875         uint64_t layers = 0;
3876
3877         /*
3878          * Layers bits could be localization, but usually the compiler will
3879          * help to do the optimization work for source code.
3880          * If no decap actions, use the layers directly.
3881          */
3882         if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
3883                 return dev_flow->handle->layers;
3884         /* Convert L3 layers with decap action. */
3885         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
3886                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3887         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
3888                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3889         /* Convert L4 layers with decap action.  */
3890         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
3891                 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3892         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
3893                 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3894         return layers;
3895 }
3896
3897 /**
3898  * Get metadata split action information.
3899  *
3900  * @param[in] actions
3901  *   Pointer to the list of actions.
3902  * @param[out] qrss
3903  *   Pointer to the return pointer.
3904  * @param[out] qrss_type
3905  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
3906  *   if no QUEUE/RSS is found.
3907  * @param[out] encap_idx
3908  *   Pointer to the index of the encap action if exists, otherwise the last
3909  *   action index.
3910  *
3911  * @return
3912  *   Total number of actions.
3913  */
3914 static int
3915 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
3916                                        const struct rte_flow_action **qrss,
3917                                        int *encap_idx)
3918 {
3919         const struct rte_flow_action_raw_encap *raw_encap;
3920         int actions_n = 0;
3921         int raw_decap_idx = -1;
3922
3923         *encap_idx = -1;
3924         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3925                 switch (actions->type) {
3926                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3927                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3928                         *encap_idx = actions_n;
3929                         break;
3930                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3931                         raw_decap_idx = actions_n;
3932                         break;
3933                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3934                         raw_encap = actions->conf;
3935                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3936                                 *encap_idx = raw_decap_idx != -1 ?
3937                                                       raw_decap_idx : actions_n;
3938                         break;
3939                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3940                 case RTE_FLOW_ACTION_TYPE_RSS:
3941                         *qrss = actions;
3942                         break;
3943                 default:
3944                         break;
3945                 }
3946                 actions_n++;
3947         }
3948         if (*encap_idx == -1)
3949                 *encap_idx = actions_n;
3950         /* Count RTE_FLOW_ACTION_TYPE_END. */
3951         return actions_n + 1;
3952 }
3953
3954 /**
3955  * Check if the action will change packet.
3956  *
3957  * @param dev
3958  *   Pointer to Ethernet device.
3959  * @param[in] type
3960  *   action type.
3961  *
3962  * @return
3963  *   true if action will change packet, false otherwise.
3964  */
3965 static bool flow_check_modify_action_type(struct rte_eth_dev *dev,
3966                                           enum rte_flow_action_type type)
3967 {
3968         struct mlx5_priv *priv = dev->data->dev_private;
3969
3970         switch (type) {
3971         case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3972         case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3973         case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3974         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3975         case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3976         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3977         case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3978         case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3979         case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3980         case RTE_FLOW_ACTION_TYPE_SET_TTL:
3981         case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
3982         case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
3983         case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
3984         case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
3985         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
3986         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
3987         case RTE_FLOW_ACTION_TYPE_SET_META:
3988         case RTE_FLOW_ACTION_TYPE_SET_TAG:
3989         case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3990         case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3991         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3992         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3993         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3994         case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3995         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3996         case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3997         case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3998         case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3999         case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
4000                 return true;
4001         case RTE_FLOW_ACTION_TYPE_FLAG:
4002         case RTE_FLOW_ACTION_TYPE_MARK:
4003                 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
4004                         return true;
4005                 else
4006                         return false;
4007         default:
4008                 return false;
4009         }
4010 }
4011
4012 /**
4013  * Check meter action from the action list.
4014  *
4015  * @param dev
4016  *   Pointer to Ethernet device.
4017  * @param[in] actions
4018  *   Pointer to the list of actions.
4019  * @param[out] has_mtr
4020  *   Pointer to the meter exist flag.
4021  * @param[out] has_modify
4022  *   Pointer to the flag showing there's packet change action.
4023  * @param[out] meter_id
4024  *   Pointer to the meter id.
4025  *
4026  * @return
4027  *   Total number of actions.
4028  */
4029 static int
4030 flow_check_meter_action(struct rte_eth_dev *dev,
4031                         const struct rte_flow_action actions[],
4032                         bool *has_mtr, bool *has_modify, uint32_t *meter_id)
4033 {
4034         const struct rte_flow_action_meter *mtr = NULL;
4035         int actions_n = 0;
4036
4037         MLX5_ASSERT(has_mtr);
4038         *has_mtr = false;
4039         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4040                 switch (actions->type) {
4041                 case RTE_FLOW_ACTION_TYPE_METER:
4042                         mtr = actions->conf;
4043                         *meter_id = mtr->mtr_id;
4044                         *has_mtr = true;
4045                         break;
4046                 default:
4047                         break;
4048                 }
4049                 if (!*has_mtr)
4050                         *has_modify |= flow_check_modify_action_type(dev,
4051                                                                 actions->type);
4052                 actions_n++;
4053         }
4054         /* Count RTE_FLOW_ACTION_TYPE_END. */
4055         return actions_n + 1;
4056 }
4057
4058 /**
4059  * Check if the flow should be split due to hairpin.
4060  * The reason for the split is that in current HW we can't
4061  * support encap and push-vlan on Rx, so if a flow contains
4062  * these actions we move it to Tx.
4063  *
4064  * @param dev
4065  *   Pointer to Ethernet device.
4066  * @param[in] attr
4067  *   Flow rule attributes.
4068  * @param[in] actions
4069  *   Associated actions (list terminated by the END action).
4070  *
4071  * @return
4072  *   > 0 the number of actions and the flow should be split,
4073  *   0 when no split required.
4074  */
4075 static int
4076 flow_check_hairpin_split(struct rte_eth_dev *dev,
4077                          const struct rte_flow_attr *attr,
4078                          const struct rte_flow_action actions[])
4079 {
4080         int queue_action = 0;
4081         int action_n = 0;
4082         int split = 0;
4083         const struct rte_flow_action_queue *queue;
4084         const struct rte_flow_action_rss *rss;
4085         const struct rte_flow_action_raw_encap *raw_encap;
4086         const struct rte_eth_hairpin_conf *conf;
4087
4088         if (!attr->ingress)
4089                 return 0;
4090         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4091                 switch (actions->type) {
4092                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4093                         queue = actions->conf;
4094                         if (queue == NULL)
4095                                 return 0;
4096                         conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
4097                         if (conf == NULL || conf->tx_explicit != 0)
4098                                 return 0;
4099                         queue_action = 1;
4100                         action_n++;
4101                         break;
4102                 case RTE_FLOW_ACTION_TYPE_RSS:
4103                         rss = actions->conf;
4104                         if (rss == NULL || rss->queue_num == 0)
4105                                 return 0;
4106                         conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
4107                         if (conf == NULL || conf->tx_explicit != 0)
4108                                 return 0;
4109                         queue_action = 1;
4110                         action_n++;
4111                         break;
4112                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4113                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4114                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4115                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4116                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4117                         split++;
4118                         action_n++;
4119                         break;
4120                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4121                         raw_encap = actions->conf;
4122                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4123                                 split++;
4124                         action_n++;
4125                         break;
4126                 default:
4127                         action_n++;
4128                         break;
4129                 }
4130         }
4131         if (split && queue_action)
4132                 return action_n;
4133         return 0;
4134 }
4135
4136 /* Declare flow create/destroy prototype in advance. */
4137 static uint32_t
4138 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
4139                  const struct rte_flow_attr *attr,
4140                  const struct rte_flow_item items[],
4141                  const struct rte_flow_action actions[],
4142                  bool external, struct rte_flow_error *error);
4143
4144 static void
4145 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
4146                   uint32_t flow_idx);
4147
4148 int
4149 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused,
4150                       struct mlx5_list_entry *entry, void *cb_ctx)
4151 {
4152         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4153         struct mlx5_flow_mreg_copy_resource *mcp_res =
4154                                container_of(entry, typeof(*mcp_res), hlist_ent);
4155
4156         return mcp_res->mark_id != *(uint32_t *)(ctx->data);
4157 }
4158
4159 struct mlx5_list_entry *
4160 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx)
4161 {
4162         struct rte_eth_dev *dev = tool_ctx;
4163         struct mlx5_priv *priv = dev->data->dev_private;
4164         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4165         struct mlx5_flow_mreg_copy_resource *mcp_res;
4166         struct rte_flow_error *error = ctx->error;
4167         uint32_t idx = 0;
4168         int ret;
4169         uint32_t mark_id = *(uint32_t *)(ctx->data);
4170         struct rte_flow_attr attr = {
4171                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4172                 .ingress = 1,
4173         };
4174         struct mlx5_rte_flow_item_tag tag_spec = {
4175                 .data = mark_id,
4176         };
4177         struct rte_flow_item items[] = {
4178                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
4179         };
4180         struct rte_flow_action_mark ftag = {
4181                 .id = mark_id,
4182         };
4183         struct mlx5_flow_action_copy_mreg cp_mreg = {
4184                 .dst = REG_B,
4185                 .src = REG_NON,
4186         };
4187         struct rte_flow_action_jump jump = {
4188                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4189         };
4190         struct rte_flow_action actions[] = {
4191                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
4192         };
4193
4194         /* Fill the register fileds in the flow. */
4195         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
4196         if (ret < 0)
4197                 return NULL;
4198         tag_spec.id = ret;
4199         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4200         if (ret < 0)
4201                 return NULL;
4202         cp_mreg.src = ret;
4203         /* Provide the full width of FLAG specific value. */
4204         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
4205                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
4206         /* Build a new flow. */
4207         if (mark_id != MLX5_DEFAULT_COPY_ID) {
4208                 items[0] = (struct rte_flow_item){
4209                         .type = (enum rte_flow_item_type)
4210                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4211                         .spec = &tag_spec,
4212                 };
4213                 items[1] = (struct rte_flow_item){
4214                         .type = RTE_FLOW_ITEM_TYPE_END,
4215                 };
4216                 actions[0] = (struct rte_flow_action){
4217                         .type = (enum rte_flow_action_type)
4218                                 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
4219                         .conf = &ftag,
4220                 };
4221                 actions[1] = (struct rte_flow_action){
4222                         .type = (enum rte_flow_action_type)
4223                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4224                         .conf = &cp_mreg,
4225                 };
4226                 actions[2] = (struct rte_flow_action){
4227                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4228                         .conf = &jump,
4229                 };
4230                 actions[3] = (struct rte_flow_action){
4231                         .type = RTE_FLOW_ACTION_TYPE_END,
4232                 };
4233         } else {
4234                 /* Default rule, wildcard match. */
4235                 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR;
4236                 items[0] = (struct rte_flow_item){
4237                         .type = RTE_FLOW_ITEM_TYPE_END,
4238                 };
4239                 actions[0] = (struct rte_flow_action){
4240                         .type = (enum rte_flow_action_type)
4241                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4242                         .conf = &cp_mreg,
4243                 };
4244                 actions[1] = (struct rte_flow_action){
4245                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4246                         .conf = &jump,
4247                 };
4248                 actions[2] = (struct rte_flow_action){
4249                         .type = RTE_FLOW_ACTION_TYPE_END,
4250                 };
4251         }
4252         /* Build a new entry. */
4253         mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
4254         if (!mcp_res) {
4255                 rte_errno = ENOMEM;
4256                 return NULL;
4257         }
4258         mcp_res->idx = idx;
4259         mcp_res->mark_id = mark_id;
4260         /*
4261          * The copy Flows are not included in any list. There
4262          * ones are referenced from other Flows and can not
4263          * be applied, removed, deleted in ardbitrary order
4264          * by list traversing.
4265          */
4266         mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP,
4267                                         &attr, items, actions, false, error);
4268         if (!mcp_res->rix_flow) {
4269                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
4270                 return NULL;
4271         }
4272         return &mcp_res->hlist_ent;
4273 }
4274
4275 struct mlx5_list_entry *
4276 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
4277                       void *cb_ctx __rte_unused)
4278 {
4279         struct rte_eth_dev *dev = tool_ctx;
4280         struct mlx5_priv *priv = dev->data->dev_private;
4281         struct mlx5_flow_mreg_copy_resource *mcp_res;
4282         uint32_t idx = 0;
4283
4284         mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
4285         if (!mcp_res) {
4286                 rte_errno = ENOMEM;
4287                 return NULL;
4288         }
4289         memcpy(mcp_res, oentry, sizeof(*mcp_res));
4290         mcp_res->idx = idx;
4291         return &mcp_res->hlist_ent;
4292 }
4293
4294 void
4295 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
4296 {
4297         struct mlx5_flow_mreg_copy_resource *mcp_res =
4298                                container_of(entry, typeof(*mcp_res), hlist_ent);
4299         struct rte_eth_dev *dev = tool_ctx;
4300         struct mlx5_priv *priv = dev->data->dev_private;
4301
4302         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
4303 }
4304
4305 /**
4306  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4307  *
4308  * As mark_id is unique, if there's already a registered flow for the mark_id,
4309  * return by increasing the reference counter of the resource. Otherwise, create
4310  * the resource (mcp_res) and flow.
4311  *
4312  * Flow looks like,
4313  *   - If ingress port is ANY and reg_c[1] is mark_id,
4314  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4315  *
4316  * For default flow (zero mark_id), flow is like,
4317  *   - If ingress port is ANY,
4318  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
4319  *
4320  * @param dev
4321  *   Pointer to Ethernet device.
4322  * @param mark_id
4323  *   ID of MARK action, zero means default flow for META.
4324  * @param[out] error
4325  *   Perform verbose error reporting if not NULL.
4326  *
4327  * @return
4328  *   Associated resource on success, NULL otherwise and rte_errno is set.
4329  */
4330 static struct mlx5_flow_mreg_copy_resource *
4331 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
4332                           struct rte_flow_error *error)
4333 {
4334         struct mlx5_priv *priv = dev->data->dev_private;
4335         struct mlx5_list_entry *entry;
4336         struct mlx5_flow_cb_ctx ctx = {
4337                 .dev = dev,
4338                 .error = error,
4339                 .data = &mark_id,
4340         };
4341
4342         /* Check if already registered. */
4343         MLX5_ASSERT(priv->mreg_cp_tbl);
4344         entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx);
4345         if (!entry)
4346                 return NULL;
4347         return container_of(entry, struct mlx5_flow_mreg_copy_resource,
4348                             hlist_ent);
4349 }
4350
4351 void
4352 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
4353 {
4354         struct mlx5_flow_mreg_copy_resource *mcp_res =
4355                                container_of(entry, typeof(*mcp_res), hlist_ent);
4356         struct rte_eth_dev *dev = tool_ctx;
4357         struct mlx5_priv *priv = dev->data->dev_private;
4358
4359         MLX5_ASSERT(mcp_res->rix_flow);
4360         flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow);
4361         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
4362 }
4363
4364 /**
4365  * Release flow in RX_CP_TBL.
4366  *
4367  * @param dev
4368  *   Pointer to Ethernet device.
4369  * @flow
4370  *   Parent flow for wich copying is provided.
4371  */
4372 static void
4373 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
4374                           struct rte_flow *flow)
4375 {
4376         struct mlx5_flow_mreg_copy_resource *mcp_res;
4377         struct mlx5_priv *priv = dev->data->dev_private;
4378
4379         if (!flow->rix_mreg_copy)
4380                 return;
4381         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
4382                                  flow->rix_mreg_copy);
4383         if (!mcp_res || !priv->mreg_cp_tbl)
4384                 return;
4385         MLX5_ASSERT(mcp_res->rix_flow);
4386         mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
4387         flow->rix_mreg_copy = 0;
4388 }
4389
4390 /**
4391  * Remove the default copy action from RX_CP_TBL.
4392  *
4393  * This functions is called in the mlx5_dev_start(). No thread safe
4394  * is guaranteed.
4395  *
4396  * @param dev
4397  *   Pointer to Ethernet device.
4398  */
4399 static void
4400 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
4401 {
4402         struct mlx5_list_entry *entry;
4403         struct mlx5_priv *priv = dev->data->dev_private;
4404         struct mlx5_flow_cb_ctx ctx;
4405         uint32_t mark_id;
4406
4407         /* Check if default flow is registered. */
4408         if (!priv->mreg_cp_tbl)
4409                 return;
4410         mark_id = MLX5_DEFAULT_COPY_ID;
4411         ctx.data = &mark_id;
4412         entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx);
4413         if (!entry)
4414                 return;
4415         mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
4416 }
4417
4418 /**
4419  * Add the default copy action in in RX_CP_TBL.
4420  *
4421  * This functions is called in the mlx5_dev_start(). No thread safe
4422  * is guaranteed.
4423  *
4424  * @param dev
4425  *   Pointer to Ethernet device.
4426  * @param[out] error
4427  *   Perform verbose error reporting if not NULL.
4428  *
4429  * @return
4430  *   0 for success, negative value otherwise and rte_errno is set.
4431  */
4432 static int
4433 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
4434                                   struct rte_flow_error *error)
4435 {
4436         struct mlx5_priv *priv = dev->data->dev_private;
4437         struct mlx5_flow_mreg_copy_resource *mcp_res;
4438         struct mlx5_flow_cb_ctx ctx;
4439         uint32_t mark_id;
4440
4441         /* Check whether extensive metadata feature is engaged. */
4442         if (!priv->config.dv_flow_en ||
4443             priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4444             !mlx5_flow_ext_mreg_supported(dev) ||
4445             !priv->sh->dv_regc0_mask)
4446                 return 0;
4447         /*
4448          * Add default mreg copy flow may be called multiple time, but
4449          * only be called once in stop. Avoid register it twice.
4450          */
4451         mark_id = MLX5_DEFAULT_COPY_ID;
4452         ctx.data = &mark_id;
4453         if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx))
4454                 return 0;
4455         mcp_res = flow_mreg_add_copy_action(dev, mark_id, error);
4456         if (!mcp_res)
4457                 return -rte_errno;
4458         return 0;
4459 }
4460
4461 /**
4462  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4463  *
4464  * All the flow having Q/RSS action should be split by
4465  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
4466  * performs the following,
4467  *   - CQE->flow_tag := reg_c[1] (MARK)
4468  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4469  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
4470  * but there should be a flow per each MARK ID set by MARK action.
4471  *
4472  * For the aforementioned reason, if there's a MARK action in flow's action
4473  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
4474  * the MARK ID to CQE's flow_tag like,
4475  *   - If reg_c[1] is mark_id,
4476  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4477  *
4478  * For SET_META action which stores value in reg_c[0], as the destination is
4479  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
4480  * MARK ID means the default flow. The default flow looks like,
4481  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4482  *
4483  * @param dev
4484  *   Pointer to Ethernet device.
4485  * @param flow
4486  *   Pointer to flow structure.
4487  * @param[in] actions
4488  *   Pointer to the list of actions.
4489  * @param[out] error
4490  *   Perform verbose error reporting if not NULL.
4491  *
4492  * @return
4493  *   0 on success, negative value otherwise and rte_errno is set.
4494  */
4495 static int
4496 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
4497                             struct rte_flow *flow,
4498                             const struct rte_flow_action *actions,
4499                             struct rte_flow_error *error)
4500 {
4501         struct mlx5_priv *priv = dev->data->dev_private;
4502         struct mlx5_dev_config *config = &priv->config;
4503         struct mlx5_flow_mreg_copy_resource *mcp_res;
4504         const struct rte_flow_action_mark *mark;
4505
4506         /* Check whether extensive metadata feature is engaged. */
4507         if (!config->dv_flow_en ||
4508             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4509             !mlx5_flow_ext_mreg_supported(dev) ||
4510             !priv->sh->dv_regc0_mask)
4511                 return 0;
4512         /* Find MARK action. */
4513         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4514                 switch (actions->type) {
4515                 case RTE_FLOW_ACTION_TYPE_FLAG:
4516                         mcp_res = flow_mreg_add_copy_action
4517                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
4518                         if (!mcp_res)
4519                                 return -rte_errno;
4520                         flow->rix_mreg_copy = mcp_res->idx;
4521                         return 0;
4522                 case RTE_FLOW_ACTION_TYPE_MARK:
4523                         mark = (const struct rte_flow_action_mark *)
4524                                 actions->conf;
4525                         mcp_res =
4526                                 flow_mreg_add_copy_action(dev, mark->id, error);
4527                         if (!mcp_res)
4528                                 return -rte_errno;
4529                         flow->rix_mreg_copy = mcp_res->idx;
4530                         return 0;
4531                 default:
4532                         break;
4533                 }
4534         }
4535         return 0;
4536 }
4537
4538 #define MLX5_MAX_SPLIT_ACTIONS 24
4539 #define MLX5_MAX_SPLIT_ITEMS 24
4540
4541 /**
4542  * Split the hairpin flow.
4543  * Since HW can't support encap and push-vlan on Rx, we move these
4544  * actions to Tx.
4545  * If the count action is after the encap then we also
4546  * move the count action. in this case the count will also measure
4547  * the outer bytes.
4548  *
4549  * @param dev
4550  *   Pointer to Ethernet device.
4551  * @param[in] actions
4552  *   Associated actions (list terminated by the END action).
4553  * @param[out] actions_rx
4554  *   Rx flow actions.
4555  * @param[out] actions_tx
4556  *   Tx flow actions..
4557  * @param[out] pattern_tx
4558  *   The pattern items for the Tx flow.
4559  * @param[out] flow_id
4560  *   The flow ID connected to this flow.
4561  *
4562  * @return
4563  *   0 on success.
4564  */
4565 static int
4566 flow_hairpin_split(struct rte_eth_dev *dev,
4567                    const struct rte_flow_action actions[],
4568                    struct rte_flow_action actions_rx[],
4569                    struct rte_flow_action actions_tx[],
4570                    struct rte_flow_item pattern_tx[],
4571                    uint32_t flow_id)
4572 {
4573         const struct rte_flow_action_raw_encap *raw_encap;
4574         const struct rte_flow_action_raw_decap *raw_decap;
4575         struct mlx5_rte_flow_action_set_tag *set_tag;
4576         struct rte_flow_action *tag_action;
4577         struct mlx5_rte_flow_item_tag *tag_item;
4578         struct rte_flow_item *item;
4579         char *addr;
4580         int encap = 0;
4581
4582         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4583                 switch (actions->type) {
4584                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4585                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4586                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4587                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4588                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4589                         rte_memcpy(actions_tx, actions,
4590                                sizeof(struct rte_flow_action));
4591                         actions_tx++;
4592                         break;
4593                 case RTE_FLOW_ACTION_TYPE_COUNT:
4594                         if (encap) {
4595                                 rte_memcpy(actions_tx, actions,
4596                                            sizeof(struct rte_flow_action));
4597                                 actions_tx++;
4598                         } else {
4599                                 rte_memcpy(actions_rx, actions,
4600                                            sizeof(struct rte_flow_action));
4601                                 actions_rx++;
4602                         }
4603                         break;
4604                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4605                         raw_encap = actions->conf;
4606                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) {
4607                                 memcpy(actions_tx, actions,
4608                                        sizeof(struct rte_flow_action));
4609                                 actions_tx++;
4610                                 encap = 1;
4611                         } else {
4612                                 rte_memcpy(actions_rx, actions,
4613                                            sizeof(struct rte_flow_action));
4614                                 actions_rx++;
4615                         }
4616                         break;
4617                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4618                         raw_decap = actions->conf;
4619                         if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) {
4620                                 memcpy(actions_tx, actions,
4621                                        sizeof(struct rte_flow_action));
4622                                 actions_tx++;
4623                         } else {
4624                                 rte_memcpy(actions_rx, actions,
4625                                            sizeof(struct rte_flow_action));
4626                                 actions_rx++;
4627                         }
4628                         break;
4629                 default:
4630                         rte_memcpy(actions_rx, actions,
4631                                    sizeof(struct rte_flow_action));
4632                         actions_rx++;
4633                         break;
4634                 }
4635         }
4636         /* Add set meta action and end action for the Rx flow. */
4637         tag_action = actions_rx;
4638         tag_action->type = (enum rte_flow_action_type)
4639                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4640         actions_rx++;
4641         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
4642         actions_rx++;
4643         set_tag = (void *)actions_rx;
4644         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
4645                 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL),
4646                 .data = flow_id,
4647         };
4648         MLX5_ASSERT(set_tag->id > REG_NON);
4649         tag_action->conf = set_tag;
4650         /* Create Tx item list. */
4651         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
4652         addr = (void *)&pattern_tx[2];
4653         item = pattern_tx;
4654         item->type = (enum rte_flow_item_type)
4655                      MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4656         tag_item = (void *)addr;
4657         tag_item->data = flow_id;
4658         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
4659         MLX5_ASSERT(set_tag->id > REG_NON);
4660         item->spec = tag_item;
4661         addr += sizeof(struct mlx5_rte_flow_item_tag);
4662         tag_item = (void *)addr;
4663         tag_item->data = UINT32_MAX;
4664         tag_item->id = UINT16_MAX;
4665         item->mask = tag_item;
4666         item->last = NULL;
4667         item++;
4668         item->type = RTE_FLOW_ITEM_TYPE_END;
4669         return 0;
4670 }
4671
4672 /**
4673  * The last stage of splitting chain, just creates the subflow
4674  * without any modification.
4675  *
4676  * @param[in] dev
4677  *   Pointer to Ethernet device.
4678  * @param[in] flow
4679  *   Parent flow structure pointer.
4680  * @param[in, out] sub_flow
4681  *   Pointer to return the created subflow, may be NULL.
4682  * @param[in] attr
4683  *   Flow rule attributes.
4684  * @param[in] items
4685  *   Pattern specification (list terminated by the END pattern item).
4686  * @param[in] actions
4687  *   Associated actions (list terminated by the END action).
4688  * @param[in] flow_split_info
4689  *   Pointer to flow split info structure.
4690  * @param[out] error
4691  *   Perform verbose error reporting if not NULL.
4692  * @return
4693  *   0 on success, negative value otherwise
4694  */
4695 static int
4696 flow_create_split_inner(struct rte_eth_dev *dev,
4697                         struct rte_flow *flow,
4698                         struct mlx5_flow **sub_flow,
4699                         const struct rte_flow_attr *attr,
4700                         const struct rte_flow_item items[],
4701                         const struct rte_flow_action actions[],
4702                         struct mlx5_flow_split_info *flow_split_info,
4703                         struct rte_flow_error *error)
4704 {
4705         struct mlx5_flow *dev_flow;
4706
4707         dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
4708                                     flow_split_info->flow_idx, error);
4709         if (!dev_flow)
4710                 return -rte_errno;
4711         dev_flow->flow = flow;
4712         dev_flow->external = flow_split_info->external;
4713         dev_flow->skip_scale = flow_split_info->skip_scale;
4714         /* Subflow object was created, we must include one in the list. */
4715         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4716                       dev_flow->handle, next);
4717         /*
4718          * If dev_flow is as one of the suffix flow, some actions in suffix
4719          * flow may need some user defined item layer flags, and pass the
4720          * Metadate rxq mark flag to suffix flow as well.
4721          */
4722         if (flow_split_info->prefix_layers)
4723                 dev_flow->handle->layers = flow_split_info->prefix_layers;
4724         if (flow_split_info->prefix_mark)
4725                 dev_flow->handle->mark = 1;
4726         if (sub_flow)
4727                 *sub_flow = dev_flow;
4728 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
4729         dev_flow->dv.table_id = flow_split_info->table_id;
4730 #endif
4731         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
4732 }
4733
4734 /**
4735  * Get the sub policy of a meter.
4736  *
4737  * @param[in] dev
4738  *   Pointer to Ethernet device.
4739  * @param[in] flow
4740  *   Parent flow structure pointer.
4741  * @param wks
4742  *   Pointer to thread flow work space.
4743  * @param[in] attr
4744  *   Flow rule attributes.
4745  * @param[in] items
4746  *   Pattern specification (list terminated by the END pattern item).
4747  * @param[out] error
4748  *   Perform verbose error reporting if not NULL.
4749  *
4750  * @return
4751  *   Pointer to the meter sub policy, NULL otherwise and rte_errno is set.
4752  */
4753 static struct mlx5_flow_meter_sub_policy *
4754 get_meter_sub_policy(struct rte_eth_dev *dev,
4755                      struct rte_flow *flow,
4756                      struct mlx5_flow_workspace *wks,
4757                      const struct rte_flow_attr *attr,
4758                      const struct rte_flow_item items[],
4759                      struct rte_flow_error *error)
4760 {
4761         struct mlx5_flow_meter_policy *policy;
4762         struct mlx5_flow_meter_policy *final_policy;
4763         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
4764
4765         policy = wks->policy;
4766         final_policy = policy->is_hierarchy ? wks->final_policy : policy;
4767         if (final_policy->is_rss || final_policy->is_queue) {
4768                 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS];
4769                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0};
4770                 uint32_t i;
4771
4772                 /*
4773                  * This is a tmp dev_flow,
4774                  * no need to register any matcher for it in translate.
4775                  */
4776                 wks->skip_matcher_reg = 1;
4777                 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
4778                         struct mlx5_flow dev_flow = {0};
4779                         struct mlx5_flow_handle dev_handle = { {0} };
4780                         uint8_t fate = final_policy->act_cnt[i].fate_action;
4781
4782                         if (fate == MLX5_FLOW_FATE_SHARED_RSS) {
4783                                 const struct rte_flow_action_rss *rss_act =
4784                                         final_policy->act_cnt[i].rss->conf;
4785                                 struct rte_flow_action rss_actions[2] = {
4786                                         [0] = {
4787                                         .type = RTE_FLOW_ACTION_TYPE_RSS,
4788                                         .conf = rss_act,
4789                                         },
4790                                         [1] = {
4791                                         .type = RTE_FLOW_ACTION_TYPE_END,
4792                                         .conf = NULL,
4793                                         }
4794                                 };
4795
4796                                 dev_flow.handle = &dev_handle;
4797                                 dev_flow.ingress = attr->ingress;
4798                                 dev_flow.flow = flow;
4799                                 dev_flow.external = 0;
4800 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
4801                                 dev_flow.dv.transfer = attr->transfer;
4802 #endif
4803                                 /**
4804                                  * Translate RSS action to get rss hash fields.
4805                                  */
4806                                 if (flow_drv_translate(dev, &dev_flow, attr,
4807                                                 items, rss_actions, error))
4808                                         goto exit;
4809                                 rss_desc_v[i] = wks->rss_desc;
4810                                 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN;
4811                                 rss_desc_v[i].hash_fields =
4812                                                 dev_flow.hash_fields;
4813                                 rss_desc_v[i].queue_num =
4814                                                 rss_desc_v[i].hash_fields ?
4815                                                 rss_desc_v[i].queue_num : 1;
4816                                 rss_desc_v[i].tunnel =
4817                                                 !!(dev_flow.handle->layers &
4818                                                    MLX5_FLOW_LAYER_TUNNEL);
4819                                 /* Use the RSS queues in the containers. */
4820                                 rss_desc_v[i].queue =
4821                                         (uint16_t *)(uintptr_t)rss_act->queue;
4822                                 rss_desc[i] = &rss_desc_v[i];
4823                         } else if (fate == MLX5_FLOW_FATE_QUEUE) {
4824                                 /* This is queue action. */
4825                                 rss_desc_v[i] = wks->rss_desc;
4826                                 rss_desc_v[i].key_len = 0;
4827                                 rss_desc_v[i].hash_fields = 0;
4828                                 rss_desc_v[i].queue =
4829                                         &final_policy->act_cnt[i].queue;
4830                                 rss_desc_v[i].queue_num = 1;
4831                                 rss_desc[i] = &rss_desc_v[i];
4832                         } else {
4833                                 rss_desc[i] = NULL;
4834                         }
4835                 }
4836                 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev,
4837                                                 flow, policy, rss_desc);
4838         } else {
4839                 enum mlx5_meter_domain mtr_domain =
4840                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
4841                                 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
4842                                                 MLX5_MTR_DOMAIN_INGRESS);
4843                 sub_policy = policy->sub_policys[mtr_domain][0];
4844         }
4845         if (!sub_policy)
4846                 rte_flow_error_set(error, EINVAL,
4847                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4848                                    "Failed to get meter sub-policy.");
4849 exit:
4850         return sub_policy;
4851 }
4852
4853 /**
4854  * Split the meter flow.
4855  *
4856  * As meter flow will split to three sub flow, other than meter
4857  * action, the other actions make sense to only meter accepts
4858  * the packet. If it need to be dropped, no other additional
4859  * actions should be take.
4860  *
4861  * One kind of special action which decapsulates the L3 tunnel
4862  * header will be in the prefix sub flow, as not to take the
4863  * L3 tunnel header into account.
4864  *
4865  * @param[in] dev
4866  *   Pointer to Ethernet device.
4867  * @param[in] flow
4868  *   Parent flow structure pointer.
4869  * @param wks
4870  *   Pointer to thread flow work space.
4871  * @param[in] attr
4872  *   Flow rule attributes.
4873  * @param[in] items
4874  *   Pattern specification (list terminated by the END pattern item).
4875  * @param[out] sfx_items
4876  *   Suffix flow match items (list terminated by the END pattern item).
4877  * @param[in] actions
4878  *   Associated actions (list terminated by the END action).
4879  * @param[out] actions_sfx
4880  *   Suffix flow actions.
4881  * @param[out] actions_pre
4882  *   Prefix flow actions.
4883  * @param[out] mtr_flow_id
4884  *   Pointer to meter flow id.
4885  * @param[out] error
4886  *   Perform verbose error reporting if not NULL.
4887  *
4888  * @return
4889  *   0 on success, a negative errno value otherwise and rte_errno is set.
4890  */
4891 static int
4892 flow_meter_split_prep(struct rte_eth_dev *dev,
4893                       struct rte_flow *flow,
4894                       struct mlx5_flow_workspace *wks,
4895                       const struct rte_flow_attr *attr,
4896                       const struct rte_flow_item items[],
4897                       struct rte_flow_item sfx_items[],
4898                       const struct rte_flow_action actions[],
4899                       struct rte_flow_action actions_sfx[],
4900                       struct rte_flow_action actions_pre[],
4901                       uint32_t *mtr_flow_id,
4902                       struct rte_flow_error *error)
4903 {
4904         struct mlx5_priv *priv = dev->data->dev_private;
4905         struct mlx5_flow_meter_info *fm = wks->fm;
4906         struct rte_flow_action *tag_action = NULL;
4907         struct rte_flow_item *tag_item;
4908         struct mlx5_rte_flow_action_set_tag *set_tag;
4909         const struct rte_flow_action_raw_encap *raw_encap;
4910         const struct rte_flow_action_raw_decap *raw_decap;
4911         struct mlx5_rte_flow_item_tag *tag_item_spec;
4912         struct mlx5_rte_flow_item_tag *tag_item_mask;
4913         uint32_t tag_id = 0;
4914         struct rte_flow_item *vlan_item_dst = NULL;
4915         const struct rte_flow_item *vlan_item_src = NULL;
4916         struct rte_flow_action *hw_mtr_action;
4917         struct rte_flow_action *action_pre_head = NULL;
4918         int32_t flow_src_port = priv->representor_id;
4919         bool mtr_first;
4920         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
4921         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
4922                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
4923         uint32_t flow_id = 0;
4924         uint32_t flow_id_reversed = 0;
4925         uint8_t flow_id_bits = 0;
4926         int shift;
4927
4928         /* Prepare the suffix subflow items. */
4929         tag_item = sfx_items++;
4930         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4931                 struct mlx5_priv *port_priv;
4932                 const struct rte_flow_item_port_id *pid_v;
4933                 int item_type = items->type;
4934
4935                 switch (item_type) {
4936                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4937                         pid_v = items->spec;
4938                         MLX5_ASSERT(pid_v);
4939                         port_priv = mlx5_port_to_eswitch_info(pid_v->id, false);
4940                         if (!port_priv)
4941                                 return rte_flow_error_set(error,
4942                                                 rte_errno,
4943                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
4944                                                 pid_v,
4945                                                 "Failed to get port info.");
4946                         flow_src_port = port_priv->representor_id;
4947                         if (!fm->def_policy && wks->policy->is_hierarchy &&
4948                             flow_src_port != priv->representor_id) {
4949                                 if (flow_drv_mtr_hierarchy_rule_create(dev,
4950                                                                 flow, fm,
4951                                                                 flow_src_port,
4952                                                                 items,
4953                                                                 error))
4954                                         return -rte_errno;
4955                         }
4956                         memcpy(sfx_items, items, sizeof(*sfx_items));
4957                         sfx_items++;
4958                         break;
4959                 case RTE_FLOW_ITEM_TYPE_VLAN:
4960                         /* Determine if copy vlan item below. */
4961                         vlan_item_src = items;
4962                         vlan_item_dst = sfx_items++;
4963                         vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID;
4964                         break;
4965                 default:
4966                         break;
4967                 }
4968         }
4969         sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
4970         sfx_items++;
4971         mtr_first = priv->sh->meter_aso_en &&
4972                 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX));
4973         /* For ASO meter, meter must be before tag in TX direction. */
4974         if (mtr_first) {
4975                 action_pre_head = actions_pre++;
4976                 /* Leave space for tag action. */
4977                 tag_action = actions_pre++;
4978         }
4979         /* Prepare the actions for prefix and suffix flow. */
4980         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4981                 struct rte_flow_action *action_cur = NULL;
4982
4983                 switch (actions->type) {
4984                 case RTE_FLOW_ACTION_TYPE_METER:
4985                         if (mtr_first) {
4986                                 action_cur = action_pre_head;
4987                         } else {
4988                                 /* Leave space for tag action. */
4989                                 tag_action = actions_pre++;
4990                                 action_cur = actions_pre++;
4991                         }
4992                         break;
4993                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4994                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4995                         action_cur = actions_pre++;
4996                         break;
4997                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4998                         raw_encap = actions->conf;
4999                         if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
5000                                 action_cur = actions_pre++;
5001                         break;
5002                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5003                         raw_decap = actions->conf;
5004                         if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
5005                                 action_cur = actions_pre++;
5006                         break;
5007                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5008                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5009                         if (vlan_item_dst && vlan_item_src) {
5010                                 memcpy(vlan_item_dst, vlan_item_src,
5011                                         sizeof(*vlan_item_dst));
5012                                 /*
5013                                  * Convert to internal match item, it is used
5014                                  * for vlan push and set vid.
5015                                  */
5016                                 vlan_item_dst->type = (enum rte_flow_item_type)
5017                                                 MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
5018                         }
5019                         break;
5020                 default:
5021                         break;
5022                 }
5023                 if (!action_cur)
5024                         action_cur = (fm->def_policy) ?
5025                                         actions_sfx++ : actions_pre++;
5026                 memcpy(action_cur, actions, sizeof(struct rte_flow_action));
5027         }
5028         /* Add end action to the actions. */
5029         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
5030         if (priv->sh->meter_aso_en) {
5031                 /**
5032                  * For ASO meter, need to add an extra jump action explicitly,
5033                  * to jump from meter to policer table.
5034                  */
5035                 struct mlx5_flow_meter_sub_policy *sub_policy;
5036                 struct mlx5_flow_tbl_data_entry *tbl_data;
5037
5038                 if (!fm->def_policy) {
5039                         sub_policy = get_meter_sub_policy(dev, flow, wks,
5040                                                           attr, items, error);
5041                         if (!sub_policy)
5042                                 return -rte_errno;
5043                 } else {
5044                         enum mlx5_meter_domain mtr_domain =
5045                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
5046                                 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
5047                                                 MLX5_MTR_DOMAIN_INGRESS);
5048
5049                         sub_policy =
5050                         &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy;
5051                 }
5052                 tbl_data = container_of(sub_policy->tbl_rsc,
5053                                         struct mlx5_flow_tbl_data_entry, tbl);
5054                 hw_mtr_action = actions_pre++;
5055                 hw_mtr_action->type = (enum rte_flow_action_type)
5056                                       MLX5_RTE_FLOW_ACTION_TYPE_JUMP;
5057                 hw_mtr_action->conf = tbl_data->jump.action;
5058         }
5059         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
5060         actions_pre++;
5061         if (!tag_action)
5062                 return rte_flow_error_set(error, ENOMEM,
5063                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5064                                           NULL, "No tag action space.");
5065         if (!mtr_flow_id) {
5066                 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID;
5067                 goto exit;
5068         }
5069         /* Only default-policy Meter creates mtr flow id. */
5070         if (fm->def_policy) {
5071                 mlx5_ipool_malloc(fm->flow_ipool, &tag_id);
5072                 if (!tag_id)
5073                         return rte_flow_error_set(error, ENOMEM,
5074                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5075                                         "Failed to allocate meter flow id.");
5076                 flow_id = tag_id - 1;
5077                 flow_id_bits = (!flow_id) ? 1 :
5078                                 (MLX5_REG_BITS - __builtin_clz(flow_id));
5079                 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) >
5080                     mtr_reg_bits) {
5081                         mlx5_ipool_free(fm->flow_ipool, tag_id);
5082                         return rte_flow_error_set(error, EINVAL,
5083                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5084                                         "Meter flow id exceeds max limit.");
5085                 }
5086                 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits)
5087                         priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits;
5088         }
5089         /* Build tag actions and items for meter_id/meter flow_id. */
5090         set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre;
5091         tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
5092         tag_item_mask = tag_item_spec + 1;
5093         /* Both flow_id and meter_id share the same register. */
5094         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
5095                 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
5096                                                             0, error),
5097                 .offset = mtr_id_offset,
5098                 .length = mtr_reg_bits,
5099                 .data = flow->meter,
5100         };
5101         /*
5102          * The color Reg bits used by flow_id are growing from
5103          * msb to lsb, so must do bit reverse for flow_id val in RegC.
5104          */
5105         for (shift = 0; shift < flow_id_bits; shift++)
5106                 flow_id_reversed = (flow_id_reversed << 1) |
5107                                 ((flow_id >> shift) & 0x1);
5108         set_tag->data |=
5109                 flow_id_reversed << (mtr_reg_bits - flow_id_bits);
5110         tag_item_spec->id = set_tag->id;
5111         tag_item_spec->data = set_tag->data << mtr_id_offset;
5112         tag_item_mask->data = UINT32_MAX << mtr_id_offset;
5113         tag_action->type = (enum rte_flow_action_type)
5114                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5115         tag_action->conf = set_tag;
5116         tag_item->type = (enum rte_flow_item_type)
5117                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG;
5118         tag_item->spec = tag_item_spec;
5119         tag_item->last = NULL;
5120         tag_item->mask = tag_item_mask;
5121 exit:
5122         if (mtr_flow_id)
5123                 *mtr_flow_id = tag_id;
5124         return 0;
5125 }
5126
5127 /**
5128  * Split action list having QUEUE/RSS for metadata register copy.
5129  *
5130  * Once Q/RSS action is detected in user's action list, the flow action
5131  * should be split in order to copy metadata registers, which will happen in
5132  * RX_CP_TBL like,
5133  *   - CQE->flow_tag := reg_c[1] (MARK)
5134  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
5135  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
5136  * This is because the last action of each flow must be a terminal action
5137  * (QUEUE, RSS or DROP).
5138  *
5139  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
5140  * stored and kept in the mlx5_flow structure per each sub_flow.
5141  *
5142  * The Q/RSS action is replaced with,
5143  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
5144  * And the following JUMP action is added at the end,
5145  *   - JUMP, to RX_CP_TBL.
5146  *
5147  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
5148  * flow_create_split_metadata() routine. The flow will look like,
5149  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
5150  *
5151  * @param dev
5152  *   Pointer to Ethernet device.
5153  * @param[out] split_actions
5154  *   Pointer to store split actions to jump to CP_TBL.
5155  * @param[in] actions
5156  *   Pointer to the list of original flow actions.
5157  * @param[in] qrss
5158  *   Pointer to the Q/RSS action.
5159  * @param[in] actions_n
5160  *   Number of original actions.
5161  * @param[out] error
5162  *   Perform verbose error reporting if not NULL.
5163  *
5164  * @return
5165  *   non-zero unique flow_id on success, otherwise 0 and
5166  *   error/rte_error are set.
5167  */
5168 static uint32_t
5169 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
5170                           struct rte_flow_action *split_actions,
5171                           const struct rte_flow_action *actions,
5172                           const struct rte_flow_action *qrss,
5173                           int actions_n, struct rte_flow_error *error)
5174 {
5175         struct mlx5_priv *priv = dev->data->dev_private;
5176         struct mlx5_rte_flow_action_set_tag *set_tag;
5177         struct rte_flow_action_jump *jump;
5178         const int qrss_idx = qrss - actions;
5179         uint32_t flow_id = 0;
5180         int ret = 0;
5181
5182         /*
5183          * Given actions will be split
5184          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
5185          * - Add jump to mreg CP_TBL.
5186          * As a result, there will be one more action.
5187          */
5188         ++actions_n;
5189         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
5190         set_tag = (void *)(split_actions + actions_n);
5191         /*
5192          * If tag action is not set to void(it means we are not the meter
5193          * suffix flow), add the tag action. Since meter suffix flow already
5194          * has the tag added.
5195          */
5196         if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
5197                 /*
5198                  * Allocate the new subflow ID. This one is unique within
5199                  * device and not shared with representors. Otherwise,
5200                  * we would have to resolve multi-thread access synch
5201                  * issue. Each flow on the shared device is appended
5202                  * with source vport identifier, so the resulting
5203                  * flows will be unique in the shared (by master and
5204                  * representors) domain even if they have coinciding
5205                  * IDs.
5206                  */
5207                 mlx5_ipool_malloc(priv->sh->ipool
5208                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id);
5209                 if (!flow_id)
5210                         return rte_flow_error_set(error, ENOMEM,
5211                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5212                                                   NULL, "can't allocate id "
5213                                                   "for split Q/RSS subflow");
5214                 /* Internal SET_TAG action to set flow ID. */
5215                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
5216                         .data = flow_id,
5217                 };
5218                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
5219                 if (ret < 0)
5220                         return ret;
5221                 set_tag->id = ret;
5222                 /* Construct new actions array. */
5223                 /* Replace QUEUE/RSS action. */
5224                 split_actions[qrss_idx] = (struct rte_flow_action){
5225                         .type = (enum rte_flow_action_type)
5226                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
5227                         .conf = set_tag,
5228                 };
5229         }
5230         /* JUMP action to jump to mreg copy table (CP_TBL). */
5231         jump = (void *)(set_tag + 1);
5232         *jump = (struct rte_flow_action_jump){
5233                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
5234         };
5235         split_actions[actions_n - 2] = (struct rte_flow_action){
5236                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
5237                 .conf = jump,
5238         };
5239         split_actions[actions_n - 1] = (struct rte_flow_action){
5240                 .type = RTE_FLOW_ACTION_TYPE_END,
5241         };
5242         return flow_id;
5243 }
5244
5245 /**
5246  * Extend the given action list for Tx metadata copy.
5247  *
5248  * Copy the given action list to the ext_actions and add flow metadata register
5249  * copy action in order to copy reg_a set by WQE to reg_c[0].
5250  *
5251  * @param[out] ext_actions
5252  *   Pointer to the extended action list.
5253  * @param[in] actions
5254  *   Pointer to the list of actions.
5255  * @param[in] actions_n
5256  *   Number of actions in the list.
5257  * @param[out] error
5258  *   Perform verbose error reporting if not NULL.
5259  * @param[in] encap_idx
5260  *   The encap action inndex.
5261  *
5262  * @return
5263  *   0 on success, negative value otherwise
5264  */
5265 static int
5266 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
5267                        struct rte_flow_action *ext_actions,
5268                        const struct rte_flow_action *actions,
5269                        int actions_n, struct rte_flow_error *error,
5270                        int encap_idx)
5271 {
5272         struct mlx5_flow_action_copy_mreg *cp_mreg =
5273                 (struct mlx5_flow_action_copy_mreg *)
5274                         (ext_actions + actions_n + 1);
5275         int ret;
5276
5277         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
5278         if (ret < 0)
5279                 return ret;
5280         cp_mreg->dst = ret;
5281         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
5282         if (ret < 0)
5283                 return ret;
5284         cp_mreg->src = ret;
5285         if (encap_idx != 0)
5286                 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
5287         if (encap_idx == actions_n - 1) {
5288                 ext_actions[actions_n - 1] = (struct rte_flow_action){
5289                         .type = (enum rte_flow_action_type)
5290                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5291                         .conf = cp_mreg,
5292                 };
5293                 ext_actions[actions_n] = (struct rte_flow_action){
5294                         .type = RTE_FLOW_ACTION_TYPE_END,
5295                 };
5296         } else {
5297                 ext_actions[encap_idx] = (struct rte_flow_action){
5298                         .type = (enum rte_flow_action_type)
5299                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5300                         .conf = cp_mreg,
5301                 };
5302                 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
5303                                 sizeof(*ext_actions) * (actions_n - encap_idx));
5304         }
5305         return 0;
5306 }
5307
5308 /**
5309  * Check the match action from the action list.
5310  *
5311  * @param[in] actions
5312  *   Pointer to the list of actions.
5313  * @param[in] attr
5314  *   Flow rule attributes.
5315  * @param[in] action
5316  *   The action to be check if exist.
5317  * @param[out] match_action_pos
5318  *   Pointer to the position of the matched action if exists, otherwise is -1.
5319  * @param[out] qrss_action_pos
5320  *   Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
5321  * @param[out] modify_after_mirror
5322  *   Pointer to the flag of modify action after FDB mirroring.
5323  *
5324  * @return
5325  *   > 0 the total number of actions.
5326  *   0 if not found match action in action list.
5327  */
5328 static int
5329 flow_check_match_action(const struct rte_flow_action actions[],
5330                         const struct rte_flow_attr *attr,
5331                         enum rte_flow_action_type action,
5332                         int *match_action_pos, int *qrss_action_pos,
5333                         int *modify_after_mirror)
5334 {
5335         const struct rte_flow_action_sample *sample;
5336         const struct rte_flow_action_raw_decap *decap;
5337         int actions_n = 0;
5338         uint32_t ratio = 0;
5339         int sub_type = 0;
5340         int flag = 0;
5341         int fdb_mirror = 0;
5342
5343         *match_action_pos = -1;
5344         *qrss_action_pos = -1;
5345         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5346                 if (actions->type == action) {
5347                         flag = 1;
5348                         *match_action_pos = actions_n;
5349                 }
5350                 switch (actions->type) {
5351                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5352                 case RTE_FLOW_ACTION_TYPE_RSS:
5353                         *qrss_action_pos = actions_n;
5354                         break;
5355                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
5356                         sample = actions->conf;
5357                         ratio = sample->ratio;
5358                         sub_type = ((const struct rte_flow_action *)
5359                                         (sample->actions))->type;
5360                         if (ratio == 1 && attr->transfer)
5361                                 fdb_mirror = 1;
5362                         break;
5363                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5364                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5365                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5366                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5367                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5368                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5369                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5370                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5371                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5372                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5373                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5374                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5375                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5376                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5377                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5378                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5379                 case RTE_FLOW_ACTION_TYPE_FLAG:
5380                 case RTE_FLOW_ACTION_TYPE_MARK:
5381                 case RTE_FLOW_ACTION_TYPE_SET_META:
5382                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5383                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5384                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5385                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5386                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5387                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5388                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5389                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
5390                 case RTE_FLOW_ACTION_TYPE_METER:
5391                         if (fdb_mirror)
5392                                 *modify_after_mirror = 1;
5393                         break;
5394                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5395                         decap = actions->conf;
5396                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5397                                 ;
5398                         actions_n++;
5399                         if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5400                                 const struct rte_flow_action_raw_encap *encap =
5401                                                                 actions->conf;
5402                                 if (decap->size <=
5403                                         MLX5_ENCAPSULATION_DECISION_SIZE &&
5404                                     encap->size >
5405                                         MLX5_ENCAPSULATION_DECISION_SIZE)
5406                                         /* L3 encap. */
5407                                         break;
5408                         }
5409                         if (fdb_mirror)
5410                                 *modify_after_mirror = 1;
5411                         break;
5412                 default:
5413                         break;
5414                 }
5415                 actions_n++;
5416         }
5417         if (flag && fdb_mirror && !*modify_after_mirror) {
5418                 /* FDB mirroring uses the destination array to implement
5419                  * instead of FLOW_SAMPLER object.
5420                  */
5421                 if (sub_type != RTE_FLOW_ACTION_TYPE_END)
5422                         flag = 0;
5423         }
5424         /* Count RTE_FLOW_ACTION_TYPE_END. */
5425         return flag ? actions_n + 1 : 0;
5426 }
5427
5428 #define SAMPLE_SUFFIX_ITEM 2
5429
5430 /**
5431  * Split the sample flow.
5432  *
5433  * As sample flow will split to two sub flow, sample flow with
5434  * sample action, the other actions will move to new suffix flow.
5435  *
5436  * Also add unique tag id with tag action in the sample flow,
5437  * the same tag id will be as match in the suffix flow.
5438  *
5439  * @param dev
5440  *   Pointer to Ethernet device.
5441  * @param[in] add_tag
5442  *   Add extra tag action flag.
5443  * @param[out] sfx_items
5444  *   Suffix flow match items (list terminated by the END pattern item).
5445  * @param[in] actions
5446  *   Associated actions (list terminated by the END action).
5447  * @param[out] actions_sfx
5448  *   Suffix flow actions.
5449  * @param[out] actions_pre
5450  *   Prefix flow actions.
5451  * @param[in] actions_n
5452  *  The total number of actions.
5453  * @param[in] sample_action_pos
5454  *   The sample action position.
5455  * @param[in] qrss_action_pos
5456  *   The Queue/RSS action position.
5457  * @param[in] jump_table
5458  *   Add extra jump action flag.
5459  * @param[out] error
5460  *   Perform verbose error reporting if not NULL.
5461  *
5462  * @return
5463  *   0 on success, or unique flow_id, a negative errno value
5464  *   otherwise and rte_errno is set.
5465  */
5466 static int
5467 flow_sample_split_prep(struct rte_eth_dev *dev,
5468                        int add_tag,
5469                        struct rte_flow_item sfx_items[],
5470                        const struct rte_flow_action actions[],
5471                        struct rte_flow_action actions_sfx[],
5472                        struct rte_flow_action actions_pre[],
5473                        int actions_n,
5474                        int sample_action_pos,
5475                        int qrss_action_pos,
5476                        int jump_table,
5477                        struct rte_flow_error *error)
5478 {
5479         struct mlx5_priv *priv = dev->data->dev_private;
5480         struct mlx5_rte_flow_action_set_tag *set_tag;
5481         struct mlx5_rte_flow_item_tag *tag_spec;
5482         struct mlx5_rte_flow_item_tag *tag_mask;
5483         struct rte_flow_action_jump *jump_action;
5484         uint32_t tag_id = 0;
5485         int index;
5486         int append_index = 0;
5487         int ret;
5488
5489         if (sample_action_pos < 0)
5490                 return rte_flow_error_set(error, EINVAL,
5491                                           RTE_FLOW_ERROR_TYPE_ACTION,
5492                                           NULL, "invalid position of sample "
5493                                           "action in list");
5494         /* Prepare the actions for prefix and suffix flow. */
5495         if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
5496                 index = qrss_action_pos;
5497                 /* Put the preceding the Queue/RSS action into prefix flow. */
5498                 if (index != 0)
5499                         memcpy(actions_pre, actions,
5500                                sizeof(struct rte_flow_action) * index);
5501                 /* Put others preceding the sample action into prefix flow. */
5502                 if (sample_action_pos > index + 1)
5503                         memcpy(actions_pre + index, actions + index + 1,
5504                                sizeof(struct rte_flow_action) *
5505                                (sample_action_pos - index - 1));
5506                 index = sample_action_pos - 1;
5507                 /* Put Queue/RSS action into Suffix flow. */
5508                 memcpy(actions_sfx, actions + qrss_action_pos,
5509                        sizeof(struct rte_flow_action));
5510                 actions_sfx++;
5511         } else {
5512                 index = sample_action_pos;
5513                 if (index != 0)
5514                         memcpy(actions_pre, actions,
5515                                sizeof(struct rte_flow_action) * index);
5516         }
5517         /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress.
5518          * For CX6DX and above, metadata registers Cx preserve their value,
5519          * add an extra tag action for NIC-RX and E-Switch Domain.
5520          */
5521         if (add_tag) {
5522                 /* Prepare the prefix tag action. */
5523                 append_index++;
5524                 set_tag = (void *)(actions_pre + actions_n + append_index);
5525                 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
5526                 if (ret < 0)
5527                         return ret;
5528                 mlx5_ipool_malloc(priv->sh->ipool
5529                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id);
5530                 *set_tag = (struct mlx5_rte_flow_action_set_tag) {
5531                         .id = ret,
5532                         .data = tag_id,
5533                 };
5534                 /* Prepare the suffix subflow items. */
5535                 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
5536                 tag_spec->data = tag_id;
5537                 tag_spec->id = set_tag->id;
5538                 tag_mask = tag_spec + 1;
5539                 tag_mask->data = UINT32_MAX;
5540                 sfx_items[0] = (struct rte_flow_item){
5541                         .type = (enum rte_flow_item_type)
5542                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5543                         .spec = tag_spec,
5544                         .last = NULL,
5545                         .mask = tag_mask,
5546                 };
5547                 sfx_items[1] = (struct rte_flow_item){
5548                         .type = (enum rte_flow_item_type)
5549                                 RTE_FLOW_ITEM_TYPE_END,
5550                 };
5551                 /* Prepare the tag action in prefix subflow. */
5552                 actions_pre[index++] =
5553                         (struct rte_flow_action){
5554                         .type = (enum rte_flow_action_type)
5555                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
5556                         .conf = set_tag,
5557                 };
5558         }
5559         memcpy(actions_pre + index, actions + sample_action_pos,
5560                sizeof(struct rte_flow_action));
5561         index += 1;
5562         /* For the modify action after the sample action in E-Switch mirroring,
5563          * Add the extra jump action in prefix subflow and jump into the next
5564          * table, then do the modify action in the new table.
5565          */
5566         if (jump_table) {
5567                 /* Prepare the prefix jump action. */
5568                 append_index++;
5569                 jump_action = (void *)(actions_pre + actions_n + append_index);
5570                 jump_action->group = jump_table;
5571                 actions_pre[index++] =
5572                         (struct rte_flow_action){
5573                         .type = (enum rte_flow_action_type)
5574                                 RTE_FLOW_ACTION_TYPE_JUMP,
5575                         .conf = jump_action,
5576                 };
5577         }
5578         actions_pre[index] = (struct rte_flow_action){
5579                 .type = (enum rte_flow_action_type)
5580                         RTE_FLOW_ACTION_TYPE_END,
5581         };
5582         /* Put the actions after sample into Suffix flow. */
5583         memcpy(actions_sfx, actions + sample_action_pos + 1,
5584                sizeof(struct rte_flow_action) *
5585                (actions_n - sample_action_pos - 1));
5586         return tag_id;
5587 }
5588
5589 /**
5590  * The splitting for metadata feature.
5591  *
5592  * - Q/RSS action on NIC Rx should be split in order to pass by
5593  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
5594  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
5595  *
5596  * - All the actions on NIC Tx should have a mreg copy action to
5597  *   copy reg_a from WQE to reg_c[0].
5598  *
5599  * @param dev
5600  *   Pointer to Ethernet device.
5601  * @param[in] flow
5602  *   Parent flow structure pointer.
5603  * @param[in] attr
5604  *   Flow rule attributes.
5605  * @param[in] items
5606  *   Pattern specification (list terminated by the END pattern item).
5607  * @param[in] actions
5608  *   Associated actions (list terminated by the END action).
5609  * @param[in] flow_split_info
5610  *   Pointer to flow split info structure.
5611  * @param[out] error
5612  *   Perform verbose error reporting if not NULL.
5613  * @return
5614  *   0 on success, negative value otherwise
5615  */
5616 static int
5617 flow_create_split_metadata(struct rte_eth_dev *dev,
5618                            struct rte_flow *flow,
5619                            const struct rte_flow_attr *attr,
5620                            const struct rte_flow_item items[],
5621                            const struct rte_flow_action actions[],
5622                            struct mlx5_flow_split_info *flow_split_info,
5623                            struct rte_flow_error *error)
5624 {
5625         struct mlx5_priv *priv = dev->data->dev_private;
5626         struct mlx5_dev_config *config = &priv->config;
5627         const struct rte_flow_action *qrss = NULL;
5628         struct rte_flow_action *ext_actions = NULL;
5629         struct mlx5_flow *dev_flow = NULL;
5630         uint32_t qrss_id = 0;
5631         int mtr_sfx = 0;
5632         size_t act_size;
5633         int actions_n;
5634         int encap_idx;
5635         int ret;
5636
5637         /* Check whether extensive metadata feature is engaged. */
5638         if (!config->dv_flow_en ||
5639             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
5640             !mlx5_flow_ext_mreg_supported(dev))
5641                 return flow_create_split_inner(dev, flow, NULL, attr, items,
5642                                                actions, flow_split_info, error);
5643         actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
5644                                                            &encap_idx);
5645         if (qrss) {
5646                 /* Exclude hairpin flows from splitting. */
5647                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
5648                         const struct rte_flow_action_queue *queue;
5649
5650                         queue = qrss->conf;
5651                         if (mlx5_rxq_get_type(dev, queue->index) ==
5652                             MLX5_RXQ_TYPE_HAIRPIN)
5653                                 qrss = NULL;
5654                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
5655                         const struct rte_flow_action_rss *rss;
5656
5657                         rss = qrss->conf;
5658                         if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
5659                             MLX5_RXQ_TYPE_HAIRPIN)
5660                                 qrss = NULL;
5661                 }
5662         }
5663         if (qrss) {
5664                 /* Check if it is in meter suffix table. */
5665                 mtr_sfx = attr->group == (attr->transfer ?
5666                           (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
5667                           MLX5_FLOW_TABLE_LEVEL_METER);
5668                 /*
5669                  * Q/RSS action on NIC Rx should be split in order to pass by
5670                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
5671                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
5672                  */
5673                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5674                            sizeof(struct rte_flow_action_set_tag) +
5675                            sizeof(struct rte_flow_action_jump);
5676                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5677                                           SOCKET_ID_ANY);
5678                 if (!ext_actions)
5679                         return rte_flow_error_set(error, ENOMEM,
5680                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5681                                                   NULL, "no memory to split "
5682                                                   "metadata flow");
5683                 /*
5684                  * If we are the suffix flow of meter, tag already exist.
5685                  * Set the tag action to void.
5686                  */
5687                 if (mtr_sfx)
5688                         ext_actions[qrss - actions].type =
5689                                                 RTE_FLOW_ACTION_TYPE_VOID;
5690                 else
5691                         ext_actions[qrss - actions].type =
5692                                                 (enum rte_flow_action_type)
5693                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5694                 /*
5695                  * Create the new actions list with removed Q/RSS action
5696                  * and appended set tag and jump to register copy table
5697                  * (RX_CP_TBL). We should preallocate unique tag ID here
5698                  * in advance, because it is needed for set tag action.
5699                  */
5700                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
5701                                                     qrss, actions_n, error);
5702                 if (!mtr_sfx && !qrss_id) {
5703                         ret = -rte_errno;
5704                         goto exit;
5705                 }
5706         } else if (attr->egress && !attr->transfer) {
5707                 /*
5708                  * All the actions on NIC Tx should have a metadata register
5709                  * copy action to copy reg_a from WQE to reg_c[meta]
5710                  */
5711                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5712                            sizeof(struct mlx5_flow_action_copy_mreg);
5713                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5714                                           SOCKET_ID_ANY);
5715                 if (!ext_actions)
5716                         return rte_flow_error_set(error, ENOMEM,
5717                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5718                                                   NULL, "no memory to split "
5719                                                   "metadata flow");
5720                 /* Create the action list appended with copy register. */
5721                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
5722                                              actions_n, error, encap_idx);
5723                 if (ret < 0)
5724                         goto exit;
5725         }
5726         /* Add the unmodified original or prefix subflow. */
5727         ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
5728                                       items, ext_actions ? ext_actions :
5729                                       actions, flow_split_info, error);
5730         if (ret < 0)
5731                 goto exit;
5732         MLX5_ASSERT(dev_flow);
5733         if (qrss) {
5734                 const struct rte_flow_attr q_attr = {
5735                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
5736                         .ingress = 1,
5737                 };
5738                 /* Internal PMD action to set register. */
5739                 struct mlx5_rte_flow_item_tag q_tag_spec = {
5740                         .data = qrss_id,
5741                         .id = REG_NON,
5742                 };
5743                 struct rte_flow_item q_items[] = {
5744                         {
5745                                 .type = (enum rte_flow_item_type)
5746                                         MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5747                                 .spec = &q_tag_spec,
5748                                 .last = NULL,
5749                                 .mask = NULL,
5750                         },
5751                         {
5752                                 .type = RTE_FLOW_ITEM_TYPE_END,
5753                         },
5754                 };
5755                 struct rte_flow_action q_actions[] = {
5756                         {
5757                                 .type = qrss->type,
5758                                 .conf = qrss->conf,
5759                         },
5760                         {
5761                                 .type = RTE_FLOW_ACTION_TYPE_END,
5762                         },
5763                 };
5764                 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
5765
5766                 /*
5767                  * Configure the tag item only if there is no meter subflow.
5768                  * Since tag is already marked in the meter suffix subflow
5769                  * we can just use the meter suffix items as is.
5770                  */
5771                 if (qrss_id) {
5772                         /* Not meter subflow. */
5773                         MLX5_ASSERT(!mtr_sfx);
5774                         /*
5775                          * Put unique id in prefix flow due to it is destroyed
5776                          * after suffix flow and id will be freed after there
5777                          * is no actual flows with this id and identifier
5778                          * reallocation becomes possible (for example, for
5779                          * other flows in other threads).
5780                          */
5781                         dev_flow->handle->split_flow_id = qrss_id;
5782                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
5783                                                    error);
5784                         if (ret < 0)
5785                                 goto exit;
5786                         q_tag_spec.id = ret;
5787                 }
5788                 dev_flow = NULL;
5789                 /* Add suffix subflow to execute Q/RSS. */
5790                 flow_split_info->prefix_layers = layers;
5791                 flow_split_info->prefix_mark = 0;
5792                 ret = flow_create_split_inner(dev, flow, &dev_flow,
5793                                               &q_attr, mtr_sfx ? items :
5794                                               q_items, q_actions,
5795                                               flow_split_info, error);
5796                 if (ret < 0)
5797                         goto exit;
5798                 /* qrss ID should be freed if failed. */
5799                 qrss_id = 0;
5800                 MLX5_ASSERT(dev_flow);
5801         }
5802
5803 exit:
5804         /*
5805          * We do not destroy the partially created sub_flows in case of error.
5806          * These ones are included into parent flow list and will be destroyed
5807          * by flow_drv_destroy.
5808          */
5809         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
5810                         qrss_id);
5811         mlx5_free(ext_actions);
5812         return ret;
5813 }
5814
5815 /**
5816  * Create meter internal drop flow with the original pattern.
5817  *
5818  * @param dev
5819  *   Pointer to Ethernet device.
5820  * @param[in] flow
5821  *   Parent flow structure pointer.
5822  * @param[in] attr
5823  *   Flow rule attributes.
5824  * @param[in] items
5825  *   Pattern specification (list terminated by the END pattern item).
5826  * @param[in] flow_split_info
5827  *   Pointer to flow split info structure.
5828  * @param[in] fm
5829  *   Pointer to flow meter structure.
5830  * @param[out] error
5831  *   Perform verbose error reporting if not NULL.
5832  * @return
5833  *   0 on success, negative value otherwise
5834  */
5835 static uint32_t
5836 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev,
5837                         struct rte_flow *flow,
5838                         const struct rte_flow_attr *attr,
5839                         const struct rte_flow_item items[],
5840                         struct mlx5_flow_split_info *flow_split_info,
5841                         struct mlx5_flow_meter_info *fm,
5842                         struct rte_flow_error *error)
5843 {
5844         struct mlx5_flow *dev_flow = NULL;
5845         struct rte_flow_attr drop_attr = *attr;
5846         struct rte_flow_action drop_actions[3];
5847         struct mlx5_flow_split_info drop_split_info = *flow_split_info;
5848
5849         MLX5_ASSERT(fm->drop_cnt);
5850         drop_actions[0].type =
5851                 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
5852         drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt;
5853         drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP;
5854         drop_actions[1].conf = NULL;
5855         drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END;
5856         drop_actions[2].conf = NULL;
5857         drop_split_info.external = false;
5858         drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT;
5859         drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP;
5860         drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER;
5861         return flow_create_split_inner(dev, flow, &dev_flow,
5862                                 &drop_attr, items, drop_actions,
5863                                 &drop_split_info, error);
5864 }
5865
5866 /**
5867  * The splitting for meter feature.
5868  *
5869  * - The meter flow will be split to two flows as prefix and
5870  *   suffix flow. The packets make sense only it pass the prefix
5871  *   meter action.
5872  *
5873  * - Reg_C_5 is used for the packet to match betweend prefix and
5874  *   suffix flow.
5875  *
5876  * @param dev
5877  *   Pointer to Ethernet device.
5878  * @param[in] flow
5879  *   Parent flow structure pointer.
5880  * @param[in] attr
5881  *   Flow rule attributes.
5882  * @param[in] items
5883  *   Pattern specification (list terminated by the END pattern item).
5884  * @param[in] actions
5885  *   Associated actions (list terminated by the END action).
5886  * @param[in] flow_split_info
5887  *   Pointer to flow split info structure.
5888  * @param[out] error
5889  *   Perform verbose error reporting if not NULL.
5890  * @return
5891  *   0 on success, negative value otherwise
5892  */
5893 static int
5894 flow_create_split_meter(struct rte_eth_dev *dev,
5895                         struct rte_flow *flow,
5896                         const struct rte_flow_attr *attr,
5897                         const struct rte_flow_item items[],
5898                         const struct rte_flow_action actions[],
5899                         struct mlx5_flow_split_info *flow_split_info,
5900                         struct rte_flow_error *error)
5901 {
5902         struct mlx5_priv *priv = dev->data->dev_private;
5903         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
5904         struct rte_flow_action *sfx_actions = NULL;
5905         struct rte_flow_action *pre_actions = NULL;
5906         struct rte_flow_item *sfx_items = NULL;
5907         struct mlx5_flow *dev_flow = NULL;
5908         struct rte_flow_attr sfx_attr = *attr;
5909         struct mlx5_flow_meter_info *fm = NULL;
5910         uint8_t skip_scale_restore;
5911         bool has_mtr = false;
5912         bool has_modify = false;
5913         bool set_mtr_reg = true;
5914         bool is_mtr_hierarchy = false;
5915         uint32_t meter_id = 0;
5916         uint32_t mtr_idx = 0;
5917         uint32_t mtr_flow_id = 0;
5918         size_t act_size;
5919         size_t item_size;
5920         int actions_n = 0;
5921         int ret = 0;
5922
5923         if (priv->mtr_en)
5924                 actions_n = flow_check_meter_action(dev, actions, &has_mtr,
5925                                                     &has_modify, &meter_id);
5926         if (has_mtr) {
5927                 if (flow->meter) {
5928                         fm = flow_dv_meter_find_by_idx(priv, flow->meter);
5929                         if (!fm)
5930                                 return rte_flow_error_set(error, EINVAL,
5931                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5932                                                 NULL, "Meter not found.");
5933                 } else {
5934                         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
5935                         if (!fm)
5936                                 return rte_flow_error_set(error, EINVAL,
5937                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5938                                                 NULL, "Meter not found.");
5939                         ret = mlx5_flow_meter_attach(priv, fm,
5940                                                      &sfx_attr, error);
5941                         if (ret)
5942                                 return -rte_errno;
5943                         flow->meter = mtr_idx;
5944                 }
5945                 MLX5_ASSERT(wks);
5946                 wks->fm = fm;
5947                 if (!fm->def_policy) {
5948                         wks->policy = mlx5_flow_meter_policy_find(dev,
5949                                                                   fm->policy_id,
5950                                                                   NULL);
5951                         MLX5_ASSERT(wks->policy);
5952                         if (wks->policy->is_hierarchy) {
5953                                 wks->final_policy =
5954                                 mlx5_flow_meter_hierarchy_get_final_policy(dev,
5955                                                                 wks->policy);
5956                                 if (!wks->final_policy)
5957                                         return rte_flow_error_set(error,
5958                                         EINVAL,
5959                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5960                                 "Failed to find terminal policy of hierarchy.");
5961                                 is_mtr_hierarchy = true;
5962                         }
5963                 }
5964                 /*
5965                  * If it isn't default-policy Meter, and
5966                  * 1. There's no action in flow to change
5967                  *    packet (modify/encap/decap etc.), OR
5968                  * 2. No drop count needed for this meter.
5969                  * 3. It's not meter hierarchy.
5970                  * Then no need to use regC to save meter id anymore.
5971                  */
5972                 if (!fm->def_policy && !is_mtr_hierarchy &&
5973                     (!has_modify || !fm->drop_cnt))
5974                         set_mtr_reg = false;
5975                 /* Prefix actions: meter, decap, encap, tag, jump, end. */
5976                 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) +
5977                            sizeof(struct mlx5_rte_flow_action_set_tag);
5978                 /* Suffix items: tag, vlan, port id, end. */
5979 #define METER_SUFFIX_ITEM 4
5980                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
5981                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5982                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
5983                                           0, SOCKET_ID_ANY);
5984                 if (!sfx_actions)
5985                         return rte_flow_error_set(error, ENOMEM,
5986                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5987                                                   NULL, "no memory to split "
5988                                                   "meter flow");
5989                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
5990                              act_size);
5991                 /* There's no suffix flow for meter of non-default policy. */
5992                 if (!fm->def_policy)
5993                         pre_actions = sfx_actions + 1;
5994                 else
5995                         pre_actions = sfx_actions + actions_n;
5996                 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr,
5997                                             items, sfx_items, actions,
5998                                             sfx_actions, pre_actions,
5999                                             (set_mtr_reg ? &mtr_flow_id : NULL),
6000                                             error);
6001                 if (ret) {
6002                         ret = -rte_errno;
6003                         goto exit;
6004                 }
6005                 /* Add the prefix subflow. */
6006                 flow_split_info->prefix_mark = 0;
6007                 skip_scale_restore = flow_split_info->skip_scale;
6008                 flow_split_info->skip_scale |=
6009                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
6010                 ret = flow_create_split_inner(dev, flow, &dev_flow,
6011                                               attr, items, pre_actions,
6012                                               flow_split_info, error);
6013                 flow_split_info->skip_scale = skip_scale_restore;
6014                 if (ret) {
6015                         if (mtr_flow_id)
6016                                 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id);
6017                         ret = -rte_errno;
6018                         goto exit;
6019                 }
6020                 if (mtr_flow_id) {
6021                         dev_flow->handle->split_flow_id = mtr_flow_id;
6022                         dev_flow->handle->is_meter_flow_id = 1;
6023                 }
6024                 if (!fm->def_policy) {
6025                         if (!set_mtr_reg && fm->drop_cnt)
6026                                 ret =
6027                         flow_meter_create_drop_flow_with_org_pattern(dev, flow,
6028                                                         &sfx_attr, items,
6029                                                         flow_split_info,
6030                                                         fm, error);
6031                         goto exit;
6032                 }
6033                 /* Setting the sfx group atrr. */
6034                 sfx_attr.group = sfx_attr.transfer ?
6035                                 (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
6036                                  MLX5_FLOW_TABLE_LEVEL_METER;
6037                 flow_split_info->prefix_layers =
6038                                 flow_get_prefix_layer_flags(dev_flow);
6039                 flow_split_info->prefix_mark = dev_flow->handle->mark;
6040                 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX;
6041         }
6042         /* Add the prefix subflow. */
6043         ret = flow_create_split_metadata(dev, flow,
6044                                          &sfx_attr, sfx_items ?
6045                                          sfx_items : items,
6046                                          sfx_actions ? sfx_actions : actions,
6047                                          flow_split_info, error);
6048 exit:
6049         if (sfx_actions)
6050                 mlx5_free(sfx_actions);
6051         return ret;
6052 }
6053
6054 /**
6055  * The splitting for sample feature.
6056  *
6057  * Once Sample action is detected in the action list, the flow actions should
6058  * be split into prefix sub flow and suffix sub flow.
6059  *
6060  * The original items remain in the prefix sub flow, all actions preceding the
6061  * sample action and the sample action itself will be copied to the prefix
6062  * sub flow, the actions following the sample action will be copied to the
6063  * suffix sub flow, Queue action always be located in the suffix sub flow.
6064  *
6065  * In order to make the packet from prefix sub flow matches with suffix sub
6066  * flow, an extra tag action be added into prefix sub flow, and the suffix sub
6067  * flow uses tag item with the unique flow id.
6068  *
6069  * @param dev
6070  *   Pointer to Ethernet device.
6071  * @param[in] flow
6072  *   Parent flow structure pointer.
6073  * @param[in] attr
6074  *   Flow rule attributes.
6075  * @param[in] items
6076  *   Pattern specification (list terminated by the END pattern item).
6077  * @param[in] actions
6078  *   Associated actions (list terminated by the END action).
6079  * @param[in] flow_split_info
6080  *   Pointer to flow split info structure.
6081  * @param[out] error
6082  *   Perform verbose error reporting if not NULL.
6083  * @return
6084  *   0 on success, negative value otherwise
6085  */
6086 static int
6087 flow_create_split_sample(struct rte_eth_dev *dev,
6088                          struct rte_flow *flow,
6089                          const struct rte_flow_attr *attr,
6090                          const struct rte_flow_item items[],
6091                          const struct rte_flow_action actions[],
6092                          struct mlx5_flow_split_info *flow_split_info,
6093                          struct rte_flow_error *error)
6094 {
6095         struct mlx5_priv *priv = dev->data->dev_private;
6096         struct rte_flow_action *sfx_actions = NULL;
6097         struct rte_flow_action *pre_actions = NULL;
6098         struct rte_flow_item *sfx_items = NULL;
6099         struct mlx5_flow *dev_flow = NULL;
6100         struct rte_flow_attr sfx_attr = *attr;
6101 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
6102         struct mlx5_flow_dv_sample_resource *sample_res;
6103         struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
6104         struct mlx5_flow_tbl_resource *sfx_tbl;
6105 #endif
6106         size_t act_size;
6107         size_t item_size;
6108         uint32_t fdb_tx = 0;
6109         int32_t tag_id = 0;
6110         int actions_n = 0;
6111         int sample_action_pos;
6112         int qrss_action_pos;
6113         int add_tag = 0;
6114         int modify_after_mirror = 0;
6115         uint16_t jump_table = 0;
6116         const uint32_t next_ft_step = 1;
6117         int ret = 0;
6118
6119         if (priv->sampler_en)
6120                 actions_n = flow_check_match_action(actions, attr,
6121                                         RTE_FLOW_ACTION_TYPE_SAMPLE,
6122                                         &sample_action_pos, &qrss_action_pos,
6123                                         &modify_after_mirror);
6124         if (actions_n) {
6125                 /* The prefix actions must includes sample, tag, end. */
6126                 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
6127                            + sizeof(struct mlx5_rte_flow_action_set_tag);
6128                 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
6129                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
6130                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
6131                                           item_size), 0, SOCKET_ID_ANY);
6132                 if (!sfx_actions)
6133                         return rte_flow_error_set(error, ENOMEM,
6134                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6135                                                   NULL, "no memory to split "
6136                                                   "sample flow");
6137                 /* The representor_id is UINT16_MAX for uplink. */
6138                 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX);
6139                 /*
6140                  * When reg_c_preserve is set, metadata registers Cx preserve
6141                  * their value even through packet duplication.
6142                  */
6143                 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve);
6144                 if (add_tag)
6145                         sfx_items = (struct rte_flow_item *)((char *)sfx_actions
6146                                         + act_size);
6147                 if (modify_after_mirror)
6148                         jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR +
6149                                      next_ft_step;
6150                 pre_actions = sfx_actions + actions_n;
6151                 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items,
6152                                                 actions, sfx_actions,
6153                                                 pre_actions, actions_n,
6154                                                 sample_action_pos,
6155                                                 qrss_action_pos, jump_table,
6156                                                 error);
6157                 if (tag_id < 0 || (add_tag && !tag_id)) {
6158                         ret = -rte_errno;
6159                         goto exit;
6160                 }
6161                 if (modify_after_mirror)
6162                         flow_split_info->skip_scale =
6163                                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
6164                 /* Add the prefix subflow. */
6165                 ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
6166                                               items, pre_actions,
6167                                               flow_split_info, error);
6168                 if (ret) {
6169                         ret = -rte_errno;
6170                         goto exit;
6171                 }
6172                 dev_flow->handle->split_flow_id = tag_id;
6173 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
6174                 if (!modify_after_mirror) {
6175                         /* Set the sfx group attr. */
6176                         sample_res = (struct mlx5_flow_dv_sample_resource *)
6177                                                 dev_flow->dv.sample_res;
6178                         sfx_tbl = (struct mlx5_flow_tbl_resource *)
6179                                                 sample_res->normal_path_tbl;
6180                         sfx_tbl_data = container_of(sfx_tbl,
6181                                                 struct mlx5_flow_tbl_data_entry,
6182                                                 tbl);
6183                         sfx_attr.group = sfx_attr.transfer ?
6184                         (sfx_tbl_data->level - 1) : sfx_tbl_data->level;
6185                 } else {
6186                         MLX5_ASSERT(attr->transfer);
6187                         sfx_attr.group = jump_table;
6188                 }
6189                 flow_split_info->prefix_layers =
6190                                 flow_get_prefix_layer_flags(dev_flow);
6191                 flow_split_info->prefix_mark = dev_flow->handle->mark;
6192                 /* Suffix group level already be scaled with factor, set
6193                  * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale
6194                  * again in translation.
6195                  */
6196                 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT;
6197 #endif
6198         }
6199         /* Add the suffix subflow. */
6200         ret = flow_create_split_meter(dev, flow, &sfx_attr,
6201                                       sfx_items ? sfx_items : items,
6202                                       sfx_actions ? sfx_actions : actions,
6203                                       flow_split_info, error);
6204 exit:
6205         if (sfx_actions)
6206                 mlx5_free(sfx_actions);
6207         return ret;
6208 }
6209
6210 /**
6211  * Split the flow to subflow set. The splitters might be linked
6212  * in the chain, like this:
6213  * flow_create_split_outer() calls:
6214  *   flow_create_split_meter() calls:
6215  *     flow_create_split_metadata(meter_subflow_0) calls:
6216  *       flow_create_split_inner(metadata_subflow_0)
6217  *       flow_create_split_inner(metadata_subflow_1)
6218  *       flow_create_split_inner(metadata_subflow_2)
6219  *     flow_create_split_metadata(meter_subflow_1) calls:
6220  *       flow_create_split_inner(metadata_subflow_0)
6221  *       flow_create_split_inner(metadata_subflow_1)
6222  *       flow_create_split_inner(metadata_subflow_2)
6223  *
6224  * This provide flexible way to add new levels of flow splitting.
6225  * The all of successfully created subflows are included to the
6226  * parent flow dev_flow list.
6227  *
6228  * @param dev
6229  *   Pointer to Ethernet device.
6230  * @param[in] flow
6231  *   Parent flow structure pointer.
6232  * @param[in] attr
6233  *   Flow rule attributes.
6234  * @param[in] items
6235  *   Pattern specification (list terminated by the END pattern item).
6236  * @param[in] actions
6237  *   Associated actions (list terminated by the END action).
6238  * @param[in] flow_split_info
6239  *   Pointer to flow split info structure.
6240  * @param[out] error
6241  *   Perform verbose error reporting if not NULL.
6242  * @return
6243  *   0 on success, negative value otherwise
6244  */
6245 static int
6246 flow_create_split_outer(struct rte_eth_dev *dev,
6247                         struct rte_flow *flow,
6248                         const struct rte_flow_attr *attr,
6249                         const struct rte_flow_item items[],
6250                         const struct rte_flow_action actions[],
6251                         struct mlx5_flow_split_info *flow_split_info,
6252                         struct rte_flow_error *error)
6253 {
6254         int ret;
6255
6256         ret = flow_create_split_sample(dev, flow, attr, items,
6257                                        actions, flow_split_info, error);
6258         MLX5_ASSERT(ret <= 0);
6259         return ret;
6260 }
6261
6262 static inline struct mlx5_flow_tunnel *
6263 flow_tunnel_from_rule(const struct mlx5_flow *flow)
6264 {
6265         struct mlx5_flow_tunnel *tunnel;
6266
6267 #pragma GCC diagnostic push
6268 #pragma GCC diagnostic ignored "-Wcast-qual"
6269         tunnel = (typeof(tunnel))flow->tunnel;
6270 #pragma GCC diagnostic pop
6271
6272         return tunnel;
6273 }
6274
6275 /**
6276  * Adjust flow RSS workspace if needed.
6277  *
6278  * @param wks
6279  *   Pointer to thread flow work space.
6280  * @param rss_desc
6281  *   Pointer to RSS descriptor.
6282  * @param[in] nrssq_num
6283  *   New RSS queue number.
6284  *
6285  * @return
6286  *   0 on success, -1 otherwise and rte_errno is set.
6287  */
6288 static int
6289 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
6290                           struct mlx5_flow_rss_desc *rss_desc,
6291                           uint32_t nrssq_num)
6292 {
6293         if (likely(nrssq_num <= wks->rssq_num))
6294                 return 0;
6295         rss_desc->queue = realloc(rss_desc->queue,
6296                           sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2));
6297         if (!rss_desc->queue) {
6298                 rte_errno = ENOMEM;
6299                 return -1;
6300         }
6301         wks->rssq_num = RTE_ALIGN(nrssq_num, 2);
6302         return 0;
6303 }
6304
6305 /**
6306  * Create a flow and add it to @p list.
6307  *
6308  * @param dev
6309  *   Pointer to Ethernet device.
6310  * @param list
6311  *   Pointer to a TAILQ flow list. If this parameter NULL,
6312  *   no list insertion occurred, flow is just created,
6313  *   this is caller's responsibility to track the
6314  *   created flow.
6315  * @param[in] attr
6316  *   Flow rule attributes.
6317  * @param[in] items
6318  *   Pattern specification (list terminated by the END pattern item).
6319  * @param[in] actions
6320  *   Associated actions (list terminated by the END action).
6321  * @param[in] external
6322  *   This flow rule is created by request external to PMD.
6323  * @param[out] error
6324  *   Perform verbose error reporting if not NULL.
6325  *
6326  * @return
6327  *   A flow index on success, 0 otherwise and rte_errno is set.
6328  */
6329 static uint32_t
6330 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
6331                  const struct rte_flow_attr *attr,
6332                  const struct rte_flow_item items[],
6333                  const struct rte_flow_action original_actions[],
6334                  bool external, struct rte_flow_error *error)
6335 {
6336         struct mlx5_priv *priv = dev->data->dev_private;
6337         struct rte_flow *flow = NULL;
6338         struct mlx5_flow *dev_flow;
6339         const struct rte_flow_action_rss *rss = NULL;
6340         struct mlx5_translated_action_handle
6341                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
6342         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
6343         union {
6344                 struct mlx5_flow_expand_rss buf;
6345                 uint8_t buffer[4096];
6346         } expand_buffer;
6347         union {
6348                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6349                 uint8_t buffer[2048];
6350         } actions_rx;
6351         union {
6352                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6353                 uint8_t buffer[2048];
6354         } actions_hairpin_tx;
6355         union {
6356                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
6357                 uint8_t buffer[2048];
6358         } items_tx;
6359         struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
6360         struct mlx5_flow_rss_desc *rss_desc;
6361         const struct rte_flow_action *p_actions_rx;
6362         uint32_t i;
6363         uint32_t idx = 0;
6364         int hairpin_flow;
6365         struct rte_flow_attr attr_tx = { .priority = 0 };
6366         const struct rte_flow_action *actions;
6367         struct rte_flow_action *translated_actions = NULL;
6368         struct mlx5_flow_tunnel *tunnel;
6369         struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
6370         struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace();
6371         struct mlx5_flow_split_info flow_split_info = {
6372                 .external = !!external,
6373                 .skip_scale = 0,
6374                 .flow_idx = 0,
6375                 .prefix_mark = 0,
6376                 .prefix_layers = 0,
6377                 .table_id = 0
6378         };
6379         int ret;
6380
6381         MLX5_ASSERT(wks);
6382         rss_desc = &wks->rss_desc;
6383         ret = flow_action_handles_translate(dev, original_actions,
6384                                             indir_actions,
6385                                             &indir_actions_n,
6386                                             &translated_actions, error);
6387         if (ret < 0) {
6388                 MLX5_ASSERT(translated_actions == NULL);
6389                 return 0;
6390         }
6391         actions = translated_actions ? translated_actions : original_actions;
6392         p_actions_rx = actions;
6393         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
6394         ret = flow_drv_validate(dev, attr, items, p_actions_rx,
6395                                 external, hairpin_flow, error);
6396         if (ret < 0)
6397                 goto error_before_hairpin_split;
6398         flow = mlx5_ipool_zmalloc(priv->flows[type], &idx);
6399         if (!flow) {
6400                 rte_errno = ENOMEM;
6401                 goto error_before_hairpin_split;
6402         }
6403         if (hairpin_flow > 0) {
6404                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
6405                         rte_errno = EINVAL;
6406                         goto error_before_hairpin_split;
6407                 }
6408                 flow_hairpin_split(dev, actions, actions_rx.actions,
6409                                    actions_hairpin_tx.actions, items_tx.items,
6410                                    idx);
6411                 p_actions_rx = actions_rx.actions;
6412         }
6413         flow_split_info.flow_idx = idx;
6414         flow->drv_type = flow_get_drv_type(dev, attr);
6415         MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
6416                     flow->drv_type < MLX5_FLOW_TYPE_MAX);
6417         memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
6418         /* RSS Action only works on NIC RX domain */
6419         if (attr->ingress && !attr->transfer)
6420                 rss = flow_get_rss_action(dev, p_actions_rx);
6421         if (rss) {
6422                 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num))
6423                         return 0;
6424                 /*
6425                  * The following information is required by
6426                  * mlx5_flow_hashfields_adjust() in advance.
6427                  */
6428                 rss_desc->level = rss->level;
6429                 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */
6430                 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types;
6431         }
6432         flow->dev_handles = 0;
6433         if (rss && rss->types) {
6434                 unsigned int graph_root;
6435
6436                 graph_root = find_graph_root(rss->level);
6437                 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
6438                                            items, rss->types,
6439                                            mlx5_support_expansion, graph_root);
6440                 MLX5_ASSERT(ret > 0 &&
6441                        (unsigned int)ret < sizeof(expand_buffer.buffer));
6442                 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) {
6443                         for (i = 0; i < buf->entries; ++i)
6444                                 mlx5_dbg__print_pattern(buf->entry[i].pattern);
6445                 }
6446         } else {
6447                 buf->entries = 1;
6448                 buf->entry[0].pattern = (void *)(uintptr_t)items;
6449         }
6450         rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions,
6451                                                       indir_actions_n);
6452         for (i = 0; i < buf->entries; ++i) {
6453                 /* Initialize flow split data. */
6454                 flow_split_info.prefix_layers = 0;
6455                 flow_split_info.prefix_mark = 0;
6456                 flow_split_info.skip_scale = 0;
6457                 /*
6458                  * The splitter may create multiple dev_flows,
6459                  * depending on configuration. In the simplest
6460                  * case it just creates unmodified original flow.
6461                  */
6462                 ret = flow_create_split_outer(dev, flow, attr,
6463                                               buf->entry[i].pattern,
6464                                               p_actions_rx, &flow_split_info,
6465                                               error);
6466                 if (ret < 0)
6467                         goto error;
6468                 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) {
6469                         ret = flow_tunnel_add_default_miss(dev, flow, attr,
6470                                                            p_actions_rx,
6471                                                            idx,
6472                                                            wks->flows[0].tunnel,
6473                                                            &default_miss_ctx,
6474                                                            error);
6475                         if (ret < 0) {
6476                                 mlx5_free(default_miss_ctx.queue);
6477                                 goto error;
6478                         }
6479                 }
6480         }
6481         /* Create the tx flow. */
6482         if (hairpin_flow) {
6483                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
6484                 attr_tx.ingress = 0;
6485                 attr_tx.egress = 1;
6486                 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
6487                                          actions_hairpin_tx.actions,
6488                                          idx, error);
6489                 if (!dev_flow)
6490                         goto error;
6491                 dev_flow->flow = flow;
6492                 dev_flow->external = 0;
6493                 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
6494                               dev_flow->handle, next);
6495                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
6496                                          items_tx.items,
6497                                          actions_hairpin_tx.actions, error);
6498                 if (ret < 0)
6499                         goto error;
6500         }
6501         /*
6502          * Update the metadata register copy table. If extensive
6503          * metadata feature is enabled and registers are supported
6504          * we might create the extra rte_flow for each unique
6505          * MARK/FLAG action ID.
6506          *
6507          * The table is updated for ingress Flows only, because
6508          * the egress Flows belong to the different device and
6509          * copy table should be updated in peer NIC Rx domain.
6510          */
6511         if (attr->ingress &&
6512             (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
6513                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
6514                 if (ret)
6515                         goto error;
6516         }
6517         /*
6518          * If the flow is external (from application) OR device is started,
6519          * OR mreg discover, then apply immediately.
6520          */
6521         if (external || dev->data->dev_started ||
6522             (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP &&
6523              attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) {
6524                 ret = flow_drv_apply(dev, flow, error);
6525                 if (ret < 0)
6526                         goto error;
6527         }
6528         flow->type = type;
6529         flow_rxq_flags_set(dev, flow);
6530         rte_free(translated_actions);
6531         tunnel = flow_tunnel_from_rule(wks->flows);
6532         if (tunnel) {
6533                 flow->tunnel = 1;
6534                 flow->tunnel_id = tunnel->tunnel_id;
6535                 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED);
6536                 mlx5_free(default_miss_ctx.queue);
6537         }
6538         mlx5_flow_pop_thread_workspace();
6539         return idx;
6540 error:
6541         MLX5_ASSERT(flow);
6542         ret = rte_errno; /* Save rte_errno before cleanup. */
6543         flow_mreg_del_copy_action(dev, flow);
6544         flow_drv_destroy(dev, flow);
6545         if (rss_desc->shared_rss)
6546                 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *)
6547                         mlx5_ipool_get
6548                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
6549                         rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
6550         mlx5_ipool_free(priv->flows[type], idx);
6551         rte_errno = ret; /* Restore rte_errno. */
6552         ret = rte_errno;
6553         rte_errno = ret;
6554         mlx5_flow_pop_thread_workspace();
6555 error_before_hairpin_split:
6556         rte_free(translated_actions);
6557         return 0;
6558 }
6559
6560 /**
6561  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
6562  * incoming packets to table 1.
6563  *
6564  * Other flow rules, requested for group n, will be created in
6565  * e-switch table n+1.
6566  * Jump action to e-switch group n will be created to group n+1.
6567  *
6568  * Used when working in switchdev mode, to utilise advantages of table 1
6569  * and above.
6570  *
6571  * @param dev
6572  *   Pointer to Ethernet device.
6573  *
6574  * @return
6575  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
6576  */
6577 struct rte_flow *
6578 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
6579 {
6580         const struct rte_flow_attr attr = {
6581                 .group = 0,
6582                 .priority = 0,
6583                 .ingress = 1,
6584                 .egress = 0,
6585                 .transfer = 1,
6586         };
6587         const struct rte_flow_item pattern = {
6588                 .type = RTE_FLOW_ITEM_TYPE_END,
6589         };
6590         struct rte_flow_action_jump jump = {
6591                 .group = 1,
6592         };
6593         const struct rte_flow_action actions[] = {
6594                 {
6595                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
6596                         .conf = &jump,
6597                 },
6598                 {
6599                         .type = RTE_FLOW_ACTION_TYPE_END,
6600                 },
6601         };
6602         struct rte_flow_error error;
6603
6604         return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
6605                                                    &attr, &pattern,
6606                                                    actions, false, &error);
6607 }
6608
6609 /**
6610  * Create a dedicated flow rule on e-switch table 1, matches ESW manager
6611  * and sq number, directs all packets to peer vport.
6612  *
6613  * @param dev
6614  *   Pointer to Ethernet device.
6615  * @param txq
6616  *   Txq index.
6617  *
6618  * @return
6619  *   Flow ID on success, 0 otherwise and rte_errno is set.
6620  */
6621 uint32_t
6622 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq)
6623 {
6624         struct rte_flow_attr attr = {
6625                 .group = 0,
6626                 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
6627                 .ingress = 1,
6628                 .egress = 0,
6629                 .transfer = 1,
6630         };
6631         struct rte_flow_item_port_id port_spec = {
6632                 .id = MLX5_PORT_ESW_MGR,
6633         };
6634         struct mlx5_rte_flow_item_tx_queue txq_spec = {
6635                 .queue = txq,
6636         };
6637         struct rte_flow_item pattern[] = {
6638                 {
6639                         .type = RTE_FLOW_ITEM_TYPE_PORT_ID,
6640                         .spec = &port_spec,
6641                 },
6642                 {
6643                         .type = (enum rte_flow_item_type)
6644                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
6645                         .spec = &txq_spec,
6646                 },
6647                 {
6648                         .type = RTE_FLOW_ITEM_TYPE_END,
6649                 },
6650         };
6651         struct rte_flow_action_jump jump = {
6652                 .group = 1,
6653         };
6654         struct rte_flow_action_port_id port = {
6655                 .id = dev->data->port_id,
6656         };
6657         struct rte_flow_action actions[] = {
6658                 {
6659                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
6660                         .conf = &jump,
6661                 },
6662                 {
6663                         .type = RTE_FLOW_ACTION_TYPE_END,
6664                 },
6665         };
6666         struct rte_flow_error error;
6667
6668         /*
6669          * Creates group 0, highest priority jump flow.
6670          * Matches txq to bypass kernel packets.
6671          */
6672         if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions,
6673                              false, &error) == 0)
6674                 return 0;
6675         /* Create group 1, lowest priority redirect flow for txq. */
6676         attr.group = 1;
6677         actions[0].conf = &port;
6678         actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID;
6679         return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern,
6680                                 actions, false, &error);
6681 }
6682
6683 /**
6684  * Validate a flow supported by the NIC.
6685  *
6686  * @see rte_flow_validate()
6687  * @see rte_flow_ops
6688  */
6689 int
6690 mlx5_flow_validate(struct rte_eth_dev *dev,
6691                    const struct rte_flow_attr *attr,
6692                    const struct rte_flow_item items[],
6693                    const struct rte_flow_action original_actions[],
6694                    struct rte_flow_error *error)
6695 {
6696         int hairpin_flow;
6697         struct mlx5_translated_action_handle
6698                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
6699         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
6700         const struct rte_flow_action *actions;
6701         struct rte_flow_action *translated_actions = NULL;
6702         int ret = flow_action_handles_translate(dev, original_actions,
6703                                                 indir_actions,
6704                                                 &indir_actions_n,
6705                                                 &translated_actions, error);
6706
6707         if (ret)
6708                 return ret;
6709         actions = translated_actions ? translated_actions : original_actions;
6710         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
6711         ret = flow_drv_validate(dev, attr, items, actions,
6712                                 true, hairpin_flow, error);
6713         rte_free(translated_actions);
6714         return ret;
6715 }
6716
6717 /**
6718  * Create a flow.
6719  *
6720  * @see rte_flow_create()
6721  * @see rte_flow_ops
6722  */
6723 struct rte_flow *
6724 mlx5_flow_create(struct rte_eth_dev *dev,
6725                  const struct rte_flow_attr *attr,
6726                  const struct rte_flow_item items[],
6727                  const struct rte_flow_action actions[],
6728                  struct rte_flow_error *error)
6729 {
6730         /*
6731          * If the device is not started yet, it is not allowed to created a
6732          * flow from application. PMD default flows and traffic control flows
6733          * are not affected.
6734          */
6735         if (unlikely(!dev->data->dev_started)) {
6736                 DRV_LOG(DEBUG, "port %u is not started when "
6737                         "inserting a flow", dev->data->port_id);
6738                 rte_flow_error_set(error, ENODEV,
6739                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6740                                    NULL,
6741                                    "port not started");
6742                 return NULL;
6743         }
6744
6745         return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN,
6746                                                    attr, items, actions,
6747                                                    true, error);
6748 }
6749
6750 /**
6751  * Destroy a flow in a list.
6752  *
6753  * @param dev
6754  *   Pointer to Ethernet device.
6755  * @param[in] flow_idx
6756  *   Index of flow to destroy.
6757  */
6758 static void
6759 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
6760                   uint32_t flow_idx)
6761 {
6762         struct mlx5_priv *priv = dev->data->dev_private;
6763         struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx);
6764
6765         if (!flow)
6766                 return;
6767         MLX5_ASSERT(flow->type == type);
6768         /*
6769          * Update RX queue flags only if port is started, otherwise it is
6770          * already clean.
6771          */
6772         if (dev->data->dev_started)
6773                 flow_rxq_flags_trim(dev, flow);
6774         flow_drv_destroy(dev, flow);
6775         if (flow->tunnel) {
6776                 struct mlx5_flow_tunnel *tunnel;
6777
6778                 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
6779                 RTE_VERIFY(tunnel);
6780                 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
6781                         mlx5_flow_tunnel_free(dev, tunnel);
6782         }
6783         flow_mreg_del_copy_action(dev, flow);
6784         mlx5_ipool_free(priv->flows[type], flow_idx);
6785 }
6786
6787 /**
6788  * Destroy all flows.
6789  *
6790  * @param dev
6791  *   Pointer to Ethernet device.
6792  * @param type
6793  *   Flow type to be flushed.
6794  * @param active
6795  *   If flushing is called avtively.
6796  */
6797 void
6798 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
6799                      bool active)
6800 {
6801         struct mlx5_priv *priv = dev->data->dev_private;
6802         uint32_t num_flushed = 0, fidx = 1;
6803         struct rte_flow *flow;
6804
6805         MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) {
6806                 flow_list_destroy(dev, type, fidx);
6807                 num_flushed++;
6808         }
6809         if (active) {
6810                 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
6811                         dev->data->port_id, num_flushed);
6812         }
6813 }
6814
6815 /**
6816  * Stop all default actions for flows.
6817  *
6818  * @param dev
6819  *   Pointer to Ethernet device.
6820  */
6821 void
6822 mlx5_flow_stop_default(struct rte_eth_dev *dev)
6823 {
6824         flow_mreg_del_default_copy_action(dev);
6825         flow_rxq_flags_clear(dev);
6826 }
6827
6828 /**
6829  * Start all default actions for flows.
6830  *
6831  * @param dev
6832  *   Pointer to Ethernet device.
6833  * @return
6834  *   0 on success, a negative errno value otherwise and rte_errno is set.
6835  */
6836 int
6837 mlx5_flow_start_default(struct rte_eth_dev *dev)
6838 {
6839         struct rte_flow_error error;
6840
6841         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
6842         return flow_mreg_add_default_copy_action(dev, &error);
6843 }
6844
6845 /**
6846  * Release key of thread specific flow workspace data.
6847  */
6848 void
6849 flow_release_workspace(void *data)
6850 {
6851         struct mlx5_flow_workspace *wks = data;
6852         struct mlx5_flow_workspace *next;
6853
6854         while (wks) {
6855                 next = wks->next;
6856                 free(wks->rss_desc.queue);
6857                 free(wks);
6858                 wks = next;
6859         }
6860 }
6861
6862 /**
6863  * Get thread specific current flow workspace.
6864  *
6865  * @return pointer to thread specific flow workspace data, NULL on error.
6866  */
6867 struct mlx5_flow_workspace*
6868 mlx5_flow_get_thread_workspace(void)
6869 {
6870         struct mlx5_flow_workspace *data;
6871
6872         data = mlx5_flow_os_get_specific_workspace();
6873         MLX5_ASSERT(data && data->inuse);
6874         if (!data || !data->inuse)
6875                 DRV_LOG(ERR, "flow workspace not initialized.");
6876         return data;
6877 }
6878
6879 /**
6880  * Allocate and init new flow workspace.
6881  *
6882  * @return pointer to flow workspace data, NULL on error.
6883  */
6884 static struct mlx5_flow_workspace*
6885 flow_alloc_thread_workspace(void)
6886 {
6887         struct mlx5_flow_workspace *data = calloc(1, sizeof(*data));
6888
6889         if (!data) {
6890                 DRV_LOG(ERR, "Failed to allocate flow workspace "
6891                         "memory.");
6892                 return NULL;
6893         }
6894         data->rss_desc.queue = calloc(1,
6895                         sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM);
6896         if (!data->rss_desc.queue)
6897                 goto err;
6898         data->rssq_num = MLX5_RSSQ_DEFAULT_NUM;
6899         return data;
6900 err:
6901         if (data->rss_desc.queue)
6902                 free(data->rss_desc.queue);
6903         free(data);
6904         return NULL;
6905 }
6906
6907 /**
6908  * Get new thread specific flow workspace.
6909  *
6910  * If current workspace inuse, create new one and set as current.
6911  *
6912  * @return pointer to thread specific flow workspace data, NULL on error.
6913  */
6914 static struct mlx5_flow_workspace*
6915 mlx5_flow_push_thread_workspace(void)
6916 {
6917         struct mlx5_flow_workspace *curr;
6918         struct mlx5_flow_workspace *data;
6919
6920         curr = mlx5_flow_os_get_specific_workspace();
6921         if (!curr) {
6922                 data = flow_alloc_thread_workspace();
6923                 if (!data)
6924                         return NULL;
6925         } else if (!curr->inuse) {
6926                 data = curr;
6927         } else if (curr->next) {
6928                 data = curr->next;
6929         } else {
6930                 data = flow_alloc_thread_workspace();
6931                 if (!data)
6932                         return NULL;
6933                 curr->next = data;
6934                 data->prev = curr;
6935         }
6936         data->inuse = 1;
6937         data->flow_idx = 0;
6938         /* Set as current workspace */
6939         if (mlx5_flow_os_set_specific_workspace(data))
6940                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
6941         return data;
6942 }
6943
6944 /**
6945  * Close current thread specific flow workspace.
6946  *
6947  * If previous workspace available, set it as current.
6948  *
6949  * @return pointer to thread specific flow workspace data, NULL on error.
6950  */
6951 static void
6952 mlx5_flow_pop_thread_workspace(void)
6953 {
6954         struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace();
6955
6956         if (!data)
6957                 return;
6958         if (!data->inuse) {
6959                 DRV_LOG(ERR, "Failed to close unused flow workspace.");
6960                 return;
6961         }
6962         data->inuse = 0;
6963         if (!data->prev)
6964                 return;
6965         if (mlx5_flow_os_set_specific_workspace(data->prev))
6966                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
6967 }
6968
6969 /**
6970  * Verify the flow list is empty
6971  *
6972  * @param dev
6973  *  Pointer to Ethernet device.
6974  *
6975  * @return the number of flows not released.
6976  */
6977 int
6978 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused)
6979 {
6980         struct mlx5_priv *priv = dev->data->dev_private;
6981         struct rte_flow *flow;
6982         uint32_t idx = 0;
6983         int ret = 0, i;
6984
6985         for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
6986                 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) {
6987                         DRV_LOG(DEBUG, "port %u flow %p still referenced",
6988                                 dev->data->port_id, (void *)flow);
6989                         ret++;
6990                 }
6991         }
6992         return ret;
6993 }
6994
6995 /**
6996  * Enable default hairpin egress flow.
6997  *
6998  * @param dev
6999  *   Pointer to Ethernet device.
7000  * @param queue
7001  *   The queue index.
7002  *
7003  * @return
7004  *   0 on success, a negative errno value otherwise and rte_errno is set.
7005  */
7006 int
7007 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
7008                             uint32_t queue)
7009 {
7010         const struct rte_flow_attr attr = {
7011                 .egress = 1,
7012                 .priority = 0,
7013         };
7014         struct mlx5_rte_flow_item_tx_queue queue_spec = {
7015                 .queue = queue,
7016         };
7017         struct mlx5_rte_flow_item_tx_queue queue_mask = {
7018                 .queue = UINT32_MAX,
7019         };
7020         struct rte_flow_item items[] = {
7021                 {
7022                         .type = (enum rte_flow_item_type)
7023                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
7024                         .spec = &queue_spec,
7025                         .last = NULL,
7026                         .mask = &queue_mask,
7027                 },
7028                 {
7029                         .type = RTE_FLOW_ITEM_TYPE_END,
7030                 },
7031         };
7032         struct rte_flow_action_jump jump = {
7033                 .group = MLX5_HAIRPIN_TX_TABLE,
7034         };
7035         struct rte_flow_action actions[2];
7036         uint32_t flow_idx;
7037         struct rte_flow_error error;
7038
7039         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
7040         actions[0].conf = &jump;
7041         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
7042         flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7043                                     &attr, items, actions, false, &error);
7044         if (!flow_idx) {
7045                 DRV_LOG(DEBUG,
7046                         "Failed to create ctrl flow: rte_errno(%d),"
7047                         " type(%d), message(%s)",
7048                         rte_errno, error.type,
7049                         error.message ? error.message : " (no stated reason)");
7050                 return -rte_errno;
7051         }
7052         return 0;
7053 }
7054
7055 /**
7056  * Enable a control flow configured from the control plane.
7057  *
7058  * @param dev
7059  *   Pointer to Ethernet device.
7060  * @param eth_spec
7061  *   An Ethernet flow spec to apply.
7062  * @param eth_mask
7063  *   An Ethernet flow mask to apply.
7064  * @param vlan_spec
7065  *   A VLAN flow spec to apply.
7066  * @param vlan_mask
7067  *   A VLAN flow mask to apply.
7068  *
7069  * @return
7070  *   0 on success, a negative errno value otherwise and rte_errno is set.
7071  */
7072 int
7073 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
7074                     struct rte_flow_item_eth *eth_spec,
7075                     struct rte_flow_item_eth *eth_mask,
7076                     struct rte_flow_item_vlan *vlan_spec,
7077                     struct rte_flow_item_vlan *vlan_mask)
7078 {
7079         struct mlx5_priv *priv = dev->data->dev_private;
7080         const struct rte_flow_attr attr = {
7081                 .ingress = 1,
7082                 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
7083         };
7084         struct rte_flow_item items[] = {
7085                 {
7086                         .type = RTE_FLOW_ITEM_TYPE_ETH,
7087                         .spec = eth_spec,
7088                         .last = NULL,
7089                         .mask = eth_mask,
7090                 },
7091                 {
7092                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
7093                                               RTE_FLOW_ITEM_TYPE_END,
7094                         .spec = vlan_spec,
7095                         .last = NULL,
7096                         .mask = vlan_mask,
7097                 },
7098                 {
7099                         .type = RTE_FLOW_ITEM_TYPE_END,
7100                 },
7101         };
7102         uint16_t queue[priv->reta_idx_n];
7103         struct rte_flow_action_rss action_rss = {
7104                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
7105                 .level = 0,
7106                 .types = priv->rss_conf.rss_hf,
7107                 .key_len = priv->rss_conf.rss_key_len,
7108                 .queue_num = priv->reta_idx_n,
7109                 .key = priv->rss_conf.rss_key,
7110                 .queue = queue,
7111         };
7112         struct rte_flow_action actions[] = {
7113                 {
7114                         .type = RTE_FLOW_ACTION_TYPE_RSS,
7115                         .conf = &action_rss,
7116                 },
7117                 {
7118                         .type = RTE_FLOW_ACTION_TYPE_END,
7119                 },
7120         };
7121         uint32_t flow_idx;
7122         struct rte_flow_error error;
7123         unsigned int i;
7124
7125         if (!priv->reta_idx_n || !priv->rxqs_n) {
7126                 return 0;
7127         }
7128         if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG))
7129                 action_rss.types = 0;
7130         for (i = 0; i != priv->reta_idx_n; ++i)
7131                 queue[i] = (*priv->reta_idx)[i];
7132         flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7133                                     &attr, items, actions, false, &error);
7134         if (!flow_idx)
7135                 return -rte_errno;
7136         return 0;
7137 }
7138
7139 /**
7140  * Enable a flow control configured from the control plane.
7141  *
7142  * @param dev
7143  *   Pointer to Ethernet device.
7144  * @param eth_spec
7145  *   An Ethernet flow spec to apply.
7146  * @param eth_mask
7147  *   An Ethernet flow mask to apply.
7148  *
7149  * @return
7150  *   0 on success, a negative errno value otherwise and rte_errno is set.
7151  */
7152 int
7153 mlx5_ctrl_flow(struct rte_eth_dev *dev,
7154                struct rte_flow_item_eth *eth_spec,
7155                struct rte_flow_item_eth *eth_mask)
7156 {
7157         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
7158 }
7159
7160 /**
7161  * Create default miss flow rule matching lacp traffic
7162  *
7163  * @param dev
7164  *   Pointer to Ethernet device.
7165  * @param eth_spec
7166  *   An Ethernet flow spec to apply.
7167  *
7168  * @return
7169  *   0 on success, a negative errno value otherwise and rte_errno is set.
7170  */
7171 int
7172 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
7173 {
7174         /*
7175          * The LACP matching is done by only using ether type since using
7176          * a multicast dst mac causes kernel to give low priority to this flow.
7177          */
7178         static const struct rte_flow_item_eth lacp_spec = {
7179                 .type = RTE_BE16(0x8809),
7180         };
7181         static const struct rte_flow_item_eth lacp_mask = {
7182                 .type = 0xffff,
7183         };
7184         const struct rte_flow_attr attr = {
7185                 .ingress = 1,
7186         };
7187         struct rte_flow_item items[] = {
7188                 {
7189                         .type = RTE_FLOW_ITEM_TYPE_ETH,
7190                         .spec = &lacp_spec,
7191                         .mask = &lacp_mask,
7192                 },
7193                 {
7194                         .type = RTE_FLOW_ITEM_TYPE_END,
7195                 },
7196         };
7197         struct rte_flow_action actions[] = {
7198                 {
7199                         .type = (enum rte_flow_action_type)
7200                                 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
7201                 },
7202                 {
7203                         .type = RTE_FLOW_ACTION_TYPE_END,
7204                 },
7205         };
7206         struct rte_flow_error error;
7207         uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7208                                         &attr, items, actions,
7209                                         false, &error);
7210
7211         if (!flow_idx)
7212                 return -rte_errno;
7213         return 0;
7214 }
7215
7216 /**
7217  * Destroy a flow.
7218  *
7219  * @see rte_flow_destroy()
7220  * @see rte_flow_ops
7221  */
7222 int
7223 mlx5_flow_destroy(struct rte_eth_dev *dev,
7224                   struct rte_flow *flow,
7225                   struct rte_flow_error *error __rte_unused)
7226 {
7227         flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
7228                                 (uintptr_t)(void *)flow);
7229         return 0;
7230 }
7231
7232 /**
7233  * Destroy all flows.
7234  *
7235  * @see rte_flow_flush()
7236  * @see rte_flow_ops
7237  */
7238 int
7239 mlx5_flow_flush(struct rte_eth_dev *dev,
7240                 struct rte_flow_error *error __rte_unused)
7241 {
7242         mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false);
7243         return 0;
7244 }
7245
7246 /**
7247  * Isolated mode.
7248  *
7249  * @see rte_flow_isolate()
7250  * @see rte_flow_ops
7251  */
7252 int
7253 mlx5_flow_isolate(struct rte_eth_dev *dev,
7254                   int enable,
7255                   struct rte_flow_error *error)
7256 {
7257         struct mlx5_priv *priv = dev->data->dev_private;
7258
7259         if (dev->data->dev_started) {
7260                 rte_flow_error_set(error, EBUSY,
7261                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7262                                    NULL,
7263                                    "port must be stopped first");
7264                 return -rte_errno;
7265         }
7266         priv->isolated = !!enable;
7267         if (enable)
7268                 dev->dev_ops = &mlx5_dev_ops_isolate;
7269         else
7270                 dev->dev_ops = &mlx5_dev_ops;
7271
7272         dev->rx_descriptor_status = mlx5_rx_descriptor_status;
7273         dev->tx_descriptor_status = mlx5_tx_descriptor_status;
7274
7275         return 0;
7276 }
7277
7278 /**
7279  * Query a flow.
7280  *
7281  * @see rte_flow_query()
7282  * @see rte_flow_ops
7283  */
7284 static int
7285 flow_drv_query(struct rte_eth_dev *dev,
7286                uint32_t flow_idx,
7287                const struct rte_flow_action *actions,
7288                void *data,
7289                struct rte_flow_error *error)
7290 {
7291         struct mlx5_priv *priv = dev->data->dev_private;
7292         const struct mlx5_flow_driver_ops *fops;
7293         struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
7294                                                flow_idx);
7295         enum mlx5_flow_drv_type ftype;
7296
7297         if (!flow) {
7298                 return rte_flow_error_set(error, ENOENT,
7299                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7300                           NULL,
7301                           "invalid flow handle");
7302         }
7303         ftype = flow->drv_type;
7304         MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
7305         fops = flow_get_drv_ops(ftype);
7306
7307         return fops->query(dev, flow, actions, data, error);
7308 }
7309
7310 /**
7311  * Query a flow.
7312  *
7313  * @see rte_flow_query()
7314  * @see rte_flow_ops
7315  */
7316 int
7317 mlx5_flow_query(struct rte_eth_dev *dev,
7318                 struct rte_flow *flow,
7319                 const struct rte_flow_action *actions,
7320                 void *data,
7321                 struct rte_flow_error *error)
7322 {
7323         int ret;
7324
7325         ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
7326                              error);
7327         if (ret < 0)
7328                 return ret;
7329         return 0;
7330 }
7331
7332 /**
7333  * Get rte_flow callbacks.
7334  *
7335  * @param dev
7336  *   Pointer to Ethernet device structure.
7337  * @param ops
7338  *   Pointer to operation-specific structure.
7339  *
7340  * @return 0
7341  */
7342 int
7343 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused,
7344                   const struct rte_flow_ops **ops)
7345 {
7346         *ops = &mlx5_flow_ops;
7347         return 0;
7348 }
7349
7350 /**
7351  * Validate meter policy actions.
7352  * Dispatcher for action type specific validation.
7353  *
7354  * @param[in] dev
7355  *   Pointer to the Ethernet device structure.
7356  * @param[in] action
7357  *   The meter policy action object to validate.
7358  * @param[in] attr
7359  *   Attributes of flow to determine steering domain.
7360  * @param[out] is_rss
7361  *   Is RSS or not.
7362  * @param[out] domain_bitmap
7363  *   Domain bitmap.
7364  * @param[out] is_def_policy
7365  *   Is default policy or not.
7366  * @param[out] error
7367  *   Perform verbose error reporting if not NULL. Initialized in case of
7368  *   error only.
7369  *
7370  * @return
7371  *   0 on success, otherwise negative errno value.
7372  */
7373 int
7374 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev,
7375                         const struct rte_flow_action *actions[RTE_COLORS],
7376                         struct rte_flow_attr *attr,
7377                         bool *is_rss,
7378                         uint8_t *domain_bitmap,
7379                         uint8_t *policy_mode,
7380                         struct rte_mtr_error *error)
7381 {
7382         const struct mlx5_flow_driver_ops *fops;
7383
7384         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7385         return fops->validate_mtr_acts(dev, actions, attr, is_rss,
7386                                        domain_bitmap, policy_mode, error);
7387 }
7388
7389 /**
7390  * Destroy the meter table set.
7391  *
7392  * @param[in] dev
7393  *   Pointer to Ethernet device.
7394  * @param[in] mtr_policy
7395  *   Meter policy struct.
7396  */
7397 void
7398 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev,
7399                       struct mlx5_flow_meter_policy *mtr_policy)
7400 {
7401         const struct mlx5_flow_driver_ops *fops;
7402
7403         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7404         fops->destroy_mtr_acts(dev, mtr_policy);
7405 }
7406
7407 /**
7408  * Create policy action, lock free,
7409  * (mutex should be acquired by caller).
7410  * Dispatcher for action type specific call.
7411  *
7412  * @param[in] dev
7413  *   Pointer to the Ethernet device structure.
7414  * @param[in] mtr_policy
7415  *   Meter policy struct.
7416  * @param[in] action
7417  *   Action specification used to create meter actions.
7418  * @param[out] error
7419  *   Perform verbose error reporting if not NULL. Initialized in case of
7420  *   error only.
7421  *
7422  * @return
7423  *   0 on success, otherwise negative errno value.
7424  */
7425 int
7426 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev,
7427                       struct mlx5_flow_meter_policy *mtr_policy,
7428                       const struct rte_flow_action *actions[RTE_COLORS],
7429                       struct rte_mtr_error *error)
7430 {
7431         const struct mlx5_flow_driver_ops *fops;
7432
7433         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7434         return fops->create_mtr_acts(dev, mtr_policy, actions, error);
7435 }
7436
7437 /**
7438  * Create policy rules, lock free,
7439  * (mutex should be acquired by caller).
7440  * Dispatcher for action type specific call.
7441  *
7442  * @param[in] dev
7443  *   Pointer to the Ethernet device structure.
7444  * @param[in] mtr_policy
7445  *   Meter policy struct.
7446  *
7447  * @return
7448  *   0 on success, -1 otherwise.
7449  */
7450 int
7451 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev,
7452                              struct mlx5_flow_meter_policy *mtr_policy)
7453 {
7454         const struct mlx5_flow_driver_ops *fops;
7455
7456         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7457         return fops->create_policy_rules(dev, mtr_policy);
7458 }
7459
7460 /**
7461  * Destroy policy rules, lock free,
7462  * (mutex should be acquired by caller).
7463  * Dispatcher for action type specific call.
7464  *
7465  * @param[in] dev
7466  *   Pointer to the Ethernet device structure.
7467  * @param[in] mtr_policy
7468  *   Meter policy struct.
7469  */
7470 void
7471 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev,
7472                              struct mlx5_flow_meter_policy *mtr_policy)
7473 {
7474         const struct mlx5_flow_driver_ops *fops;
7475
7476         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7477         fops->destroy_policy_rules(dev, mtr_policy);
7478 }
7479
7480 /**
7481  * Destroy the default policy table set.
7482  *
7483  * @param[in] dev
7484  *   Pointer to Ethernet device.
7485  */
7486 void
7487 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev)
7488 {
7489         const struct mlx5_flow_driver_ops *fops;
7490
7491         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7492         fops->destroy_def_policy(dev);
7493 }
7494
7495 /**
7496  * Destroy the default policy table set.
7497  *
7498  * @param[in] dev
7499  *   Pointer to Ethernet device.
7500  *
7501  * @return
7502  *   0 on success, -1 otherwise.
7503  */
7504 int
7505 mlx5_flow_create_def_policy(struct rte_eth_dev *dev)
7506 {
7507         const struct mlx5_flow_driver_ops *fops;
7508
7509         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7510         return fops->create_def_policy(dev);
7511 }
7512
7513 /**
7514  * Create the needed meter and suffix tables.
7515  *
7516  * @param[in] dev
7517  *   Pointer to Ethernet device.
7518  *
7519  * @return
7520  *   0 on success, -1 otherwise.
7521  */
7522 int
7523 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
7524                         struct mlx5_flow_meter_info *fm,
7525                         uint32_t mtr_idx,
7526                         uint8_t domain_bitmap)
7527 {
7528         const struct mlx5_flow_driver_ops *fops;
7529
7530         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7531         return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap);
7532 }
7533
7534 /**
7535  * Destroy the meter table set.
7536  *
7537  * @param[in] dev
7538  *   Pointer to Ethernet device.
7539  * @param[in] tbl
7540  *   Pointer to the meter table set.
7541  */
7542 void
7543 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
7544                            struct mlx5_flow_meter_info *fm)
7545 {
7546         const struct mlx5_flow_driver_ops *fops;
7547
7548         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7549         fops->destroy_mtr_tbls(dev, fm);
7550 }
7551
7552 /**
7553  * Destroy the global meter drop table.
7554  *
7555  * @param[in] dev
7556  *   Pointer to Ethernet device.
7557  */
7558 void
7559 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
7560 {
7561         const struct mlx5_flow_driver_ops *fops;
7562
7563         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7564         fops->destroy_mtr_drop_tbls(dev);
7565 }
7566
7567 /**
7568  * Destroy the sub policy table with RX queue.
7569  *
7570  * @param[in] dev
7571  *   Pointer to Ethernet device.
7572  * @param[in] mtr_policy
7573  *   Pointer to meter policy table.
7574  */
7575 void
7576 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
7577                 struct mlx5_flow_meter_policy *mtr_policy)
7578 {
7579         const struct mlx5_flow_driver_ops *fops;
7580
7581         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7582         fops->destroy_sub_policy_with_rxq(dev, mtr_policy);
7583 }
7584
7585 /**
7586  * Allocate the needed aso flow meter id.
7587  *
7588  * @param[in] dev
7589  *   Pointer to Ethernet device.
7590  *
7591  * @return
7592  *   Index to aso flow meter on success, NULL otherwise.
7593  */
7594 uint32_t
7595 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev)
7596 {
7597         const struct mlx5_flow_driver_ops *fops;
7598
7599         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7600         return fops->create_meter(dev);
7601 }
7602
7603 /**
7604  * Free the aso flow meter id.
7605  *
7606  * @param[in] dev
7607  *   Pointer to Ethernet device.
7608  * @param[in] mtr_idx
7609  *  Index to aso flow meter to be free.
7610  *
7611  * @return
7612  *   0 on success.
7613  */
7614 void
7615 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx)
7616 {
7617         const struct mlx5_flow_driver_ops *fops;
7618
7619         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7620         fops->free_meter(dev, mtr_idx);
7621 }
7622
7623 /**
7624  * Allocate a counter.
7625  *
7626  * @param[in] dev
7627  *   Pointer to Ethernet device structure.
7628  *
7629  * @return
7630  *   Index to allocated counter  on success, 0 otherwise.
7631  */
7632 uint32_t
7633 mlx5_counter_alloc(struct rte_eth_dev *dev)
7634 {
7635         const struct mlx5_flow_driver_ops *fops;
7636         struct rte_flow_attr attr = { .transfer = 0 };
7637
7638         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7639                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7640                 return fops->counter_alloc(dev);
7641         }
7642         DRV_LOG(ERR,
7643                 "port %u counter allocate is not supported.",
7644                  dev->data->port_id);
7645         return 0;
7646 }
7647
7648 /**
7649  * Free a counter.
7650  *
7651  * @param[in] dev
7652  *   Pointer to Ethernet device structure.
7653  * @param[in] cnt
7654  *   Index to counter to be free.
7655  */
7656 void
7657 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
7658 {
7659         const struct mlx5_flow_driver_ops *fops;
7660         struct rte_flow_attr attr = { .transfer = 0 };
7661
7662         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7663                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7664                 fops->counter_free(dev, cnt);
7665                 return;
7666         }
7667         DRV_LOG(ERR,
7668                 "port %u counter free is not supported.",
7669                  dev->data->port_id);
7670 }
7671
7672 /**
7673  * Query counter statistics.
7674  *
7675  * @param[in] dev
7676  *   Pointer to Ethernet device structure.
7677  * @param[in] cnt
7678  *   Index to counter to query.
7679  * @param[in] clear
7680  *   Set to clear counter statistics.
7681  * @param[out] pkts
7682  *   The counter hits packets number to save.
7683  * @param[out] bytes
7684  *   The counter hits bytes number to save.
7685  *
7686  * @return
7687  *   0 on success, a negative errno value otherwise.
7688  */
7689 int
7690 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
7691                    bool clear, uint64_t *pkts, uint64_t *bytes)
7692 {
7693         const struct mlx5_flow_driver_ops *fops;
7694         struct rte_flow_attr attr = { .transfer = 0 };
7695
7696         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7697                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7698                 return fops->counter_query(dev, cnt, clear, pkts, bytes);
7699         }
7700         DRV_LOG(ERR,
7701                 "port %u counter query is not supported.",
7702                  dev->data->port_id);
7703         return -ENOTSUP;
7704 }
7705
7706 /**
7707  * Allocate a new memory for the counter values wrapped by all the needed
7708  * management.
7709  *
7710  * @param[in] sh
7711  *   Pointer to mlx5_dev_ctx_shared object.
7712  *
7713  * @return
7714  *   0 on success, a negative errno value otherwise.
7715  */
7716 static int
7717 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
7718 {
7719         struct mlx5_devx_mkey_attr mkey_attr;
7720         struct mlx5_counter_stats_mem_mng *mem_mng;
7721         volatile struct flow_counter_stats *raw_data;
7722         int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
7723         int size = (sizeof(struct flow_counter_stats) *
7724                         MLX5_COUNTERS_PER_POOL +
7725                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
7726                         sizeof(struct mlx5_counter_stats_mem_mng);
7727         size_t pgsize = rte_mem_page_size();
7728         uint8_t *mem;
7729         int i;
7730
7731         if (pgsize == (size_t)-1) {
7732                 DRV_LOG(ERR, "Failed to get mem page size");
7733                 rte_errno = ENOMEM;
7734                 return -ENOMEM;
7735         }
7736         mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
7737         if (!mem) {
7738                 rte_errno = ENOMEM;
7739                 return -ENOMEM;
7740         }
7741         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
7742         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
7743         mem_mng->umem = mlx5_os_umem_reg(sh->cdev->ctx, mem, size,
7744                                                  IBV_ACCESS_LOCAL_WRITE);
7745         if (!mem_mng->umem) {
7746                 rte_errno = errno;
7747                 mlx5_free(mem);
7748                 return -rte_errno;
7749         }
7750         memset(&mkey_attr, 0, sizeof(mkey_attr));
7751         mkey_attr.addr = (uintptr_t)mem;
7752         mkey_attr.size = size;
7753         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
7754         mkey_attr.pd = sh->cdev->pdn;
7755         mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write;
7756         mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read;
7757         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->cdev->ctx, &mkey_attr);
7758         if (!mem_mng->dm) {
7759                 mlx5_os_umem_dereg(mem_mng->umem);
7760                 rte_errno = errno;
7761                 mlx5_free(mem);
7762                 return -rte_errno;
7763         }
7764         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
7765         raw_data = (volatile struct flow_counter_stats *)mem;
7766         for (i = 0; i < raws_n; ++i) {
7767                 mem_mng->raws[i].mem_mng = mem_mng;
7768                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
7769         }
7770         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
7771                 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
7772                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
7773                                  next);
7774         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
7775         sh->cmng.mem_mng = mem_mng;
7776         return 0;
7777 }
7778
7779 /**
7780  * Set the statistic memory to the new counter pool.
7781  *
7782  * @param[in] sh
7783  *   Pointer to mlx5_dev_ctx_shared object.
7784  * @param[in] pool
7785  *   Pointer to the pool to set the statistic memory.
7786  *
7787  * @return
7788  *   0 on success, a negative errno value otherwise.
7789  */
7790 static int
7791 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
7792                                struct mlx5_flow_counter_pool *pool)
7793 {
7794         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7795         /* Resize statistic memory once used out. */
7796         if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
7797             mlx5_flow_create_counter_stat_mem_mng(sh)) {
7798                 DRV_LOG(ERR, "Cannot resize counter stat mem.");
7799                 return -1;
7800         }
7801         rte_spinlock_lock(&pool->sl);
7802         pool->raw = cmng->mem_mng->raws + pool->index %
7803                     MLX5_CNT_CONTAINER_RESIZE;
7804         rte_spinlock_unlock(&pool->sl);
7805         pool->raw_hw = NULL;
7806         return 0;
7807 }
7808
7809 #define MLX5_POOL_QUERY_FREQ_US 1000000
7810
7811 /**
7812  * Set the periodic procedure for triggering asynchronous batch queries for all
7813  * the counter pools.
7814  *
7815  * @param[in] sh
7816  *   Pointer to mlx5_dev_ctx_shared object.
7817  */
7818 void
7819 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
7820 {
7821         uint32_t pools_n, us;
7822
7823         pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED);
7824         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
7825         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
7826         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
7827                 sh->cmng.query_thread_on = 0;
7828                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
7829         } else {
7830                 sh->cmng.query_thread_on = 1;
7831         }
7832 }
7833
7834 /**
7835  * The periodic procedure for triggering asynchronous batch queries for all the
7836  * counter pools. This function is probably called by the host thread.
7837  *
7838  * @param[in] arg
7839  *   The parameter for the alarm process.
7840  */
7841 void
7842 mlx5_flow_query_alarm(void *arg)
7843 {
7844         struct mlx5_dev_ctx_shared *sh = arg;
7845         int ret;
7846         uint16_t pool_index = sh->cmng.pool_index;
7847         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7848         struct mlx5_flow_counter_pool *pool;
7849         uint16_t n_valid;
7850
7851         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
7852                 goto set_alarm;
7853         rte_spinlock_lock(&cmng->pool_update_sl);
7854         pool = cmng->pools[pool_index];
7855         n_valid = cmng->n_valid;
7856         rte_spinlock_unlock(&cmng->pool_update_sl);
7857         /* Set the statistic memory to the new created pool. */
7858         if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
7859                 goto set_alarm;
7860         if (pool->raw_hw)
7861                 /* There is a pool query in progress. */
7862                 goto set_alarm;
7863         pool->raw_hw =
7864                 LIST_FIRST(&sh->cmng.free_stat_raws);
7865         if (!pool->raw_hw)
7866                 /* No free counter statistics raw memory. */
7867                 goto set_alarm;
7868         /*
7869          * Identify the counters released between query trigger and query
7870          * handle more efficiently. The counter released in this gap period
7871          * should wait for a new round of query as the new arrived packets
7872          * will not be taken into account.
7873          */
7874         pool->query_gen++;
7875         ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
7876                                                MLX5_COUNTERS_PER_POOL,
7877                                                NULL, NULL,
7878                                                pool->raw_hw->mem_mng->dm->id,
7879                                                (void *)(uintptr_t)
7880                                                pool->raw_hw->data,
7881                                                sh->devx_comp,
7882                                                (uint64_t)(uintptr_t)pool);
7883         if (ret) {
7884                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
7885                         " %d", pool->min_dcs->id);
7886                 pool->raw_hw = NULL;
7887                 goto set_alarm;
7888         }
7889         LIST_REMOVE(pool->raw_hw, next);
7890         sh->cmng.pending_queries++;
7891         pool_index++;
7892         if (pool_index >= n_valid)
7893                 pool_index = 0;
7894 set_alarm:
7895         sh->cmng.pool_index = pool_index;
7896         mlx5_set_query_alarm(sh);
7897 }
7898
7899 /**
7900  * Check and callback event for new aged flow in the counter pool
7901  *
7902  * @param[in] sh
7903  *   Pointer to mlx5_dev_ctx_shared object.
7904  * @param[in] pool
7905  *   Pointer to Current counter pool.
7906  */
7907 static void
7908 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
7909                    struct mlx5_flow_counter_pool *pool)
7910 {
7911         struct mlx5_priv *priv;
7912         struct mlx5_flow_counter *cnt;
7913         struct mlx5_age_info *age_info;
7914         struct mlx5_age_param *age_param;
7915         struct mlx5_counter_stats_raw *cur = pool->raw_hw;
7916         struct mlx5_counter_stats_raw *prev = pool->raw;
7917         const uint64_t curr_time = MLX5_CURR_TIME_SEC;
7918         const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
7919         uint16_t expected = AGE_CANDIDATE;
7920         uint32_t i;
7921
7922         pool->time_of_last_age_check = curr_time;
7923         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
7924                 cnt = MLX5_POOL_GET_CNT(pool, i);
7925                 age_param = MLX5_CNT_TO_AGE(cnt);
7926                 if (__atomic_load_n(&age_param->state,
7927                                     __ATOMIC_RELAXED) != AGE_CANDIDATE)
7928                         continue;
7929                 if (cur->data[i].hits != prev->data[i].hits) {
7930                         __atomic_store_n(&age_param->sec_since_last_hit, 0,
7931                                          __ATOMIC_RELAXED);
7932                         continue;
7933                 }
7934                 if (__atomic_add_fetch(&age_param->sec_since_last_hit,
7935                                        time_delta,
7936                                        __ATOMIC_RELAXED) <= age_param->timeout)
7937                         continue;
7938                 /**
7939                  * Hold the lock first, or if between the
7940                  * state AGE_TMOUT and tailq operation the
7941                  * release happened, the release procedure
7942                  * may delete a non-existent tailq node.
7943                  */
7944                 priv = rte_eth_devices[age_param->port_id].data->dev_private;
7945                 age_info = GET_PORT_AGE_INFO(priv);
7946                 rte_spinlock_lock(&age_info->aged_sl);
7947                 if (__atomic_compare_exchange_n(&age_param->state, &expected,
7948                                                 AGE_TMOUT, false,
7949                                                 __ATOMIC_RELAXED,
7950                                                 __ATOMIC_RELAXED)) {
7951                         TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
7952                         MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
7953                 }
7954                 rte_spinlock_unlock(&age_info->aged_sl);
7955         }
7956         mlx5_age_event_prepare(sh);
7957 }
7958
7959 /**
7960  * Handler for the HW respond about ready values from an asynchronous batch
7961  * query. This function is probably called by the host thread.
7962  *
7963  * @param[in] sh
7964  *   The pointer to the shared device context.
7965  * @param[in] async_id
7966  *   The Devx async ID.
7967  * @param[in] status
7968  *   The status of the completion.
7969  */
7970 void
7971 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
7972                                   uint64_t async_id, int status)
7973 {
7974         struct mlx5_flow_counter_pool *pool =
7975                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
7976         struct mlx5_counter_stats_raw *raw_to_free;
7977         uint8_t query_gen = pool->query_gen ^ 1;
7978         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7979         enum mlx5_counter_type cnt_type =
7980                 pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
7981                                 MLX5_COUNTER_TYPE_ORIGIN;
7982
7983         if (unlikely(status)) {
7984                 raw_to_free = pool->raw_hw;
7985         } else {
7986                 raw_to_free = pool->raw;
7987                 if (pool->is_aged)
7988                         mlx5_flow_aging_check(sh, pool);
7989                 rte_spinlock_lock(&pool->sl);
7990                 pool->raw = pool->raw_hw;
7991                 rte_spinlock_unlock(&pool->sl);
7992                 /* Be sure the new raw counters data is updated in memory. */
7993                 rte_io_wmb();
7994                 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
7995                         rte_spinlock_lock(&cmng->csl[cnt_type]);
7996                         TAILQ_CONCAT(&cmng->counters[cnt_type],
7997                                      &pool->counters[query_gen], next);
7998                         rte_spinlock_unlock(&cmng->csl[cnt_type]);
7999                 }
8000         }
8001         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
8002         pool->raw_hw = NULL;
8003         sh->cmng.pending_queries--;
8004 }
8005
8006 static int
8007 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
8008                     const struct flow_grp_info *grp_info,
8009                     struct rte_flow_error *error)
8010 {
8011         if (grp_info->transfer && grp_info->external &&
8012             grp_info->fdb_def_rule) {
8013                 if (group == UINT32_MAX)
8014                         return rte_flow_error_set
8015                                                 (error, EINVAL,
8016                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
8017                                                  NULL,
8018                                                  "group index not supported");
8019                 *table = group + 1;
8020         } else {
8021                 *table = group;
8022         }
8023         DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
8024         return 0;
8025 }
8026
8027 /**
8028  * Translate the rte_flow group index to HW table value.
8029  *
8030  * If tunnel offload is disabled, all group ids converted to flow table
8031  * id using the standard method.
8032  * If tunnel offload is enabled, group id can be converted using the
8033  * standard or tunnel conversion method. Group conversion method
8034  * selection depends on flags in `grp_info` parameter:
8035  * - Internal (grp_info.external == 0) groups conversion uses the
8036  *   standard method.
8037  * - Group ids in JUMP action converted with the tunnel conversion.
8038  * - Group id in rule attribute conversion depends on a rule type and
8039  *   group id value:
8040  *   ** non zero group attributes converted with the tunnel method
8041  *   ** zero group attribute in non-tunnel rule is converted using the
8042  *      standard method - there's only one root table
8043  *   ** zero group attribute in steer tunnel rule is converted with the
8044  *      standard method - single root table
8045  *   ** zero group attribute in match tunnel rule is a special OvS
8046  *      case: that value is used for portability reasons. That group
8047  *      id is converted with the tunnel conversion method.
8048  *
8049  * @param[in] dev
8050  *   Port device
8051  * @param[in] tunnel
8052  *   PMD tunnel offload object
8053  * @param[in] group
8054  *   rte_flow group index value.
8055  * @param[out] table
8056  *   HW table value.
8057  * @param[in] grp_info
8058  *   flags used for conversion
8059  * @param[out] error
8060  *   Pointer to error structure.
8061  *
8062  * @return
8063  *   0 on success, a negative errno value otherwise and rte_errno is set.
8064  */
8065 int
8066 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
8067                          const struct mlx5_flow_tunnel *tunnel,
8068                          uint32_t group, uint32_t *table,
8069                          const struct flow_grp_info *grp_info,
8070                          struct rte_flow_error *error)
8071 {
8072         int ret;
8073         bool standard_translation;
8074
8075         if (!grp_info->skip_scale && grp_info->external &&
8076             group < MLX5_MAX_TABLES_EXTERNAL)
8077                 group *= MLX5_FLOW_TABLE_FACTOR;
8078         if (is_tunnel_offload_active(dev)) {
8079                 standard_translation = !grp_info->external ||
8080                                         grp_info->std_tbl_fix;
8081         } else {
8082                 standard_translation = true;
8083         }
8084         DRV_LOG(DEBUG,
8085                 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s",
8086                 dev->data->port_id, group, grp_info->transfer,
8087                 grp_info->external, grp_info->fdb_def_rule,
8088                 standard_translation ? "STANDARD" : "TUNNEL");
8089         if (standard_translation)
8090                 ret = flow_group_to_table(dev->data->port_id, group, table,
8091                                           grp_info, error);
8092         else
8093                 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
8094                                                       table, error);
8095
8096         return ret;
8097 }
8098
8099 /**
8100  * Discover availability of metadata reg_c's.
8101  *
8102  * Iteratively use test flows to check availability.
8103  *
8104  * @param[in] dev
8105  *   Pointer to the Ethernet device structure.
8106  *
8107  * @return
8108  *   0 on success, a negative errno value otherwise and rte_errno is set.
8109  */
8110 int
8111 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
8112 {
8113         struct mlx5_priv *priv = dev->data->dev_private;
8114         enum modify_reg idx;
8115         int n = 0;
8116
8117         /* reg_c[0] and reg_c[1] are reserved. */
8118         priv->sh->flow_mreg_c[n++] = REG_C_0;
8119         priv->sh->flow_mreg_c[n++] = REG_C_1;
8120         /* Discover availability of other reg_c's. */
8121         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
8122                 struct rte_flow_attr attr = {
8123                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
8124                         .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
8125                         .ingress = 1,
8126                 };
8127                 struct rte_flow_item items[] = {
8128                         [0] = {
8129                                 .type = RTE_FLOW_ITEM_TYPE_END,
8130                         },
8131                 };
8132                 struct rte_flow_action actions[] = {
8133                         [0] = {
8134                                 .type = (enum rte_flow_action_type)
8135                                         MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
8136                                 .conf = &(struct mlx5_flow_action_copy_mreg){
8137                                         .src = REG_C_1,
8138                                         .dst = idx,
8139                                 },
8140                         },
8141                         [1] = {
8142                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
8143                                 .conf = &(struct rte_flow_action_jump){
8144                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
8145                                 },
8146                         },
8147                         [2] = {
8148                                 .type = RTE_FLOW_ACTION_TYPE_END,
8149                         },
8150                 };
8151                 uint32_t flow_idx;
8152                 struct rte_flow *flow;
8153                 struct rte_flow_error error;
8154
8155                 if (!priv->config.dv_flow_en)
8156                         break;
8157                 /* Create internal flow, validation skips copy action. */
8158                 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
8159                                         items, actions, false, &error);
8160                 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
8161                                       flow_idx);
8162                 if (!flow)
8163                         continue;
8164                 priv->sh->flow_mreg_c[n++] = idx;
8165                 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
8166         }
8167         for (; n < MLX5_MREG_C_NUM; ++n)
8168                 priv->sh->flow_mreg_c[n] = REG_NON;
8169         priv->sh->metadata_regc_check_flag = 1;
8170         return 0;
8171 }
8172
8173 int
8174 save_dump_file(const uint8_t *data, uint32_t size,
8175         uint32_t type, uint64_t id, void *arg, FILE *file)
8176 {
8177         char line[BUF_SIZE];
8178         uint32_t out = 0;
8179         uint32_t k;
8180         uint32_t actions_num;
8181         struct rte_flow_query_count *count;
8182
8183         memset(line, 0, BUF_SIZE);
8184         switch (type) {
8185         case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR:
8186                 actions_num = *(uint32_t *)(arg);
8187                 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,",
8188                                 type, id, actions_num);
8189                 break;
8190         case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT:
8191                 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",",
8192                                 type, id);
8193                 break;
8194         case DR_DUMP_REC_TYPE_PMD_COUNTER:
8195                 count = (struct rte_flow_query_count *)arg;
8196                 fprintf(file,
8197                         "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n",
8198                         type, id, count->hits, count->bytes);
8199                 return 0;
8200         default:
8201                 return -1;
8202         }
8203
8204         for (k = 0; k < size; k++) {
8205                 /* Make sure we do not overrun the line buffer length. */
8206                 if (out >= BUF_SIZE - 4) {
8207                         line[out] = '\0';
8208                         break;
8209                 }
8210                 out += snprintf(line + out, BUF_SIZE - out, "%02x",
8211                                 (data[k]) & 0xff);
8212         }
8213         fprintf(file, "%s\n", line);
8214         return 0;
8215 }
8216
8217 int
8218 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow,
8219         struct rte_flow_query_count *count, struct rte_flow_error *error)
8220 {
8221         struct rte_flow_action action[2];
8222         enum mlx5_flow_drv_type ftype;
8223         const struct mlx5_flow_driver_ops *fops;
8224
8225         if (!flow) {
8226                 return rte_flow_error_set(error, ENOENT,
8227                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8228                                 NULL,
8229                                 "invalid flow handle");
8230         }
8231         action[0].type = RTE_FLOW_ACTION_TYPE_COUNT;
8232         action[1].type = RTE_FLOW_ACTION_TYPE_END;
8233         if (flow->counter) {
8234                 memset(count, 0, sizeof(struct rte_flow_query_count));
8235                 ftype = (enum mlx5_flow_drv_type)(flow->drv_type);
8236                 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN &&
8237                                                 ftype < MLX5_FLOW_TYPE_MAX);
8238                 fops = flow_get_drv_ops(ftype);
8239                 return fops->query(dev, flow, action, count, error);
8240         }
8241         return -1;
8242 }
8243
8244 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8245 /**
8246  * Dump flow ipool data to file
8247  *
8248  * @param[in] dev
8249  *   The pointer to Ethernet device.
8250  * @param[in] file
8251  *   A pointer to a file for output.
8252  * @param[out] error
8253  *   Perform verbose error reporting if not NULL. PMDs initialize this
8254  *   structure in case of error only.
8255  * @return
8256  *   0 on success, a negative value otherwise.
8257  */
8258 int
8259 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev,
8260         struct rte_flow *flow, FILE *file,
8261         struct rte_flow_error *error)
8262 {
8263         struct mlx5_priv *priv = dev->data->dev_private;
8264         struct mlx5_flow_dv_modify_hdr_resource  *modify_hdr;
8265         struct mlx5_flow_dv_encap_decap_resource *encap_decap;
8266         uint32_t handle_idx;
8267         struct mlx5_flow_handle *dh;
8268         struct rte_flow_query_count count;
8269         uint32_t actions_num;
8270         const uint8_t *data;
8271         size_t size;
8272         uint64_t id;
8273         uint32_t type;
8274         void *action = NULL;
8275
8276         if (!flow) {
8277                 return rte_flow_error_set(error, ENOENT,
8278                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8279                                 NULL,
8280                                 "invalid flow handle");
8281         }
8282         handle_idx = flow->dev_handles;
8283         while (handle_idx) {
8284                 dh = mlx5_ipool_get(priv->sh->ipool
8285                                 [MLX5_IPOOL_MLX5_FLOW], handle_idx);
8286                 if (!dh)
8287                         continue;
8288                 handle_idx = dh->next.next;
8289
8290                 /* query counter */
8291                 type = DR_DUMP_REC_TYPE_PMD_COUNTER;
8292                 flow_dv_query_count_ptr(dev, flow->counter,
8293                                                 &action, error);
8294                 if (action) {
8295                         id = (uint64_t)(uintptr_t)action;
8296                         if (!mlx5_flow_query_counter(dev, flow, &count, error))
8297                                 save_dump_file(NULL, 0, type,
8298                                                 id, (void *)&count, file);
8299                 }
8300                 /* Get modify_hdr and encap_decap buf from ipools. */
8301                 encap_decap = NULL;
8302                 modify_hdr = dh->dvh.modify_hdr;
8303
8304                 if (dh->dvh.rix_encap_decap) {
8305                         encap_decap = mlx5_ipool_get(priv->sh->ipool
8306                                                 [MLX5_IPOOL_DECAP_ENCAP],
8307                                                 dh->dvh.rix_encap_decap);
8308                 }
8309                 if (modify_hdr) {
8310                         data = (const uint8_t *)modify_hdr->actions;
8311                         size = (size_t)(modify_hdr->actions_num) * 8;
8312                         id = (uint64_t)(uintptr_t)modify_hdr->action;
8313                         actions_num = modify_hdr->actions_num;
8314                         type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR;
8315                         save_dump_file(data, size, type, id,
8316                                                 (void *)(&actions_num), file);
8317                 }
8318                 if (encap_decap) {
8319                         data = encap_decap->buf;
8320                         size = encap_decap->size;
8321                         id = (uint64_t)(uintptr_t)encap_decap->action;
8322                         type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT;
8323                         save_dump_file(data, size, type,
8324                                                 id, NULL, file);
8325                 }
8326         }
8327         return 0;
8328 }
8329
8330 /**
8331  * Dump all flow's encap_decap/modify_hdr/counter data to file
8332  *
8333  * @param[in] dev
8334  *   The pointer to Ethernet device.
8335  * @param[in] file
8336  *   A pointer to a file for output.
8337  * @param[out] error
8338  *   Perform verbose error reporting if not NULL. PMDs initialize this
8339  *   structure in case of error only.
8340  * @return
8341  *   0 on success, a negative value otherwise.
8342  */
8343 static int
8344 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev,
8345         FILE *file, struct rte_flow_error *error)
8346 {
8347         struct mlx5_priv *priv = dev->data->dev_private;
8348         struct mlx5_dev_ctx_shared *sh = priv->sh;
8349         struct mlx5_hlist *h;
8350         struct mlx5_flow_dv_modify_hdr_resource  *modify_hdr;
8351         struct mlx5_flow_dv_encap_decap_resource *encap_decap;
8352         struct rte_flow_query_count count;
8353         uint32_t actions_num;
8354         const uint8_t *data;
8355         size_t size;
8356         uint64_t id;
8357         uint32_t type;
8358         uint32_t i;
8359         uint32_t j;
8360         struct mlx5_list_inconst *l_inconst;
8361         struct mlx5_list_entry *e;
8362         int lcore_index;
8363         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
8364         uint32_t max;
8365         void *action;
8366
8367         /* encap_decap hlist is lcore_share, get global core cache. */
8368         i = MLX5_LIST_GLOBAL;
8369         h = sh->encaps_decaps;
8370         if (h) {
8371                 for (j = 0; j <= h->mask; j++) {
8372                         l_inconst = &h->buckets[j].l;
8373                         if (!l_inconst || !l_inconst->cache[i])
8374                                 continue;
8375
8376                         e = LIST_FIRST(&l_inconst->cache[i]->h);
8377                         while (e) {
8378                                 encap_decap =
8379                                 (struct mlx5_flow_dv_encap_decap_resource *)e;
8380                                 data = encap_decap->buf;
8381                                 size = encap_decap->size;
8382                                 id = (uint64_t)(uintptr_t)encap_decap->action;
8383                                 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT;
8384                                 save_dump_file(data, size, type,
8385                                         id, NULL, file);
8386                                 e = LIST_NEXT(e, next);
8387                         }
8388                 }
8389         }
8390
8391         /* get modify_hdr */
8392         h = sh->modify_cmds;
8393         if (h) {
8394                 lcore_index = rte_lcore_index(rte_lcore_id());
8395                 if (unlikely(lcore_index == -1)) {
8396                         lcore_index = MLX5_LIST_NLCORE;
8397                         rte_spinlock_lock(&h->l_const.lcore_lock);
8398                 }
8399                 i = lcore_index;
8400
8401                 for (j = 0; j <= h->mask; j++) {
8402                         l_inconst = &h->buckets[j].l;
8403                         if (!l_inconst || !l_inconst->cache[i])
8404                                 continue;
8405
8406                         e = LIST_FIRST(&l_inconst->cache[i]->h);
8407                         while (e) {
8408                                 modify_hdr =
8409                                 (struct mlx5_flow_dv_modify_hdr_resource *)e;
8410                                 data = (const uint8_t *)modify_hdr->actions;
8411                                 size = (size_t)(modify_hdr->actions_num) * 8;
8412                                 actions_num = modify_hdr->actions_num;
8413                                 id = (uint64_t)(uintptr_t)modify_hdr->action;
8414                                 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR;
8415                                 save_dump_file(data, size, type, id,
8416                                                 (void *)(&actions_num), file);
8417                                 e = LIST_NEXT(e, next);
8418                         }
8419                 }
8420
8421                 if (unlikely(lcore_index == MLX5_LIST_NLCORE))
8422                         rte_spinlock_unlock(&h->l_const.lcore_lock);
8423         }
8424
8425         /* get counter */
8426         MLX5_ASSERT(cmng->n_valid <= cmng->n);
8427         max = MLX5_COUNTERS_PER_POOL * cmng->n_valid;
8428         for (j = 1; j <= max; j++) {
8429                 action = NULL;
8430                 flow_dv_query_count_ptr(dev, j, &action, error);
8431                 if (action) {
8432                         if (!flow_dv_query_count(dev, j, &count, error)) {
8433                                 type = DR_DUMP_REC_TYPE_PMD_COUNTER;
8434                                 id = (uint64_t)(uintptr_t)action;
8435                                 save_dump_file(NULL, 0, type,
8436                                                 id, (void *)&count, file);
8437                         }
8438                 }
8439         }
8440         return 0;
8441 }
8442 #endif
8443
8444 /**
8445  * Dump flow raw hw data to file
8446  *
8447  * @param[in] dev
8448  *    The pointer to Ethernet device.
8449  * @param[in] file
8450  *   A pointer to a file for output.
8451  * @param[out] error
8452  *   Perform verbose error reporting if not NULL. PMDs initialize this
8453  *   structure in case of error only.
8454  * @return
8455  *   0 on success, a nagative value otherwise.
8456  */
8457 int
8458 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
8459                    FILE *file,
8460                    struct rte_flow_error *error __rte_unused)
8461 {
8462         struct mlx5_priv *priv = dev->data->dev_private;
8463         struct mlx5_dev_ctx_shared *sh = priv->sh;
8464         uint32_t handle_idx;
8465         int ret;
8466         struct mlx5_flow_handle *dh;
8467         struct rte_flow *flow;
8468
8469         if (!priv->config.dv_flow_en) {
8470                 if (fputs("device dv flow disabled\n", file) <= 0)
8471                         return -errno;
8472                 return -ENOTSUP;
8473         }
8474
8475         /* dump all */
8476         if (!flow_idx) {
8477 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8478                 if (mlx5_flow_dev_dump_sh_all(dev, file, error))
8479                         return -EINVAL;
8480 #endif
8481                 return mlx5_devx_cmd_flow_dump(sh->fdb_domain,
8482                                         sh->rx_domain,
8483                                         sh->tx_domain, file);
8484         }
8485         /* dump one */
8486         flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
8487                         (uintptr_t)(void *)flow_idx);
8488         if (!flow)
8489                 return -EINVAL;
8490
8491 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8492         mlx5_flow_dev_dump_ipool(dev, flow, file, error);
8493 #endif
8494         handle_idx = flow->dev_handles;
8495         while (handle_idx) {
8496                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
8497                                 handle_idx);
8498                 if (!dh)
8499                         return -ENOENT;
8500                 if (dh->drv_flow) {
8501                         ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow,
8502                                         file);
8503                         if (ret)
8504                                 return -ENOENT;
8505                 }
8506                 handle_idx = dh->next.next;
8507         }
8508         return 0;
8509 }
8510
8511 /**
8512  * Get aged-out flows.
8513  *
8514  * @param[in] dev
8515  *   Pointer to the Ethernet device structure.
8516  * @param[in] context
8517  *   The address of an array of pointers to the aged-out flows contexts.
8518  * @param[in] nb_countexts
8519  *   The length of context array pointers.
8520  * @param[out] error
8521  *   Perform verbose error reporting if not NULL. Initialized in case of
8522  *   error only.
8523  *
8524  * @return
8525  *   how many contexts get in success, otherwise negative errno value.
8526  *   if nb_contexts is 0, return the amount of all aged contexts.
8527  *   if nb_contexts is not 0 , return the amount of aged flows reported
8528  *   in the context array.
8529  */
8530 int
8531 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
8532                         uint32_t nb_contexts, struct rte_flow_error *error)
8533 {
8534         const struct mlx5_flow_driver_ops *fops;
8535         struct rte_flow_attr attr = { .transfer = 0 };
8536
8537         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
8538                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8539                 return fops->get_aged_flows(dev, contexts, nb_contexts,
8540                                                     error);
8541         }
8542         DRV_LOG(ERR,
8543                 "port %u get aged flows is not supported.",
8544                  dev->data->port_id);
8545         return -ENOTSUP;
8546 }
8547
8548 /* Wrapper for driver action_validate op callback */
8549 static int
8550 flow_drv_action_validate(struct rte_eth_dev *dev,
8551                          const struct rte_flow_indir_action_conf *conf,
8552                          const struct rte_flow_action *action,
8553                          const struct mlx5_flow_driver_ops *fops,
8554                          struct rte_flow_error *error)
8555 {
8556         static const char err_msg[] = "indirect action validation unsupported";
8557
8558         if (!fops->action_validate) {
8559                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8560                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8561                                    NULL, err_msg);
8562                 return -rte_errno;
8563         }
8564         return fops->action_validate(dev, conf, action, error);
8565 }
8566
8567 /**
8568  * Destroys the shared action by handle.
8569  *
8570  * @param dev
8571  *   Pointer to Ethernet device structure.
8572  * @param[in] handle
8573  *   Handle for the indirect action object to be destroyed.
8574  * @param[out] error
8575  *   Perform verbose error reporting if not NULL. PMDs initialize this
8576  *   structure in case of error only.
8577  *
8578  * @return
8579  *   0 on success, a negative errno value otherwise and rte_errno is set.
8580  *
8581  * @note: wrapper for driver action_create op callback.
8582  */
8583 static int
8584 mlx5_action_handle_destroy(struct rte_eth_dev *dev,
8585                            struct rte_flow_action_handle *handle,
8586                            struct rte_flow_error *error)
8587 {
8588         static const char err_msg[] = "indirect action destruction unsupported";
8589         struct rte_flow_attr attr = { .transfer = 0 };
8590         const struct mlx5_flow_driver_ops *fops =
8591                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8592
8593         if (!fops->action_destroy) {
8594                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8595                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8596                                    NULL, err_msg);
8597                 return -rte_errno;
8598         }
8599         return fops->action_destroy(dev, handle, error);
8600 }
8601
8602 /* Wrapper for driver action_destroy op callback */
8603 static int
8604 flow_drv_action_update(struct rte_eth_dev *dev,
8605                        struct rte_flow_action_handle *handle,
8606                        const void *update,
8607                        const struct mlx5_flow_driver_ops *fops,
8608                        struct rte_flow_error *error)
8609 {
8610         static const char err_msg[] = "indirect action update unsupported";
8611
8612         if (!fops->action_update) {
8613                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8614                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8615                                    NULL, err_msg);
8616                 return -rte_errno;
8617         }
8618         return fops->action_update(dev, handle, update, error);
8619 }
8620
8621 /* Wrapper for driver action_destroy op callback */
8622 static int
8623 flow_drv_action_query(struct rte_eth_dev *dev,
8624                       const struct rte_flow_action_handle *handle,
8625                       void *data,
8626                       const struct mlx5_flow_driver_ops *fops,
8627                       struct rte_flow_error *error)
8628 {
8629         static const char err_msg[] = "indirect action query unsupported";
8630
8631         if (!fops->action_query) {
8632                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8633                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8634                                    NULL, err_msg);
8635                 return -rte_errno;
8636         }
8637         return fops->action_query(dev, handle, data, error);
8638 }
8639
8640 /**
8641  * Create indirect action for reuse in multiple flow rules.
8642  *
8643  * @param dev
8644  *   Pointer to Ethernet device structure.
8645  * @param conf
8646  *   Pointer to indirect action object configuration.
8647  * @param[in] action
8648  *   Action configuration for indirect action object creation.
8649  * @param[out] error
8650  *   Perform verbose error reporting if not NULL. PMDs initialize this
8651  *   structure in case of error only.
8652  * @return
8653  *   A valid handle in case of success, NULL otherwise and rte_errno is set.
8654  */
8655 static struct rte_flow_action_handle *
8656 mlx5_action_handle_create(struct rte_eth_dev *dev,
8657                           const struct rte_flow_indir_action_conf *conf,
8658                           const struct rte_flow_action *action,
8659                           struct rte_flow_error *error)
8660 {
8661         static const char err_msg[] = "indirect action creation unsupported";
8662         struct rte_flow_attr attr = { .transfer = 0 };
8663         const struct mlx5_flow_driver_ops *fops =
8664                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8665
8666         if (flow_drv_action_validate(dev, conf, action, fops, error))
8667                 return NULL;
8668         if (!fops->action_create) {
8669                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8670                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8671                                    NULL, err_msg);
8672                 return NULL;
8673         }
8674         return fops->action_create(dev, conf, action, error);
8675 }
8676
8677 /**
8678  * Updates inplace the indirect action configuration pointed by *handle*
8679  * with the configuration provided as *update* argument.
8680  * The update of the indirect action configuration effects all flow rules
8681  * reusing the action via handle.
8682  *
8683  * @param dev
8684  *   Pointer to Ethernet device structure.
8685  * @param[in] handle
8686  *   Handle for the indirect action to be updated.
8687  * @param[in] update
8688  *   Action specification used to modify the action pointed by handle.
8689  *   *update* could be of same type with the action pointed by the *handle*
8690  *   handle argument, or some other structures like a wrapper, depending on
8691  *   the indirect action type.
8692  * @param[out] error
8693  *   Perform verbose error reporting if not NULL. PMDs initialize this
8694  *   structure in case of error only.
8695  *
8696  * @return
8697  *   0 on success, a negative errno value otherwise and rte_errno is set.
8698  */
8699 static int
8700 mlx5_action_handle_update(struct rte_eth_dev *dev,
8701                 struct rte_flow_action_handle *handle,
8702                 const void *update,
8703                 struct rte_flow_error *error)
8704 {
8705         struct rte_flow_attr attr = { .transfer = 0 };
8706         const struct mlx5_flow_driver_ops *fops =
8707                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8708         int ret;
8709
8710         ret = flow_drv_action_validate(dev, NULL,
8711                         (const struct rte_flow_action *)update, fops, error);
8712         if (ret)
8713                 return ret;
8714         return flow_drv_action_update(dev, handle, update, fops,
8715                                       error);
8716 }
8717
8718 /**
8719  * Query the indirect action by handle.
8720  *
8721  * This function allows retrieving action-specific data such as counters.
8722  * Data is gathered by special action which may be present/referenced in
8723  * more than one flow rule definition.
8724  *
8725  * see @RTE_FLOW_ACTION_TYPE_COUNT
8726  *
8727  * @param dev
8728  *   Pointer to Ethernet device structure.
8729  * @param[in] handle
8730  *   Handle for the indirect action to query.
8731  * @param[in, out] data
8732  *   Pointer to storage for the associated query data type.
8733  * @param[out] error
8734  *   Perform verbose error reporting if not NULL. PMDs initialize this
8735  *   structure in case of error only.
8736  *
8737  * @return
8738  *   0 on success, a negative errno value otherwise and rte_errno is set.
8739  */
8740 static int
8741 mlx5_action_handle_query(struct rte_eth_dev *dev,
8742                          const struct rte_flow_action_handle *handle,
8743                          void *data,
8744                          struct rte_flow_error *error)
8745 {
8746         struct rte_flow_attr attr = { .transfer = 0 };
8747         const struct mlx5_flow_driver_ops *fops =
8748                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8749
8750         return flow_drv_action_query(dev, handle, data, fops, error);
8751 }
8752
8753 /**
8754  * Destroy all indirect actions (shared RSS).
8755  *
8756  * @param dev
8757  *   Pointer to Ethernet device.
8758  *
8759  * @return
8760  *   0 on success, a negative errno value otherwise and rte_errno is set.
8761  */
8762 int
8763 mlx5_action_handle_flush(struct rte_eth_dev *dev)
8764 {
8765         struct rte_flow_error error;
8766         struct mlx5_priv *priv = dev->data->dev_private;
8767         struct mlx5_shared_action_rss *shared_rss;
8768         int ret = 0;
8769         uint32_t idx;
8770
8771         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
8772                       priv->rss_shared_actions, idx, shared_rss, next) {
8773                 ret |= mlx5_action_handle_destroy(dev,
8774                        (struct rte_flow_action_handle *)(uintptr_t)idx, &error);
8775         }
8776         return ret;
8777 }
8778
8779 #ifndef HAVE_MLX5DV_DR
8780 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1))
8781 #else
8782 #define MLX5_DOMAIN_SYNC_FLOW \
8783         (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW)
8784 #endif
8785
8786 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
8787 {
8788         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
8789         const struct mlx5_flow_driver_ops *fops;
8790         int ret;
8791         struct rte_flow_attr attr = { .transfer = 0 };
8792
8793         fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8794         ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW);
8795         if (ret > 0)
8796                 ret = -ret;
8797         return ret;
8798 }
8799
8800 const struct mlx5_flow_tunnel *
8801 mlx5_get_tof(const struct rte_flow_item *item,
8802              const struct rte_flow_action *action,
8803              enum mlx5_tof_rule_type *rule_type)
8804 {
8805         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
8806                 if (item->type == (typeof(item->type))
8807                                   MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) {
8808                         *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE;
8809                         return flow_items_to_tunnel(item);
8810                 }
8811         }
8812         for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) {
8813                 if (action->type == (typeof(action->type))
8814                                     MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) {
8815                         *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE;
8816                         return flow_actions_to_tunnel(action);
8817                 }
8818         }
8819         return NULL;
8820 }
8821
8822 /**
8823  * tunnel offload functionalilty is defined for DV environment only
8824  */
8825 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8826 __extension__
8827 union tunnel_offload_mark {
8828         uint32_t val;
8829         struct {
8830                 uint32_t app_reserve:8;
8831                 uint32_t table_id:15;
8832                 uint32_t transfer:1;
8833                 uint32_t _unused_:8;
8834         };
8835 };
8836
8837 static bool
8838 mlx5_access_tunnel_offload_db
8839         (struct rte_eth_dev *dev,
8840          bool (*match)(struct rte_eth_dev *,
8841                        struct mlx5_flow_tunnel *, const void *),
8842          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
8843          void (*miss)(struct rte_eth_dev *, void *),
8844          void *ctx, bool lock_op);
8845
8846 static int
8847 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
8848                              struct rte_flow *flow,
8849                              const struct rte_flow_attr *attr,
8850                              const struct rte_flow_action *app_actions,
8851                              uint32_t flow_idx,
8852                              const struct mlx5_flow_tunnel *tunnel,
8853                              struct tunnel_default_miss_ctx *ctx,
8854                              struct rte_flow_error *error)
8855 {
8856         struct mlx5_priv *priv = dev->data->dev_private;
8857         struct mlx5_flow *dev_flow;
8858         struct rte_flow_attr miss_attr = *attr;
8859         const struct rte_flow_item miss_items[2] = {
8860                 {
8861                         .type = RTE_FLOW_ITEM_TYPE_ETH,
8862                         .spec = NULL,
8863                         .last = NULL,
8864                         .mask = NULL
8865                 },
8866                 {
8867                         .type = RTE_FLOW_ITEM_TYPE_END,
8868                         .spec = NULL,
8869                         .last = NULL,
8870                         .mask = NULL
8871                 }
8872         };
8873         union tunnel_offload_mark mark_id;
8874         struct rte_flow_action_mark miss_mark;
8875         struct rte_flow_action miss_actions[3] = {
8876                 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
8877                 [2] = { .type = RTE_FLOW_ACTION_TYPE_END,  .conf = NULL }
8878         };
8879         const struct rte_flow_action_jump *jump_data;
8880         uint32_t i, flow_table = 0; /* prevent compilation warning */
8881         struct flow_grp_info grp_info = {
8882                 .external = 1,
8883                 .transfer = attr->transfer,
8884                 .fdb_def_rule = !!priv->fdb_def_rule,
8885                 .std_tbl_fix = 0,
8886         };
8887         int ret;
8888
8889         if (!attr->transfer) {
8890                 uint32_t q_size;
8891
8892                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
8893                 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
8894                 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
8895                                          0, SOCKET_ID_ANY);
8896                 if (!ctx->queue)
8897                         return rte_flow_error_set
8898                                 (error, ENOMEM,
8899                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8900                                 NULL, "invalid default miss RSS");
8901                 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
8902                 ctx->action_rss.level = 0,
8903                 ctx->action_rss.types = priv->rss_conf.rss_hf,
8904                 ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
8905                 ctx->action_rss.queue_num = priv->reta_idx_n,
8906                 ctx->action_rss.key = priv->rss_conf.rss_key,
8907                 ctx->action_rss.queue = ctx->queue;
8908                 if (!priv->reta_idx_n || !priv->rxqs_n)
8909                         return rte_flow_error_set
8910                                 (error, EINVAL,
8911                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8912                                 NULL, "invalid port configuration");
8913                 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG))
8914                         ctx->action_rss.types = 0;
8915                 for (i = 0; i != priv->reta_idx_n; ++i)
8916                         ctx->queue[i] = (*priv->reta_idx)[i];
8917         } else {
8918                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
8919                 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
8920         }
8921         miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
8922         for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
8923         jump_data = app_actions->conf;
8924         miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
8925         miss_attr.group = jump_data->group;
8926         ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
8927                                        &flow_table, &grp_info, error);
8928         if (ret)
8929                 return rte_flow_error_set(error, EINVAL,
8930                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8931                                           NULL, "invalid tunnel id");
8932         mark_id.app_reserve = 0;
8933         mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
8934         mark_id.transfer = !!attr->transfer;
8935         mark_id._unused_ = 0;
8936         miss_mark.id = mark_id.val;
8937         dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
8938                                     miss_items, miss_actions, flow_idx, error);
8939         if (!dev_flow)
8940                 return -rte_errno;
8941         dev_flow->flow = flow;
8942         dev_flow->external = true;
8943         dev_flow->tunnel = tunnel;
8944         dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE;
8945         /* Subflow object was created, we must include one in the list. */
8946         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
8947                       dev_flow->handle, next);
8948         DRV_LOG(DEBUG,
8949                 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
8950                 dev->data->port_id, tunnel->app_tunnel.type,
8951                 tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
8952         ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
8953                                   miss_actions, error);
8954         if (!ret)
8955                 ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
8956                                                   error);
8957
8958         return ret;
8959 }
8960
8961 static const struct mlx5_flow_tbl_data_entry  *
8962 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
8963 {
8964         struct mlx5_priv *priv = dev->data->dev_private;
8965         struct mlx5_dev_ctx_shared *sh = priv->sh;
8966         struct mlx5_list_entry *he;
8967         union tunnel_offload_mark mbits = { .val = mark };
8968         union mlx5_flow_tbl_key table_key = {
8969                 {
8970                         .level = tunnel_id_to_flow_tbl(mbits.table_id),
8971                         .id = 0,
8972                         .reserved = 0,
8973                         .dummy = 0,
8974                         .is_fdb = !!mbits.transfer,
8975                         .is_egress = 0,
8976                 }
8977         };
8978         struct mlx5_flow_cb_ctx ctx = {
8979                 .data = &table_key.v64,
8980         };
8981
8982         he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx);
8983         return he ?
8984                container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
8985 }
8986
8987 static void
8988 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx,
8989                                    struct mlx5_list_entry *entry)
8990 {
8991         struct mlx5_dev_ctx_shared *sh = tool_ctx;
8992         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
8993
8994         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8995                         tunnel_flow_tbl_to_id(tte->flow_table));
8996         mlx5_free(tte);
8997 }
8998
8999 static int
9000 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused,
9001                                   struct mlx5_list_entry *entry, void *cb_ctx)
9002 {
9003         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
9004         union tunnel_tbl_key tbl = {
9005                 .val = *(uint64_t *)(ctx->data),
9006         };
9007         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
9008
9009         return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
9010 }
9011
9012 static struct mlx5_list_entry *
9013 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx)
9014 {
9015         struct mlx5_dev_ctx_shared *sh = tool_ctx;
9016         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
9017         struct tunnel_tbl_entry *tte;
9018         union tunnel_tbl_key tbl = {
9019                 .val = *(uint64_t *)(ctx->data),
9020         };
9021
9022         tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
9023                           sizeof(*tte), 0,
9024                           SOCKET_ID_ANY);
9025         if (!tte)
9026                 goto err;
9027         mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
9028                           &tte->flow_table);
9029         if (tte->flow_table >= MLX5_MAX_TABLES) {
9030                 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
9031                         tte->flow_table);
9032                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
9033                                 tte->flow_table);
9034                 goto err;
9035         } else if (!tte->flow_table) {
9036                 goto err;
9037         }
9038         tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
9039         tte->tunnel_id = tbl.tunnel_id;
9040         tte->group = tbl.group;
9041         return &tte->hash;
9042 err:
9043         if (tte)
9044                 mlx5_free(tte);
9045         return NULL;
9046 }
9047
9048 static struct mlx5_list_entry *
9049 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused,
9050                                   struct mlx5_list_entry *oentry,
9051                                   void *cb_ctx __rte_unused)
9052 {
9053         struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte),
9054                                                    0, SOCKET_ID_ANY);
9055
9056         if (!tte)
9057                 return NULL;
9058         memcpy(tte, oentry, sizeof(*tte));
9059         return &tte->hash;
9060 }
9061
9062 static void
9063 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused,
9064                                        struct mlx5_list_entry *entry)
9065 {
9066         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
9067
9068         mlx5_free(tte);
9069 }
9070
9071 static uint32_t
9072 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
9073                                 const struct mlx5_flow_tunnel *tunnel,
9074                                 uint32_t group, uint32_t *table,
9075                                 struct rte_flow_error *error)
9076 {
9077         struct mlx5_list_entry *he;
9078         struct tunnel_tbl_entry *tte;
9079         union tunnel_tbl_key key = {
9080                 .tunnel_id = tunnel ? tunnel->tunnel_id : 0,
9081                 .group = group
9082         };
9083         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
9084         struct mlx5_hlist *group_hash;
9085         struct mlx5_flow_cb_ctx ctx = {
9086                 .data = &key.val,
9087         };
9088
9089         group_hash = tunnel ? tunnel->groups : thub->groups;
9090         he = mlx5_hlist_register(group_hash, key.val, &ctx);
9091         if (!he)
9092                 return rte_flow_error_set(error, EINVAL,
9093                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
9094                                           NULL,
9095                                           "tunnel group index not supported");
9096         tte = container_of(he, typeof(*tte), hash);
9097         *table = tte->flow_table;
9098         DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
9099                 dev->data->port_id, key.tunnel_id, group, *table);
9100         return 0;
9101 }
9102
9103 static void
9104 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
9105                       struct mlx5_flow_tunnel *tunnel)
9106 {
9107         struct mlx5_priv *priv = dev->data->dev_private;
9108         struct mlx5_indexed_pool *ipool;
9109
9110         DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
9111                 dev->data->port_id, tunnel->tunnel_id);
9112         LIST_REMOVE(tunnel, chain);
9113         mlx5_hlist_destroy(tunnel->groups);
9114         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
9115         mlx5_ipool_free(ipool, tunnel->tunnel_id);
9116 }
9117
9118 static bool
9119 mlx5_access_tunnel_offload_db
9120         (struct rte_eth_dev *dev,
9121          bool (*match)(struct rte_eth_dev *,
9122                        struct mlx5_flow_tunnel *, const void *),
9123          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
9124          void (*miss)(struct rte_eth_dev *, void *),
9125          void *ctx, bool lock_op)
9126 {
9127         bool verdict = false;
9128         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
9129         struct mlx5_flow_tunnel *tunnel;
9130
9131         rte_spinlock_lock(&thub->sl);
9132         LIST_FOREACH(tunnel, &thub->tunnels, chain) {
9133                 verdict = match(dev, tunnel, (const void *)ctx);
9134                 if (verdict)
9135                         break;
9136         }
9137         if (!lock_op)
9138                 rte_spinlock_unlock(&thub->sl);
9139         if (verdict && hit)
9140                 hit(dev, tunnel, ctx);
9141         if (!verdict && miss)
9142                 miss(dev, ctx);
9143         if (lock_op)
9144                 rte_spinlock_unlock(&thub->sl);
9145
9146         return verdict;
9147 }
9148
9149 struct tunnel_db_find_tunnel_id_ctx {
9150         uint32_t tunnel_id;
9151         struct mlx5_flow_tunnel *tunnel;
9152 };
9153
9154 static bool
9155 find_tunnel_id_match(struct rte_eth_dev *dev,
9156                      struct mlx5_flow_tunnel *tunnel, const void *x)
9157 {
9158         const struct tunnel_db_find_tunnel_id_ctx *ctx = x;
9159
9160         RTE_SET_USED(dev);
9161         return tunnel->tunnel_id == ctx->tunnel_id;
9162 }
9163
9164 static void
9165 find_tunnel_id_hit(struct rte_eth_dev *dev,
9166                    struct mlx5_flow_tunnel *tunnel, void *x)
9167 {
9168         struct tunnel_db_find_tunnel_id_ctx *ctx = x;
9169         RTE_SET_USED(dev);
9170         ctx->tunnel = tunnel;
9171 }
9172
9173 static struct mlx5_flow_tunnel *
9174 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
9175 {
9176         struct tunnel_db_find_tunnel_id_ctx ctx = {
9177                 .tunnel_id = id,
9178         };
9179
9180         mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match,
9181                                       find_tunnel_id_hit, NULL, &ctx, true);
9182
9183         return ctx.tunnel;
9184 }
9185
9186 static struct mlx5_flow_tunnel *
9187 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
9188                           const struct rte_flow_tunnel *app_tunnel)
9189 {
9190         struct mlx5_priv *priv = dev->data->dev_private;
9191         struct mlx5_indexed_pool *ipool;
9192         struct mlx5_flow_tunnel *tunnel;
9193         uint32_t id;
9194
9195         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
9196         tunnel = mlx5_ipool_zmalloc(ipool, &id);
9197         if (!tunnel)
9198                 return NULL;
9199         if (id >= MLX5_MAX_TUNNELS) {
9200                 mlx5_ipool_free(ipool, id);
9201                 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
9202                 return NULL;
9203         }
9204         tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true,
9205                                            priv->sh,
9206                                            mlx5_flow_tunnel_grp2tbl_create_cb,
9207                                            mlx5_flow_tunnel_grp2tbl_match_cb,
9208                                            mlx5_flow_tunnel_grp2tbl_remove_cb,
9209                                            mlx5_flow_tunnel_grp2tbl_clone_cb,
9210                                         mlx5_flow_tunnel_grp2tbl_clone_free_cb);
9211         if (!tunnel->groups) {
9212                 mlx5_ipool_free(ipool, id);
9213                 return NULL;
9214         }
9215         /* initiate new PMD tunnel */
9216         memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
9217         tunnel->tunnel_id = id;
9218         tunnel->action.type = (typeof(tunnel->action.type))
9219                               MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
9220         tunnel->action.conf = tunnel;
9221         tunnel->item.type = (typeof(tunnel->item.type))
9222                             MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
9223         tunnel->item.spec = tunnel;
9224         tunnel->item.last = NULL;
9225         tunnel->item.mask = NULL;
9226
9227         DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
9228                 dev->data->port_id, tunnel->tunnel_id);
9229
9230         return tunnel;
9231 }
9232
9233 struct tunnel_db_get_tunnel_ctx {
9234         const struct rte_flow_tunnel *app_tunnel;
9235         struct mlx5_flow_tunnel *tunnel;
9236 };
9237
9238 static bool get_tunnel_match(struct rte_eth_dev *dev,
9239                              struct mlx5_flow_tunnel *tunnel, const void *x)
9240 {
9241         const struct tunnel_db_get_tunnel_ctx *ctx = x;
9242
9243         RTE_SET_USED(dev);
9244         return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel,
9245                        sizeof(*ctx->app_tunnel));
9246 }
9247
9248 static void get_tunnel_hit(struct rte_eth_dev *dev,
9249                            struct mlx5_flow_tunnel *tunnel, void *x)
9250 {
9251         /* called under tunnel spinlock protection */
9252         struct tunnel_db_get_tunnel_ctx *ctx = x;
9253
9254         RTE_SET_USED(dev);
9255         tunnel->refctn++;
9256         ctx->tunnel = tunnel;
9257 }
9258
9259 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x)
9260 {
9261         /* called under tunnel spinlock protection */
9262         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
9263         struct tunnel_db_get_tunnel_ctx *ctx = x;
9264
9265         rte_spinlock_unlock(&thub->sl);
9266         ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel);
9267         rte_spinlock_lock(&thub->sl);
9268         if (ctx->tunnel) {
9269                 ctx->tunnel->refctn = 1;
9270                 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain);
9271         }
9272 }
9273
9274
9275 static int
9276 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
9277                      const struct rte_flow_tunnel *app_tunnel,
9278                      struct mlx5_flow_tunnel **tunnel)
9279 {
9280         struct tunnel_db_get_tunnel_ctx ctx = {
9281                 .app_tunnel = app_tunnel,
9282         };
9283
9284         mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit,
9285                                       get_tunnel_miss, &ctx, true);
9286         *tunnel = ctx.tunnel;
9287         return ctx.tunnel ? 0 : -ENOMEM;
9288 }
9289
9290 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
9291 {
9292         struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
9293
9294         if (!thub)
9295                 return;
9296         if (!LIST_EMPTY(&thub->tunnels))
9297                 DRV_LOG(WARNING, "port %u tunnels present", port_id);
9298         mlx5_hlist_destroy(thub->groups);
9299         mlx5_free(thub);
9300 }
9301
9302 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
9303 {
9304         int err;
9305         struct mlx5_flow_tunnel_hub *thub;
9306
9307         thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
9308                            0, SOCKET_ID_ANY);
9309         if (!thub)
9310                 return -ENOMEM;
9311         LIST_INIT(&thub->tunnels);
9312         rte_spinlock_init(&thub->sl);
9313         thub->groups = mlx5_hlist_create("flow groups", 64,
9314                                          false, true, sh,
9315                                          mlx5_flow_tunnel_grp2tbl_create_cb,
9316                                          mlx5_flow_tunnel_grp2tbl_match_cb,
9317                                          mlx5_flow_tunnel_grp2tbl_remove_cb,
9318                                          mlx5_flow_tunnel_grp2tbl_clone_cb,
9319                                         mlx5_flow_tunnel_grp2tbl_clone_free_cb);
9320         if (!thub->groups) {
9321                 err = -rte_errno;
9322                 goto err;
9323         }
9324         sh->tunnel_hub = thub;
9325
9326         return 0;
9327
9328 err:
9329         if (thub->groups)
9330                 mlx5_hlist_destroy(thub->groups);
9331         if (thub)
9332                 mlx5_free(thub);
9333         return err;
9334 }
9335
9336 static inline bool
9337 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
9338                           struct rte_flow_tunnel *tunnel,
9339                           const char *err_msg)
9340 {
9341         err_msg = NULL;
9342         if (!is_tunnel_offload_active(dev)) {
9343                 err_msg = "tunnel offload was not activated";
9344                 goto out;
9345         } else if (!tunnel) {
9346                 err_msg = "no application tunnel";
9347                 goto out;
9348         }
9349
9350         switch (tunnel->type) {
9351         default:
9352                 err_msg = "unsupported tunnel type";
9353                 goto out;
9354         case RTE_FLOW_ITEM_TYPE_VXLAN:
9355         case RTE_FLOW_ITEM_TYPE_GRE:
9356         case RTE_FLOW_ITEM_TYPE_NVGRE:
9357         case RTE_FLOW_ITEM_TYPE_GENEVE:
9358                 break;
9359         }
9360
9361 out:
9362         return !err_msg;
9363 }
9364
9365 static int
9366 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
9367                     struct rte_flow_tunnel *app_tunnel,
9368                     struct rte_flow_action **actions,
9369                     uint32_t *num_of_actions,
9370                     struct rte_flow_error *error)
9371 {
9372         int ret;
9373         struct mlx5_flow_tunnel *tunnel;
9374         const char *err_msg = NULL;
9375         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
9376
9377         if (!verdict)
9378                 return rte_flow_error_set(error, EINVAL,
9379                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
9380                                           err_msg);
9381         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
9382         if (ret < 0) {
9383                 return rte_flow_error_set(error, ret,
9384                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
9385                                           "failed to initialize pmd tunnel");
9386         }
9387         *actions = &tunnel->action;
9388         *num_of_actions = 1;
9389         return 0;
9390 }
9391
9392 static int
9393 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
9394                        struct rte_flow_tunnel *app_tunnel,
9395                        struct rte_flow_item **items,
9396                        uint32_t *num_of_items,
9397                        struct rte_flow_error *error)
9398 {
9399         int ret;
9400         struct mlx5_flow_tunnel *tunnel;
9401         const char *err_msg = NULL;
9402         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
9403
9404         if (!verdict)
9405                 return rte_flow_error_set(error, EINVAL,
9406                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
9407                                           err_msg);
9408         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
9409         if (ret < 0) {
9410                 return rte_flow_error_set(error, ret,
9411                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
9412                                           "failed to initialize pmd tunnel");
9413         }
9414         *items = &tunnel->item;
9415         *num_of_items = 1;
9416         return 0;
9417 }
9418
9419 struct tunnel_db_element_release_ctx {
9420         struct rte_flow_item *items;
9421         struct rte_flow_action *actions;
9422         uint32_t num_elements;
9423         struct rte_flow_error *error;
9424         int ret;
9425 };
9426
9427 static bool
9428 tunnel_element_release_match(struct rte_eth_dev *dev,
9429                              struct mlx5_flow_tunnel *tunnel, const void *x)
9430 {
9431         const struct tunnel_db_element_release_ctx *ctx = x;
9432
9433         RTE_SET_USED(dev);
9434         if (ctx->num_elements != 1)
9435                 return false;
9436         else if (ctx->items)
9437                 return ctx->items == &tunnel->item;
9438         else if (ctx->actions)
9439                 return ctx->actions == &tunnel->action;
9440
9441         return false;
9442 }
9443
9444 static void
9445 tunnel_element_release_hit(struct rte_eth_dev *dev,
9446                            struct mlx5_flow_tunnel *tunnel, void *x)
9447 {
9448         struct tunnel_db_element_release_ctx *ctx = x;
9449         ctx->ret = 0;
9450         if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
9451                 mlx5_flow_tunnel_free(dev, tunnel);
9452 }
9453
9454 static void
9455 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x)
9456 {
9457         struct tunnel_db_element_release_ctx *ctx = x;
9458         RTE_SET_USED(dev);
9459         ctx->ret = rte_flow_error_set(ctx->error, EINVAL,
9460                                       RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
9461                                       "invalid argument");
9462 }
9463
9464 static int
9465 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
9466                        struct rte_flow_item *pmd_items,
9467                        uint32_t num_items, struct rte_flow_error *err)
9468 {
9469         struct tunnel_db_element_release_ctx ctx = {
9470                 .items = pmd_items,
9471                 .actions = NULL,
9472                 .num_elements = num_items,
9473                 .error = err,
9474         };
9475
9476         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
9477                                       tunnel_element_release_hit,
9478                                       tunnel_element_release_miss, &ctx, false);
9479
9480         return ctx.ret;
9481 }
9482
9483 static int
9484 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
9485                          struct rte_flow_action *pmd_actions,
9486                          uint32_t num_actions, struct rte_flow_error *err)
9487 {
9488         struct tunnel_db_element_release_ctx ctx = {
9489                 .items = NULL,
9490                 .actions = pmd_actions,
9491                 .num_elements = num_actions,
9492                 .error = err,
9493         };
9494
9495         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
9496                                       tunnel_element_release_hit,
9497                                       tunnel_element_release_miss, &ctx, false);
9498
9499         return ctx.ret;
9500 }
9501
9502 static int
9503 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
9504                                   struct rte_mbuf *m,
9505                                   struct rte_flow_restore_info *info,
9506                                   struct rte_flow_error *err)
9507 {
9508         uint64_t ol_flags = m->ol_flags;
9509         const struct mlx5_flow_tbl_data_entry *tble;
9510         const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID;
9511
9512         if (!is_tunnel_offload_active(dev)) {
9513                 info->flags = 0;
9514                 return 0;
9515         }
9516
9517         if ((ol_flags & mask) != mask)
9518                 goto err;
9519         tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
9520         if (!tble) {
9521                 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
9522                         dev->data->port_id, m->hash.fdir.hi);
9523                 goto err;
9524         }
9525         MLX5_ASSERT(tble->tunnel);
9526         memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
9527         info->group_id = tble->group_id;
9528         info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
9529                       RTE_FLOW_RESTORE_INFO_GROUP_ID |
9530                       RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
9531
9532         return 0;
9533
9534 err:
9535         return rte_flow_error_set(err, EINVAL,
9536                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9537                                   "failed to get restore info");
9538 }
9539
9540 #else /* HAVE_IBV_FLOW_DV_SUPPORT */
9541 static int
9542 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev,
9543                            __rte_unused struct rte_flow_tunnel *app_tunnel,
9544                            __rte_unused struct rte_flow_action **actions,
9545                            __rte_unused uint32_t *num_of_actions,
9546                            __rte_unused struct rte_flow_error *error)
9547 {
9548         return -ENOTSUP;
9549 }
9550
9551 static int
9552 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev,
9553                        __rte_unused struct rte_flow_tunnel *app_tunnel,
9554                        __rte_unused struct rte_flow_item **items,
9555                        __rte_unused uint32_t *num_of_items,
9556                        __rte_unused struct rte_flow_error *error)
9557 {
9558         return -ENOTSUP;
9559 }
9560
9561 static int
9562 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev,
9563                               __rte_unused struct rte_flow_item *pmd_items,
9564                               __rte_unused uint32_t num_items,
9565                               __rte_unused struct rte_flow_error *err)
9566 {
9567         return -ENOTSUP;
9568 }
9569
9570 static int
9571 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev,
9572                                 __rte_unused struct rte_flow_action *pmd_action,
9573                                 __rte_unused uint32_t num_actions,
9574                                 __rte_unused struct rte_flow_error *err)
9575 {
9576         return -ENOTSUP;
9577 }
9578
9579 static int
9580 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev,
9581                                   __rte_unused struct rte_mbuf *m,
9582                                   __rte_unused struct rte_flow_restore_info *i,
9583                                   __rte_unused struct rte_flow_error *err)
9584 {
9585         return -ENOTSUP;
9586 }
9587
9588 static int
9589 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev,
9590                              __rte_unused struct rte_flow *flow,
9591                              __rte_unused const struct rte_flow_attr *attr,
9592                              __rte_unused const struct rte_flow_action *actions,
9593                              __rte_unused uint32_t flow_idx,
9594                              __rte_unused const struct mlx5_flow_tunnel *tunnel,
9595                              __rte_unused struct tunnel_default_miss_ctx *ctx,
9596                              __rte_unused struct rte_flow_error *error)
9597 {
9598         return -ENOTSUP;
9599 }
9600
9601 static struct mlx5_flow_tunnel *
9602 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev,
9603                     __rte_unused uint32_t id)
9604 {
9605         return NULL;
9606 }
9607
9608 static void
9609 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev,
9610                       __rte_unused struct mlx5_flow_tunnel *tunnel)
9611 {
9612 }
9613
9614 static uint32_t
9615 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev,
9616                                 __rte_unused const struct mlx5_flow_tunnel *t,
9617                                 __rte_unused uint32_t group,
9618                                 __rte_unused uint32_t *table,
9619                                 struct rte_flow_error *error)
9620 {
9621         return rte_flow_error_set(error, ENOTSUP,
9622                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9623                                   "tunnel offload requires DV support");
9624 }
9625
9626 void
9627 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh,
9628                         __rte_unused  uint16_t port_id)
9629 {
9630 }
9631 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
9632
9633 static void
9634 mlx5_dbg__print_pattern(const struct rte_flow_item *item)
9635 {
9636         int ret;
9637         struct rte_flow_error error;
9638
9639         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
9640                 char *item_name;
9641                 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name,
9642                                     sizeof(item_name),
9643                                     (void *)(uintptr_t)item->type, &error);
9644                 if (ret > 0)
9645                         printf("%s ", item_name);
9646                 else
9647                         printf("%d\n", (int)item->type);
9648         }
9649         printf("END\n");
9650 }
9651
9652 static int
9653 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item)
9654 {
9655         const struct rte_flow_item_udp *spec = udp_item->spec;
9656         const struct rte_flow_item_udp *mask = udp_item->mask;
9657         uint16_t udp_dport = 0;
9658
9659         if (spec != NULL) {
9660                 if (!mask)
9661                         mask = &rte_flow_item_udp_mask;
9662                 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port &
9663                                 mask->hdr.dst_port);
9664         }
9665         return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN);
9666 }
9667
9668 static const struct mlx5_flow_expand_node *
9669 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern,
9670                 unsigned int item_idx,
9671                 const struct mlx5_flow_expand_node graph[],
9672                 const struct mlx5_flow_expand_node *node)
9673 {
9674         const struct rte_flow_item *item = pattern + item_idx, *prev_item;
9675         switch (item->type) {
9676         case RTE_FLOW_ITEM_TYPE_VXLAN:
9677                 MLX5_ASSERT(item_idx > 0);
9678                 prev_item = pattern + item_idx - 1;
9679                 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP);
9680                 if (mlx5_flow_is_std_vxlan_port(prev_item))
9681                         return &graph[MLX5_EXPANSION_STD_VXLAN];
9682                 else
9683                         return &graph[MLX5_EXPANSION_L3_VXLAN];
9684                 break;
9685         default:
9686                 return node;
9687         }
9688 }