drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <stdbool.h>
  10 #include <sys/queue.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_ether.h>
  14 #include <ethdev_driver.h>
  15 #include <rte_eal_paging.h>
  16 #include <rte_flow.h>
  17 #include <rte_cycles.h>
  18 #include <rte_flow_driver.h>
  19 #include <rte_malloc.h>
  20 #include <rte_ip.h>
  21
  22 #include <mlx5_glue.h>
  23 #include <mlx5_devx_cmds.h>
  24 #include <mlx5_prm.h>
  25 #include <mlx5_malloc.h>
  26
  27 #include "mlx5_defs.h"
  28 #include "mlx5.h"
  29 #include "mlx5_flow.h"
  30 #include "mlx5_flow_os.h"
  31 #include "mlx5_rx.h"
  32 #include "mlx5_tx.h"
  33 #include "mlx5_common_os.h"
  34 #include "rte_pmd_mlx5.h"
  35
  36 struct tunnel_default_miss_ctx {
  37         uint16_t *queue;
  38         __extension__
  39         union {
  40                 struct rte_flow_action_rss action_rss;
  41                 struct rte_flow_action_queue miss_queue;
  42                 struct rte_flow_action_jump miss_jump;
  43                 uint8_t raw[0];
  44         };
  45 };
  46
  47 static int
  48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
  49                              struct rte_flow *flow,
  50                              const struct rte_flow_attr *attr,
  51                              const struct rte_flow_action *app_actions,
  52                              uint32_t flow_idx,
  53                              const struct mlx5_flow_tunnel *tunnel,
  54                              struct tunnel_default_miss_ctx *ctx,
  55                              struct rte_flow_error *error);
  56 static struct mlx5_flow_tunnel *
  57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
  58 static void
  59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
  60 static uint32_t
  61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
  62                                 const struct mlx5_flow_tunnel *tunnel,
  63                                 uint32_t group, uint32_t *table,
  64                                 struct rte_flow_error *error);
  65
  66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void);
  67 static void mlx5_flow_pop_thread_workspace(void);
  68
  69
  70 /** Device flow drivers. */
  71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  72
  73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  74
  75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  76         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  78         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  79         [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops,
  80 #endif
  81         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  82         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  83 };
  84
  85 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
  86 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
  87         (const int []){ \
  88                 __VA_ARGS__, 0, \
  89         }
  90
  91 /** Node object of input graph for mlx5_flow_expand_rss(). */
  92 struct mlx5_flow_expand_node {
  93         const int *const next;
  94         /**<
  95          * List of next node indexes. Index 0 is interpreted as a terminator.
  96          */
  97         const enum rte_flow_item_type type;
  98         /**< Pattern item type of current node. */
  99         uint64_t rss_types;
 100         /**<
 101          * RSS types bit-field associated with this node
 102          * (see RTE_ETH_RSS_* definitions).
 103          */
 104         uint64_t node_flags;
 105         /**<
 106          *  Bit-fields that define how the node is used in the expansion.
 107          * (see MLX5_EXPANSION_NODE_* definitions).
 108          */
 109 };
 110
 111 /* Optional expand field. The expansion alg will not go deeper. */
 112 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0)
 113
 114 /* The node is not added implicitly as expansion to the flow pattern.
 115  * If the node type does not match the flow pattern item type, the
 116  * expansion alg will go deeper to its next items.
 117  * In the current implementation, the list of next nodes indexes can
 118  * have up to one node with this flag set and it has to be the last
 119  * node index (before the list terminator).
 120  */
 121 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1)
 122
 123 /** Object returned by mlx5_flow_expand_rss(). */
 124 struct mlx5_flow_expand_rss {
 125         uint32_t entries;
 126         /**< Number of entries @p patterns and @p priorities. */
 127         struct {
 128                 struct rte_flow_item *pattern; /**< Expanded pattern array. */
 129                 uint32_t priority; /**< Priority offset for each expansion. */
 130         } entry[];
 131 };
 132
 133 static void
 134 mlx5_dbg__print_pattern(const struct rte_flow_item *item);
 135
 136 static const struct mlx5_flow_expand_node *
 137 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern,
 138                 unsigned int item_idx,
 139                 const struct mlx5_flow_expand_node graph[],
 140                 const struct mlx5_flow_expand_node *node);
 141
 142 static bool
 143 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item)
 144 {
 145         switch (item->type) {
 146         case RTE_FLOW_ITEM_TYPE_ETH:
 147         case RTE_FLOW_ITEM_TYPE_VLAN:
 148         case RTE_FLOW_ITEM_TYPE_IPV4:
 149         case RTE_FLOW_ITEM_TYPE_IPV6:
 150         case RTE_FLOW_ITEM_TYPE_UDP:
 151         case RTE_FLOW_ITEM_TYPE_TCP:
 152         case RTE_FLOW_ITEM_TYPE_ESP:
 153         case RTE_FLOW_ITEM_TYPE_VXLAN:
 154         case RTE_FLOW_ITEM_TYPE_NVGRE:
 155         case RTE_FLOW_ITEM_TYPE_GRE:
 156         case RTE_FLOW_ITEM_TYPE_GENEVE:
 157         case RTE_FLOW_ITEM_TYPE_MPLS:
 158         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 159         case RTE_FLOW_ITEM_TYPE_GRE_KEY:
 160         case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
 161         case RTE_FLOW_ITEM_TYPE_GTP:
 162                 return true;
 163         default:
 164                 break;
 165         }
 166         return false;
 167 }
 168
 169 /**
 170  * Network Service Header (NSH) and its next protocol values
 171  * are described in RFC-8393.
 172  */
 173 static enum rte_flow_item_type
 174 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask)
 175 {
 176         enum rte_flow_item_type type;
 177
 178         switch (proto_mask & proto_spec) {
 179         case 0:
 180                 type = RTE_FLOW_ITEM_TYPE_VOID;
 181                 break;
 182         case RTE_VXLAN_GPE_TYPE_IPV4:
 183                 type = RTE_FLOW_ITEM_TYPE_IPV4;
 184                 break;
 185         case RTE_VXLAN_GPE_TYPE_IPV6:
 186                 type = RTE_VXLAN_GPE_TYPE_IPV6;
 187                 break;
 188         case RTE_VXLAN_GPE_TYPE_ETH:
 189                 type = RTE_FLOW_ITEM_TYPE_ETH;
 190                 break;
 191         default:
 192                 type = RTE_FLOW_ITEM_TYPE_END;
 193         }
 194         return type;
 195 }
 196
 197 static enum rte_flow_item_type
 198 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask)
 199 {
 200         enum rte_flow_item_type type;
 201
 202         switch (proto_mask & proto_spec) {
 203         case 0:
 204                 type = RTE_FLOW_ITEM_TYPE_VOID;
 205                 break;
 206         case IPPROTO_UDP:
 207                 type = RTE_FLOW_ITEM_TYPE_UDP;
 208                 break;
 209         case IPPROTO_TCP:
 210                 type = RTE_FLOW_ITEM_TYPE_TCP;
 211                 break;
 212         case IPPROTO_IPIP:
 213                 type = RTE_FLOW_ITEM_TYPE_IPV4;
 214                 break;
 215         case IPPROTO_IPV6:
 216                 type = RTE_FLOW_ITEM_TYPE_IPV6;
 217                 break;
 218         case IPPROTO_ESP:
 219                 type = RTE_FLOW_ITEM_TYPE_ESP;
 220                 break;
 221         default:
 222                 type = RTE_FLOW_ITEM_TYPE_END;
 223         }
 224         return type;
 225 }
 226
 227 static enum rte_flow_item_type
 228 mlx5_ethertype_to_item_type(rte_be16_t type_spec,
 229                             rte_be16_t type_mask, bool is_tunnel)
 230 {
 231         enum rte_flow_item_type type;
 232
 233         switch (rte_be_to_cpu_16(type_spec & type_mask)) {
 234         case 0:
 235                 type = RTE_FLOW_ITEM_TYPE_VOID;
 236                 break;
 237         case RTE_ETHER_TYPE_TEB:
 238                 type = is_tunnel ?
 239                        RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END;
 240                 break;
 241         case RTE_ETHER_TYPE_VLAN:
 242                 type = !is_tunnel ?
 243                        RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END;
 244                 break;
 245         case RTE_ETHER_TYPE_IPV4:
 246                 type = RTE_FLOW_ITEM_TYPE_IPV4;
 247                 break;
 248         case RTE_ETHER_TYPE_IPV6:
 249                 type = RTE_FLOW_ITEM_TYPE_IPV6;
 250                 break;
 251         default:
 252                 type = RTE_FLOW_ITEM_TYPE_END;
 253         }
 254         return type;
 255 }
 256
 257 static enum rte_flow_item_type
 258 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
 259 {
 260 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld)                              \
 261         do {                                                             \
 262                 const void *m = item->mask;                              \
 263                 const void *s = item->spec;                              \
 264                 mask = m ?                                               \
 265                         ((const struct rte_flow_item_##type *)m)->fld :  \
 266                         rte_flow_item_##type##_mask.fld;                 \
 267                 spec = ((const struct rte_flow_item_##type *)s)->fld;    \
 268         } while (0)
 269
 270         enum rte_flow_item_type ret;
 271         uint16_t spec, mask;
 272
 273         if (item == NULL || item->spec == NULL)
 274                 return RTE_FLOW_ITEM_TYPE_VOID;
 275         switch (item->type) {
 276         case RTE_FLOW_ITEM_TYPE_ETH:
 277                 MLX5_XSET_ITEM_MASK_SPEC(eth, type);
 278                 if (!mask)
 279                         return RTE_FLOW_ITEM_TYPE_VOID;
 280                 ret = mlx5_ethertype_to_item_type(spec, mask, false);
 281                 break;
 282         case RTE_FLOW_ITEM_TYPE_VLAN:
 283                 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type);
 284                 if (!mask)
 285                         return RTE_FLOW_ITEM_TYPE_VOID;
 286                 ret = mlx5_ethertype_to_item_type(spec, mask, false);
 287                 break;
 288         case RTE_FLOW_ITEM_TYPE_IPV4:
 289                 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id);
 290                 if (!mask)
 291                         return RTE_FLOW_ITEM_TYPE_VOID;
 292                 ret = mlx5_inet_proto_to_item_type(spec, mask);
 293                 break;
 294         case RTE_FLOW_ITEM_TYPE_IPV6:
 295                 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto);
 296                 if (!mask)
 297                         return RTE_FLOW_ITEM_TYPE_VOID;
 298                 ret = mlx5_inet_proto_to_item_type(spec, mask);
 299                 break;
 300         case RTE_FLOW_ITEM_TYPE_GENEVE:
 301                 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol);
 302                 ret = mlx5_ethertype_to_item_type(spec, mask, true);
 303                 break;
 304         case RTE_FLOW_ITEM_TYPE_GRE:
 305                 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol);
 306                 ret = mlx5_ethertype_to_item_type(spec, mask, true);
 307                 break;
 308         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 309                 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol);
 310                 ret = mlx5_nsh_proto_to_item_type(spec, mask);
 311                 break;
 312         default:
 313                 ret = RTE_FLOW_ITEM_TYPE_VOID;
 314                 break;
 315         }
 316         return ret;
 317 #undef MLX5_XSET_ITEM_MASK_SPEC
 318 }
 319
 320 static const int *
 321 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[],
 322                 const int *next_node)
 323 {
 324         const struct mlx5_flow_expand_node *node = NULL;
 325         const int *next = next_node;
 326
 327         while (next && *next) {
 328                 /*
 329                  * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT
 330                  * flag set, because they were not found in the flow pattern.
 331                  */
 332                 node = &graph[*next];
 333                 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT))
 334                         break;
 335                 next = node->next;
 336         }
 337         return next;
 338 }
 339
 340 #define MLX5_RSS_EXP_ELT_N 16
 341
 342 /**
 343  * Expand RSS flows into several possible flows according to the RSS hash
 344  * fields requested and the driver capabilities.
 345  *
 346  * @param[out] buf
 347  *   Buffer to store the result expansion.
 348  * @param[in] size
 349  *   Buffer size in bytes. If 0, @p buf can be NULL.
 350  * @param[in] pattern
 351  *   User flow pattern.
 352  * @param[in] types
 353  *   RSS types to expand (see RTE_ETH_RSS_* definitions).
 354  * @param[in] graph
 355  *   Input graph to expand @p pattern according to @p types.
 356  * @param[in] graph_root_index
 357  *   Index of root node in @p graph, typically 0.
 358  *
 359  * @return
 360  *   A positive value representing the size of @p buf in bytes regardless of
 361  *   @p size on success, a negative errno value otherwise and rte_errno is
 362  *   set, the following errors are defined:
 363  *
 364  *   -E2BIG: graph-depth @p graph is too deep.
 365  *   -EINVAL: @p size has not enough space for expanded pattern.
 366  */
 367 static int
 368 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 369                      const struct rte_flow_item *pattern, uint64_t types,
 370                      const struct mlx5_flow_expand_node graph[],
 371                      int graph_root_index)
 372 {
 373         const struct rte_flow_item *item;
 374         const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
 375         const int *next_node;
 376         const int *stack[MLX5_RSS_EXP_ELT_N];
 377         int stack_pos = 0;
 378         struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N];
 379         unsigned int i, item_idx, last_expand_item_idx = 0;
 380         size_t lsize;
 381         size_t user_pattern_size = 0;
 382         void *addr = NULL;
 383         const struct mlx5_flow_expand_node *next = NULL;
 384         struct rte_flow_item missed_item;
 385         int missed = 0;
 386         int elt = 0;
 387         const struct rte_flow_item *last_expand_item = NULL;
 388
 389         memset(&missed_item, 0, sizeof(missed_item));
 390         lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
 391                 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]);
 392         if (lsize > size)
 393                 return -EINVAL;
 394         buf->entry[0].priority = 0;
 395         buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N];
 396         buf->entries = 0;
 397         addr = buf->entry[0].pattern;
 398         for (item = pattern, item_idx = 0;
 399                         item->type != RTE_FLOW_ITEM_TYPE_END;
 400                         item++, item_idx++) {
 401                 if (!mlx5_flow_is_rss_expandable_item(item)) {
 402                         user_pattern_size += sizeof(*item);
 403                         continue;
 404                 }
 405                 last_expand_item = item;
 406                 last_expand_item_idx = item_idx;
 407                 i = 0;
 408                 while (node->next && node->next[i]) {
 409                         next = &graph[node->next[i]];
 410                         if (next->type == item->type)
 411                                 break;
 412                         if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) {
 413                                 node = next;
 414                                 i = 0;
 415                         } else {
 416                                 ++i;
 417                         }
 418                 }
 419                 if (next)
 420                         node = next;
 421                 user_pattern_size += sizeof(*item);
 422         }
 423         user_pattern_size += sizeof(*item); /* Handle END item. */
 424         lsize += user_pattern_size;
 425         if (lsize > size)
 426                 return -EINVAL;
 427         /* Copy the user pattern in the first entry of the buffer. */
 428         rte_memcpy(addr, pattern, user_pattern_size);
 429         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 430         buf->entries = 1;
 431         /* Start expanding. */
 432         memset(flow_items, 0, sizeof(flow_items));
 433         user_pattern_size -= sizeof(*item);
 434         /*
 435          * Check if the last valid item has spec set, need complete pattern,
 436          * and the pattern can be used for expansion.
 437          */
 438         missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item);
 439         if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 440                 /* Item type END indicates expansion is not required. */
 441                 return lsize;
 442         }
 443         if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 444                 next = NULL;
 445                 missed = 1;
 446                 i = 0;
 447                 while (node->next && node->next[i]) {
 448                         next = &graph[node->next[i]];
 449                         if (next->type == missed_item.type) {
 450                                 flow_items[0].type = missed_item.type;
 451                                 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 452                                 break;
 453                         }
 454                         if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) {
 455                                 node = next;
 456                                 i = 0;
 457                         } else {
 458                                 ++i;
 459                         }
 460                         next = NULL;
 461                 }
 462         }
 463         if (next && missed) {
 464                 elt = 2; /* missed item + item end. */
 465                 node = next;
 466                 lsize += elt * sizeof(*item) + user_pattern_size;
 467                 if (lsize > size)
 468                         return -EINVAL;
 469                 if (node->rss_types & types) {
 470                         buf->entry[buf->entries].priority = 1;
 471                         buf->entry[buf->entries].pattern = addr;
 472                         buf->entries++;
 473                         rte_memcpy(addr, buf->entry[0].pattern,
 474                                    user_pattern_size);
 475                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 476                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 477                         addr = (void *)(((uintptr_t)addr) +
 478                                         elt * sizeof(*item));
 479                 }
 480         } else if (last_expand_item != NULL) {
 481                 node = mlx5_flow_expand_rss_adjust_node(pattern,
 482                                 last_expand_item_idx, graph, node);
 483         }
 484         memset(flow_items, 0, sizeof(flow_items));
 485         next_node = mlx5_flow_expand_rss_skip_explicit(graph,
 486                         node->next);
 487         stack[stack_pos] = next_node;
 488         node = next_node ? &graph[*next_node] : NULL;
 489         while (node) {
 490                 flow_items[stack_pos].type = node->type;
 491                 if (node->rss_types & types) {
 492                         size_t n;
 493                         /*
 494                          * compute the number of items to copy from the
 495                          * expansion and copy it.
 496                          * When the stack_pos is 0, there are 1 element in it,
 497                          * plus the addition END item.
 498                          */
 499                         elt = stack_pos + 2;
 500                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 501                         lsize += elt * sizeof(*item) + user_pattern_size;
 502                         if (lsize > size)
 503                                 return -EINVAL;
 504                         n = elt * sizeof(*item);
 505                         buf->entry[buf->entries].priority =
 506                                 stack_pos + 1 + missed;
 507                         buf->entry[buf->entries].pattern = addr;
 508                         buf->entries++;
 509                         rte_memcpy(addr, buf->entry[0].pattern,
 510                                    user_pattern_size);
 511                         addr = (void *)(((uintptr_t)addr) +
 512                                         user_pattern_size);
 513                         rte_memcpy(addr, &missed_item,
 514                                    missed * sizeof(*item));
 515                         addr = (void *)(((uintptr_t)addr) +
 516                                 missed * sizeof(*item));
 517                         rte_memcpy(addr, flow_items, n);
 518                         addr = (void *)(((uintptr_t)addr) + n);
 519                 }
 520                 /* Go deeper. */
 521                 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) &&
 522                                 node->next) {
 523                         next_node = mlx5_flow_expand_rss_skip_explicit(graph,
 524                                         node->next);
 525                         if (stack_pos++ == MLX5_RSS_EXP_ELT_N) {
 526                                 rte_errno = E2BIG;
 527                                 return -rte_errno;
 528                         }
 529                         stack[stack_pos] = next_node;
 530                 } else if (*(next_node + 1)) {
 531                         /* Follow up with the next possibility. */
 532                         next_node = mlx5_flow_expand_rss_skip_explicit(graph,
 533                                         ++next_node);
 534                 } else if (!stack_pos) {
 535                         /*
 536                          * Completing the traverse over the different paths.
 537                          * The next_node is advanced to the terminator.
 538                          */
 539                         ++next_node;
 540                 } else {
 541                         /* Move to the next path. */
 542                         while (stack_pos) {
 543                                 next_node = stack[--stack_pos];
 544                                 next_node++;
 545                                 if (*next_node)
 546                                         break;
 547                         }
 548                         next_node = mlx5_flow_expand_rss_skip_explicit(graph,
 549                                         next_node);
 550                         stack[stack_pos] = next_node;
 551                 }
 552                 node = next_node && *next_node ? &graph[*next_node] : NULL;
 553         };
 554         return lsize;
 555 }
 556
 557 enum mlx5_expansion {
 558         MLX5_EXPANSION_ROOT,
 559         MLX5_EXPANSION_ROOT_OUTER,
 560         MLX5_EXPANSION_OUTER_ETH,
 561         MLX5_EXPANSION_OUTER_VLAN,
 562         MLX5_EXPANSION_OUTER_IPV4,
 563         MLX5_EXPANSION_OUTER_IPV4_UDP,
 564         MLX5_EXPANSION_OUTER_IPV4_TCP,
 565         MLX5_EXPANSION_OUTER_IPV4_ESP,
 566         MLX5_EXPANSION_OUTER_IPV6,
 567         MLX5_EXPANSION_OUTER_IPV6_UDP,
 568         MLX5_EXPANSION_OUTER_IPV6_TCP,
 569         MLX5_EXPANSION_OUTER_IPV6_ESP,
 570         MLX5_EXPANSION_VXLAN,
 571         MLX5_EXPANSION_STD_VXLAN,
 572         MLX5_EXPANSION_L3_VXLAN,
 573         MLX5_EXPANSION_VXLAN_GPE,
 574         MLX5_EXPANSION_GRE,
 575         MLX5_EXPANSION_NVGRE,
 576         MLX5_EXPANSION_GRE_KEY,
 577         MLX5_EXPANSION_MPLS,
 578         MLX5_EXPANSION_ETH,
 579         MLX5_EXPANSION_VLAN,
 580         MLX5_EXPANSION_IPV4,
 581         MLX5_EXPANSION_IPV4_UDP,
 582         MLX5_EXPANSION_IPV4_TCP,
 583         MLX5_EXPANSION_IPV4_ESP,
 584         MLX5_EXPANSION_IPV6,
 585         MLX5_EXPANSION_IPV6_UDP,
 586         MLX5_EXPANSION_IPV6_TCP,
 587         MLX5_EXPANSION_IPV6_ESP,
 588         MLX5_EXPANSION_IPV6_FRAG_EXT,
 589         MLX5_EXPANSION_GTP,
 590         MLX5_EXPANSION_GENEVE,
 591 };
 592
 593 /** Supported expansion of items. */
 594 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 595         [MLX5_EXPANSION_ROOT] = {
 596                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 597                                                   MLX5_EXPANSION_IPV4,
 598                                                   MLX5_EXPANSION_IPV6),
 599                 .type = RTE_FLOW_ITEM_TYPE_END,
 600         },
 601         [MLX5_EXPANSION_ROOT_OUTER] = {
 602                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 603                                                   MLX5_EXPANSION_OUTER_IPV4,
 604                                                   MLX5_EXPANSION_OUTER_IPV6),
 605                 .type = RTE_FLOW_ITEM_TYPE_END,
 606         },
 607         [MLX5_EXPANSION_OUTER_ETH] = {
 608                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 609                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 610                 .rss_types = 0,
 611         },
 612         [MLX5_EXPANSION_OUTER_VLAN] = {
 613                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 614                                                   MLX5_EXPANSION_OUTER_IPV6),
 615                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 616                 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
 617         },
 618         [MLX5_EXPANSION_OUTER_IPV4] = {
 619                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 620                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 621                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 622                          MLX5_EXPANSION_OUTER_IPV4_ESP,
 623                          MLX5_EXPANSION_GRE,
 624                          MLX5_EXPANSION_NVGRE,
 625                          MLX5_EXPANSION_IPV4,
 626                          MLX5_EXPANSION_IPV6),
 627                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 628                 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 |
 629                         RTE_ETH_RSS_NONFRAG_IPV4_OTHER,
 630         },
 631         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 632                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 633                                                   MLX5_EXPANSION_VXLAN_GPE,
 634                                                   MLX5_EXPANSION_MPLS,
 635                                                   MLX5_EXPANSION_GENEVE,
 636                                                   MLX5_EXPANSION_GTP),
 637                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 638                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP,
 639         },
 640         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 641                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 642                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP,
 643         },
 644         [MLX5_EXPANSION_OUTER_IPV4_ESP] = {
 645                 .type = RTE_FLOW_ITEM_TYPE_ESP,
 646                 .rss_types = RTE_ETH_RSS_ESP,
 647         },
 648         [MLX5_EXPANSION_OUTER_IPV6] = {
 649                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 650                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 651                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 652                          MLX5_EXPANSION_OUTER_IPV6_ESP,
 653                          MLX5_EXPANSION_IPV4,
 654                          MLX5_EXPANSION_IPV6,
 655                          MLX5_EXPANSION_GRE,
 656                          MLX5_EXPANSION_NVGRE),
 657                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 658                 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 |
 659                         RTE_ETH_RSS_NONFRAG_IPV6_OTHER,
 660         },
 661         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 662                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 663                                                   MLX5_EXPANSION_VXLAN_GPE,
 664                                                   MLX5_EXPANSION_MPLS,
 665                                                   MLX5_EXPANSION_GENEVE,
 666                                                   MLX5_EXPANSION_GTP),
 667                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 668                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP,
 669         },
 670         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 671                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 672                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP,
 673         },
 674         [MLX5_EXPANSION_OUTER_IPV6_ESP] = {
 675                 .type = RTE_FLOW_ITEM_TYPE_ESP,
 676                 .rss_types = RTE_ETH_RSS_ESP,
 677         },
 678         [MLX5_EXPANSION_VXLAN] = {
 679                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 680                                                   MLX5_EXPANSION_IPV4,
 681                                                   MLX5_EXPANSION_IPV6),
 682                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 683         },
 684         [MLX5_EXPANSION_STD_VXLAN] = {
 685                         .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
 686                                         .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 687         },
 688         [MLX5_EXPANSION_L3_VXLAN] = {
 689                         .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 690                                         MLX5_EXPANSION_IPV6),
 691                                         .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 692         },
 693         [MLX5_EXPANSION_VXLAN_GPE] = {
 694                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 695                                                   MLX5_EXPANSION_IPV4,
 696                                                   MLX5_EXPANSION_IPV6),
 697                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 698         },
 699         [MLX5_EXPANSION_GRE] = {
 700                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 701                                                   MLX5_EXPANSION_IPV4,
 702                                                   MLX5_EXPANSION_IPV6,
 703                                                   MLX5_EXPANSION_GRE_KEY,
 704                                                   MLX5_EXPANSION_MPLS),
 705                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 706         },
 707         [MLX5_EXPANSION_GRE_KEY] = {
 708                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 709                                                   MLX5_EXPANSION_IPV6,
 710                                                   MLX5_EXPANSION_MPLS),
 711                 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY,
 712                 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
 713         },
 714         [MLX5_EXPANSION_NVGRE] = {
 715                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
 716                 .type = RTE_FLOW_ITEM_TYPE_NVGRE,
 717         },
 718         [MLX5_EXPANSION_MPLS] = {
 719                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 720                                                   MLX5_EXPANSION_IPV6,
 721                                                   MLX5_EXPANSION_ETH),
 722                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 723                 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
 724         },
 725         [MLX5_EXPANSION_ETH] = {
 726                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 727                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 728         },
 729         [MLX5_EXPANSION_VLAN] = {
 730                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 731                                                   MLX5_EXPANSION_IPV6),
 732                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 733                 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
 734         },
 735         [MLX5_EXPANSION_IPV4] = {
 736                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 737                                                   MLX5_EXPANSION_IPV4_TCP,
 738                                                   MLX5_EXPANSION_IPV4_ESP),
 739                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 740                 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 |
 741                         RTE_ETH_RSS_NONFRAG_IPV4_OTHER,
 742         },
 743         [MLX5_EXPANSION_IPV4_UDP] = {
 744                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 745                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP,
 746         },
 747         [MLX5_EXPANSION_IPV4_TCP] = {
 748                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 749                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP,
 750         },
 751         [MLX5_EXPANSION_IPV4_ESP] = {
 752                 .type = RTE_FLOW_ITEM_TYPE_ESP,
 753                 .rss_types = RTE_ETH_RSS_ESP,
 754         },
 755         [MLX5_EXPANSION_IPV6] = {
 756                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 757                                                   MLX5_EXPANSION_IPV6_TCP,
 758                                                   MLX5_EXPANSION_IPV6_ESP,
 759                                                   MLX5_EXPANSION_IPV6_FRAG_EXT),
 760                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 761                 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 |
 762                         RTE_ETH_RSS_NONFRAG_IPV6_OTHER,
 763         },
 764         [MLX5_EXPANSION_IPV6_UDP] = {
 765                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 766                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP,
 767         },
 768         [MLX5_EXPANSION_IPV6_TCP] = {
 769                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 770                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP,
 771         },
 772         [MLX5_EXPANSION_IPV6_ESP] = {
 773                 .type = RTE_FLOW_ITEM_TYPE_ESP,
 774                 .rss_types = RTE_ETH_RSS_ESP,
 775         },
 776         [MLX5_EXPANSION_IPV6_FRAG_EXT] = {
 777                 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT,
 778         },
 779         [MLX5_EXPANSION_GTP] = {
 780                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 781                                                   MLX5_EXPANSION_IPV6),
 782                 .type = RTE_FLOW_ITEM_TYPE_GTP,
 783         },
 784         [MLX5_EXPANSION_GENEVE] = {
 785                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 786                                                   MLX5_EXPANSION_IPV4,
 787                                                   MLX5_EXPANSION_IPV6),
 788                 .type = RTE_FLOW_ITEM_TYPE_GENEVE,
 789         },
 790 };
 791
 792 static struct rte_flow_action_handle *
 793 mlx5_action_handle_create(struct rte_eth_dev *dev,
 794                           const struct rte_flow_indir_action_conf *conf,
 795                           const struct rte_flow_action *action,
 796                           struct rte_flow_error *error);
 797 static int mlx5_action_handle_destroy
 798                                 (struct rte_eth_dev *dev,
 799                                  struct rte_flow_action_handle *handle,
 800                                  struct rte_flow_error *error);
 801 static int mlx5_action_handle_update
 802                                 (struct rte_eth_dev *dev,
 803                                  struct rte_flow_action_handle *handle,
 804                                  const void *update,
 805                                  struct rte_flow_error *error);
 806 static int mlx5_action_handle_query
 807                                 (struct rte_eth_dev *dev,
 808                                  const struct rte_flow_action_handle *handle,
 809                                  void *data,
 810                                  struct rte_flow_error *error);
 811 static int
 812 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
 813                     struct rte_flow_tunnel *app_tunnel,
 814                     struct rte_flow_action **actions,
 815                     uint32_t *num_of_actions,
 816                     struct rte_flow_error *error);
 817 static int
 818 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
 819                        struct rte_flow_tunnel *app_tunnel,
 820                        struct rte_flow_item **items,
 821                        uint32_t *num_of_items,
 822                        struct rte_flow_error *error);
 823 static int
 824 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
 825                               struct rte_flow_item *pmd_items,
 826                               uint32_t num_items, struct rte_flow_error *err);
 827 static int
 828 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
 829                                 struct rte_flow_action *pmd_actions,
 830                                 uint32_t num_actions,
 831                                 struct rte_flow_error *err);
 832 static int
 833 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
 834                                   struct rte_mbuf *m,
 835                                   struct rte_flow_restore_info *info,
 836                                   struct rte_flow_error *err);
 837 static struct rte_flow_item_flex_handle *
 838 mlx5_flow_flex_item_create(struct rte_eth_dev *dev,
 839                            const struct rte_flow_item_flex_conf *conf,
 840                            struct rte_flow_error *error);
 841 static int
 842 mlx5_flow_flex_item_release(struct rte_eth_dev *dev,
 843                             const struct rte_flow_item_flex_handle *handle,
 844                             struct rte_flow_error *error);
 845 static int
 846 mlx5_flow_info_get(struct rte_eth_dev *dev,
 847                    struct rte_flow_port_info *port_info,
 848                    struct rte_flow_queue_info *queue_info,
 849                    struct rte_flow_error *error);
 850 static int
 851 mlx5_flow_port_configure(struct rte_eth_dev *dev,
 852                          const struct rte_flow_port_attr *port_attr,
 853                          uint16_t nb_queue,
 854                          const struct rte_flow_queue_attr *queue_attr[],
 855                          struct rte_flow_error *err);
 856
 857 static struct rte_flow_pattern_template *
 858 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev,
 859                 const struct rte_flow_pattern_template_attr *attr,
 860                 const struct rte_flow_item items[],
 861                 struct rte_flow_error *error);
 862
 863 static int
 864 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev,
 865                                    struct rte_flow_pattern_template *template,
 866                                    struct rte_flow_error *error);
 867 static struct rte_flow_actions_template *
 868 mlx5_flow_actions_template_create(struct rte_eth_dev *dev,
 869                         const struct rte_flow_actions_template_attr *attr,
 870                         const struct rte_flow_action actions[],
 871                         const struct rte_flow_action masks[],
 872                         struct rte_flow_error *error);
 873 static int
 874 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev,
 875                                    struct rte_flow_actions_template *template,
 876                                    struct rte_flow_error *error);
 877
 878 static struct rte_flow_template_table *
 879 mlx5_flow_table_create(struct rte_eth_dev *dev,
 880                        const struct rte_flow_template_table_attr *attr,
 881                        struct rte_flow_pattern_template *item_templates[],
 882                        uint8_t nb_item_templates,
 883                        struct rte_flow_actions_template *action_templates[],
 884                        uint8_t nb_action_templates,
 885                        struct rte_flow_error *error);
 886 static int
 887 mlx5_flow_table_destroy(struct rte_eth_dev *dev,
 888                         struct rte_flow_template_table *table,
 889                         struct rte_flow_error *error);
 890 static struct rte_flow *
 891 mlx5_flow_async_flow_create(struct rte_eth_dev *dev,
 892                             uint32_t queue,
 893                             const struct rte_flow_op_attr *attr,
 894                             struct rte_flow_template_table *table,
 895                             const struct rte_flow_item items[],
 896                             uint8_t pattern_template_index,
 897                             const struct rte_flow_action actions[],
 898                             uint8_t action_template_index,
 899                             void *user_data,
 900                             struct rte_flow_error *error);
 901 static int
 902 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev,
 903                              uint32_t queue,
 904                              const struct rte_flow_op_attr *attr,
 905                              struct rte_flow *flow,
 906                              void *user_data,
 907                              struct rte_flow_error *error);
 908 static int
 909 mlx5_flow_pull(struct rte_eth_dev *dev,
 910                uint32_t queue,
 911                struct rte_flow_op_result res[],
 912                uint16_t n_res,
 913                struct rte_flow_error *error);
 914 static int
 915 mlx5_flow_push(struct rte_eth_dev *dev,
 916                uint32_t queue,
 917                struct rte_flow_error *error);
 918
 919 static struct rte_flow_action_handle *
 920 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue,
 921                                  const struct rte_flow_op_attr *attr,
 922                                  const struct rte_flow_indir_action_conf *conf,
 923                                  const struct rte_flow_action *action,
 924                                  void *user_data,
 925                                  struct rte_flow_error *error);
 926
 927 static int
 928 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue,
 929                                  const struct rte_flow_op_attr *attr,
 930                                  struct rte_flow_action_handle *handle,
 931                                  const void *update,
 932                                  void *user_data,
 933                                  struct rte_flow_error *error);
 934
 935 static int
 936 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue,
 937                                   const struct rte_flow_op_attr *attr,
 938                                   struct rte_flow_action_handle *handle,
 939                                   void *user_data,
 940                                   struct rte_flow_error *error);
 941
 942 static const struct rte_flow_ops mlx5_flow_ops = {
 943         .validate = mlx5_flow_validate,
 944         .create = mlx5_flow_create,
 945         .destroy = mlx5_flow_destroy,
 946         .flush = mlx5_flow_flush,
 947         .isolate = mlx5_flow_isolate,
 948         .query = mlx5_flow_query,
 949         .dev_dump = mlx5_flow_dev_dump,
 950         .get_aged_flows = mlx5_flow_get_aged_flows,
 951         .action_handle_create = mlx5_action_handle_create,
 952         .action_handle_destroy = mlx5_action_handle_destroy,
 953         .action_handle_update = mlx5_action_handle_update,
 954         .action_handle_query = mlx5_action_handle_query,
 955         .tunnel_decap_set = mlx5_flow_tunnel_decap_set,
 956         .tunnel_match = mlx5_flow_tunnel_match,
 957         .tunnel_action_decap_release = mlx5_flow_tunnel_action_release,
 958         .tunnel_item_release = mlx5_flow_tunnel_item_release,
 959         .get_restore_info = mlx5_flow_tunnel_get_restore_info,
 960         .flex_item_create = mlx5_flow_flex_item_create,
 961         .flex_item_release = mlx5_flow_flex_item_release,
 962         .info_get = mlx5_flow_info_get,
 963         .configure = mlx5_flow_port_configure,
 964         .pattern_template_create = mlx5_flow_pattern_template_create,
 965         .pattern_template_destroy = mlx5_flow_pattern_template_destroy,
 966         .actions_template_create = mlx5_flow_actions_template_create,
 967         .actions_template_destroy = mlx5_flow_actions_template_destroy,
 968         .template_table_create = mlx5_flow_table_create,
 969         .template_table_destroy = mlx5_flow_table_destroy,
 970         .async_create = mlx5_flow_async_flow_create,
 971         .async_destroy = mlx5_flow_async_flow_destroy,
 972         .pull = mlx5_flow_pull,
 973         .push = mlx5_flow_push,
 974         .async_action_handle_create = mlx5_flow_async_action_handle_create,
 975         .async_action_handle_update = mlx5_flow_async_action_handle_update,
 976         .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy,
 977 };
 978
 979 /* Tunnel information. */
 980 struct mlx5_flow_tunnel_info {
 981         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 982         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 983 };
 984
 985 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 986         {
 987                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 988                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 989         },
 990         {
 991                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 992                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 993         },
 994         {
 995                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 996                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 997         },
 998         {
 999                 .tunnel = MLX5_FLOW_LAYER_GRE,
1000                 .ptype = RTE_PTYPE_TUNNEL_GRE,
1001         },
1002         {
1003                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
1004                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
1005         },
1006         {
1007                 .tunnel = MLX5_FLOW_LAYER_MPLS,
1008                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
1009         },
1010         {
1011                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
1012                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
1013         },
1014         {
1015                 .tunnel = MLX5_FLOW_LAYER_IPIP,
1016                 .ptype = RTE_PTYPE_TUNNEL_IP,
1017         },
1018         {
1019                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
1020                 .ptype = RTE_PTYPE_TUNNEL_IP,
1021         },
1022         {
1023                 .tunnel = MLX5_FLOW_LAYER_GTP,
1024                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
1025         },
1026 };
1027
1028
1029
1030 /**
1031  * Translate tag ID to register.
1032  *
1033  * @param[in] dev
1034  *   Pointer to the Ethernet device structure.
1035  * @param[in] feature
1036  *   The feature that request the register.
1037  * @param[in] id
1038  *   The request register ID.
1039  * @param[out] error
1040  *   Error description in case of any.
1041  *
1042  * @return
1043  *   The request register on success, a negative errno
1044  *   value otherwise and rte_errno is set.
1045  */
1046 int
1047 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
1048                      enum mlx5_feature_name feature,
1049                      uint32_t id,
1050                      struct rte_flow_error *error)
1051 {
1052         struct mlx5_priv *priv = dev->data->dev_private;
1053         struct mlx5_sh_config *config = &priv->sh->config;
1054         enum modify_reg start_reg;
1055         bool skip_mtr_reg = false;
1056
1057         switch (feature) {
1058         case MLX5_HAIRPIN_RX:
1059                 return REG_B;
1060         case MLX5_HAIRPIN_TX:
1061                 return REG_A;
1062         case MLX5_METADATA_RX:
1063                 switch (config->dv_xmeta_en) {
1064                 case MLX5_XMETA_MODE_LEGACY:
1065                         return REG_B;
1066                 case MLX5_XMETA_MODE_META16:
1067                         return REG_C_0;
1068                 case MLX5_XMETA_MODE_META32:
1069                         return REG_C_1;
1070                 }
1071                 break;
1072         case MLX5_METADATA_TX:
1073                 return REG_A;
1074         case MLX5_METADATA_FDB:
1075                 switch (config->dv_xmeta_en) {
1076                 case MLX5_XMETA_MODE_LEGACY:
1077                         return REG_NON;
1078                 case MLX5_XMETA_MODE_META16:
1079                         return REG_C_0;
1080                 case MLX5_XMETA_MODE_META32:
1081                         return REG_C_1;
1082                 }
1083                 break;
1084         case MLX5_FLOW_MARK:
1085                 switch (config->dv_xmeta_en) {
1086                 case MLX5_XMETA_MODE_LEGACY:
1087                         return REG_NON;
1088                 case MLX5_XMETA_MODE_META16:
1089                         return REG_C_1;
1090                 case MLX5_XMETA_MODE_META32:
1091                         return REG_C_0;
1092                 }
1093                 break;
1094         case MLX5_MTR_ID:
1095                 /*
1096                  * If meter color and meter id share one register, flow match
1097                  * should use the meter color register for match.
1098                  */
1099                 if (priv->mtr_reg_share)
1100                         return priv->mtr_color_reg;
1101                 else
1102                         return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
1103                                REG_C_3;
1104         case MLX5_MTR_COLOR:
1105         case MLX5_ASO_FLOW_HIT:
1106         case MLX5_ASO_CONNTRACK:
1107         case MLX5_SAMPLE_ID:
1108                 /* All features use the same REG_C. */
1109                 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
1110                 return priv->mtr_color_reg;
1111         case MLX5_COPY_MARK:
1112                 /*
1113                  * Metadata COPY_MARK register using is in meter suffix sub
1114                  * flow while with meter. It's safe to share the same register.
1115                  */
1116                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
1117         case MLX5_APP_TAG:
1118                 /*
1119                  * If meter is enable, it will engage the register for color
1120                  * match and flow match. If meter color match is not using the
1121                  * REG_C_2, need to skip the REG_C_x be used by meter color
1122                  * match.
1123                  * If meter is disable, free to use all available registers.
1124                  */
1125                 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
1126                             (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
1127                 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
1128                 if (id > (uint32_t)(REG_C_7 - start_reg))
1129                         return rte_flow_error_set(error, EINVAL,
1130                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1131                                                   NULL, "invalid tag id");
1132                 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
1133                         return rte_flow_error_set(error, ENOTSUP,
1134                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1135                                                   NULL, "unsupported tag id");
1136                 /*
1137                  * This case means meter is using the REG_C_x great than 2.
1138                  * Take care not to conflict with meter color REG_C_x.
1139                  * If the available index REG_C_y >= REG_C_x, skip the
1140                  * color register.
1141                  */
1142                 if (skip_mtr_reg && priv->sh->flow_mreg_c
1143                     [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
1144                         if (id >= (uint32_t)(REG_C_7 - start_reg))
1145                                 return rte_flow_error_set(error, EINVAL,
1146                                                        RTE_FLOW_ERROR_TYPE_ITEM,
1147                                                         NULL, "invalid tag id");
1148                         if (priv->sh->flow_mreg_c
1149                             [id + 1 + start_reg - REG_C_0] != REG_NON)
1150                                 return priv->sh->flow_mreg_c
1151                                                [id + 1 + start_reg - REG_C_0];
1152                         return rte_flow_error_set(error, ENOTSUP,
1153                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1154                                                   NULL, "unsupported tag id");
1155                 }
1156                 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0];
1157         }
1158         MLX5_ASSERT(false);
1159         return rte_flow_error_set(error, EINVAL,
1160                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1161                                   NULL, "invalid feature name");
1162 }
1163
1164 /**
1165  * Check extensive flow metadata register support.
1166  *
1167  * @param dev
1168  *   Pointer to rte_eth_dev structure.
1169  *
1170  * @return
1171  *   True if device supports extensive flow metadata register, otherwise false.
1172  */
1173 bool
1174 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
1175 {
1176         struct mlx5_priv *priv = dev->data->dev_private;
1177
1178         /*
1179          * Having available reg_c can be regarded inclusively as supporting
1180          * extensive flow metadata register, which could mean,
1181          * - metadata register copy action by modify header.
1182          * - 16 modify header actions is supported.
1183          * - reg_c's are preserved across different domain (FDB and NIC) on
1184          *   packet loopback by flow lookup miss.
1185          */
1186         return priv->sh->flow_mreg_c[2] != REG_NON;
1187 }
1188
1189 /**
1190  * Get the lowest priority.
1191  *
1192  * @param[in] dev
1193  *   Pointer to the Ethernet device structure.
1194  * @param[in] attributes
1195  *   Pointer to device flow rule attributes.
1196  *
1197  * @return
1198  *   The value of lowest priority of flow.
1199  */
1200 uint32_t
1201 mlx5_get_lowest_priority(struct rte_eth_dev *dev,
1202                           const struct rte_flow_attr *attr)
1203 {
1204         struct mlx5_priv *priv = dev->data->dev_private;
1205
1206         if (!attr->group && !attr->transfer)
1207                 return priv->sh->flow_max_priority - 2;
1208         return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1;
1209 }
1210
1211 /**
1212  * Calculate matcher priority of the flow.
1213  *
1214  * @param[in] dev
1215  *   Pointer to the Ethernet device structure.
1216  * @param[in] attr
1217  *   Pointer to device flow rule attributes.
1218  * @param[in] subpriority
1219  *   The priority based on the items.
1220  * @param[in] external
1221  *   Flow is user flow.
1222  * @return
1223  *   The matcher priority of the flow.
1224  */
1225 uint16_t
1226 mlx5_get_matcher_priority(struct rte_eth_dev *dev,
1227                           const struct rte_flow_attr *attr,
1228                           uint32_t subpriority, bool external)
1229 {
1230         uint16_t priority = (uint16_t)attr->priority;
1231         struct mlx5_priv *priv = dev->data->dev_private;
1232
1233         if (!attr->group && !attr->transfer) {
1234                 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
1235                         priority = priv->sh->flow_max_priority - 1;
1236                 return mlx5_os_flow_adjust_priority(dev, priority, subpriority);
1237         } else if (!external && attr->transfer && attr->group == 0 &&
1238                    attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) {
1239                 return (priv->sh->flow_max_priority - 1) * 3;
1240         }
1241         if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
1242                 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO;
1243         return priority * 3 + subpriority;
1244 }
1245
1246 /**
1247  * Verify the @p item specifications (spec, last, mask) are compatible with the
1248  * NIC capabilities.
1249  *
1250  * @param[in] item
1251  *   Item specification.
1252  * @param[in] mask
1253  *   @p item->mask or flow default bit-masks.
1254  * @param[in] nic_mask
1255  *   Bit-masks covering supported fields by the NIC to compare with user mask.
1256  * @param[in] size
1257  *   Bit-masks size in bytes.
1258  * @param[in] range_accepted
1259  *   True if range of values is accepted for specific fields, false otherwise.
1260  * @param[out] error
1261  *   Pointer to error structure.
1262  *
1263  * @return
1264  *   0 on success, a negative errno value otherwise and rte_errno is set.
1265  */
1266 int
1267 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
1268                           const uint8_t *mask,
1269                           const uint8_t *nic_mask,
1270                           unsigned int size,
1271                           bool range_accepted,
1272                           struct rte_flow_error *error)
1273 {
1274         unsigned int i;
1275
1276         MLX5_ASSERT(nic_mask);
1277         for (i = 0; i < size; ++i)
1278                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
1279                         return rte_flow_error_set(error, ENOTSUP,
1280                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1281                                                   item,
1282                                                   "mask enables non supported"
1283                                                   " bits");
1284         if (!item->spec && (item->mask || item->last))
1285                 return rte_flow_error_set(error, EINVAL,
1286                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1287                                           "mask/last without a spec is not"
1288                                           " supported");
1289         if (item->spec && item->last && !range_accepted) {
1290                 uint8_t spec[size];
1291                 uint8_t last[size];
1292                 unsigned int i;
1293                 int ret;
1294
1295                 for (i = 0; i < size; ++i) {
1296                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
1297                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
1298                 }
1299                 ret = memcmp(spec, last, size);
1300                 if (ret != 0)
1301                         return rte_flow_error_set(error, EINVAL,
1302                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1303                                                   item,
1304                                                   "range is not valid");
1305         }
1306         return 0;
1307 }
1308
1309 /**
1310  * Adjust the hash fields according to the @p flow information.
1311  *
1312  * @param[in] dev_flow.
1313  *   Pointer to the mlx5_flow.
1314  * @param[in] tunnel
1315  *   1 when the hash field is for a tunnel item.
1316  * @param[in] layer_types
1317  *   RTE_ETH_RSS_* types.
1318  * @param[in] hash_fields
1319  *   Item hash fields.
1320  *
1321  * @return
1322  *   The hash fields that should be used.
1323  */
1324 uint64_t
1325 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
1326                             int tunnel __rte_unused, uint64_t layer_types,
1327                             uint64_t hash_fields)
1328 {
1329 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1330         int rss_request_inner = rss_desc->level >= 2;
1331
1332         /* Check RSS hash level for tunnel. */
1333         if (tunnel && rss_request_inner)
1334                 hash_fields |= IBV_RX_HASH_INNER;
1335         else if (tunnel || rss_request_inner)
1336                 return 0;
1337 #endif
1338         /* Check if requested layer matches RSS hash fields. */
1339         if (!(rss_desc->types & layer_types))
1340                 return 0;
1341         return hash_fields;
1342 }
1343
1344 /**
1345  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
1346  * if several tunnel rules are used on this queue, the tunnel ptype will be
1347  * cleared.
1348  *
1349  * @param rxq_ctrl
1350  *   Rx queue to update.
1351  */
1352 static void
1353 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
1354 {
1355         unsigned int i;
1356         uint32_t tunnel_ptype = 0;
1357
1358         /* Look up for the ptype to use. */
1359         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
1360                 if (!rxq_ctrl->flow_tunnels_n[i])
1361                         continue;
1362                 if (!tunnel_ptype) {
1363                         tunnel_ptype = tunnels_info[i].ptype;
1364                 } else {
1365                         tunnel_ptype = 0;
1366                         break;
1367                 }
1368         }
1369         rxq_ctrl->rxq.tunnel = tunnel_ptype;
1370 }
1371
1372 /**
1373  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device
1374  * flow.
1375  *
1376  * @param[in] dev
1377  *   Pointer to the Ethernet device structure.
1378  * @param[in] dev_handle
1379  *   Pointer to device flow handle structure.
1380  */
1381 void
1382 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
1383                        struct mlx5_flow_handle *dev_handle)
1384 {
1385         struct mlx5_priv *priv = dev->data->dev_private;
1386         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1387         struct mlx5_ind_table_obj *ind_tbl = NULL;
1388         unsigned int i;
1389
1390         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1391                 struct mlx5_hrxq *hrxq;
1392
1393                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1394                               dev_handle->rix_hrxq);
1395                 if (hrxq)
1396                         ind_tbl = hrxq->ind_table;
1397         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1398                 struct mlx5_shared_action_rss *shared_rss;
1399
1400                 shared_rss = mlx5_ipool_get
1401                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1402                          dev_handle->rix_srss);
1403                 if (shared_rss)
1404                         ind_tbl = shared_rss->ind_tbl;
1405         }
1406         if (!ind_tbl)
1407                 return;
1408         for (i = 0; i != ind_tbl->queues_n; ++i) {
1409                 int idx = ind_tbl->queues[i];
1410                 struct mlx5_rxq_ctrl *rxq_ctrl;
1411
1412                 if (mlx5_is_external_rxq(dev, idx))
1413                         continue;
1414                 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx);
1415                 MLX5_ASSERT(rxq_ctrl != NULL);
1416                 if (rxq_ctrl == NULL)
1417                         continue;
1418                 /*
1419                  * To support metadata register copy on Tx loopback,
1420                  * this must be always enabled (metadata may arive
1421                  * from other port - not from local flows only.
1422                  */
1423                 if (tunnel) {
1424                         unsigned int j;
1425
1426                         /* Increase the counter matching the flow. */
1427                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1428                                 if ((tunnels_info[j].tunnel &
1429                                      dev_handle->layers) ==
1430                                     tunnels_info[j].tunnel) {
1431                                         rxq_ctrl->flow_tunnels_n[j]++;
1432                                         break;
1433                                 }
1434                         }
1435                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1436                 }
1437         }
1438 }
1439
1440 static void
1441 flow_rxq_mark_flag_set(struct rte_eth_dev *dev)
1442 {
1443         struct mlx5_priv *priv = dev->data->dev_private;
1444         struct mlx5_rxq_ctrl *rxq_ctrl;
1445
1446         if (priv->mark_enabled)
1447                 return;
1448         LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) {
1449                 rxq_ctrl->rxq.mark = 1;
1450         }
1451         priv->mark_enabled = 1;
1452 }
1453
1454 /**
1455  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1456  *
1457  * @param[in] dev
1458  *   Pointer to the Ethernet device structure.
1459  * @param[in] flow
1460  *   Pointer to flow structure.
1461  */
1462 static void
1463 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1464 {
1465         struct mlx5_priv *priv = dev->data->dev_private;
1466         uint32_t handle_idx;
1467         struct mlx5_flow_handle *dev_handle;
1468         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
1469
1470         MLX5_ASSERT(wks);
1471         if (wks->mark)
1472                 flow_rxq_mark_flag_set(dev);
1473         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1474                        handle_idx, dev_handle, next)
1475                 flow_drv_rxq_flags_set(dev, dev_handle);
1476 }
1477
1478 /**
1479  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1480  * device flow if no other flow uses it with the same kind of request.
1481  *
1482  * @param dev
1483  *   Pointer to Ethernet device.
1484  * @param[in] dev_handle
1485  *   Pointer to the device flow handle structure.
1486  */
1487 static void
1488 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1489                         struct mlx5_flow_handle *dev_handle)
1490 {
1491         struct mlx5_priv *priv = dev->data->dev_private;
1492         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1493         struct mlx5_ind_table_obj *ind_tbl = NULL;
1494         unsigned int i;
1495
1496         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1497                 struct mlx5_hrxq *hrxq;
1498
1499                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1500                               dev_handle->rix_hrxq);
1501                 if (hrxq)
1502                         ind_tbl = hrxq->ind_table;
1503         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1504                 struct mlx5_shared_action_rss *shared_rss;
1505
1506                 shared_rss = mlx5_ipool_get
1507                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1508                          dev_handle->rix_srss);
1509                 if (shared_rss)
1510                         ind_tbl = shared_rss->ind_tbl;
1511         }
1512         if (!ind_tbl)
1513                 return;
1514         MLX5_ASSERT(dev->data->dev_started);
1515         for (i = 0; i != ind_tbl->queues_n; ++i) {
1516                 int idx = ind_tbl->queues[i];
1517                 struct mlx5_rxq_ctrl *rxq_ctrl;
1518
1519                 if (mlx5_is_external_rxq(dev, idx))
1520                         continue;
1521                 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx);
1522                 MLX5_ASSERT(rxq_ctrl != NULL);
1523                 if (rxq_ctrl == NULL)
1524                         continue;
1525                 if (tunnel) {
1526                         unsigned int j;
1527
1528                         /* Decrease the counter matching the flow. */
1529                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1530                                 if ((tunnels_info[j].tunnel &
1531                                      dev_handle->layers) ==
1532                                     tunnels_info[j].tunnel) {
1533                                         rxq_ctrl->flow_tunnels_n[j]--;
1534                                         break;
1535                                 }
1536                         }
1537                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1538                 }
1539         }
1540 }
1541
1542 /**
1543  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1544  * @p flow if no other flow uses it with the same kind of request.
1545  *
1546  * @param dev
1547  *   Pointer to Ethernet device.
1548  * @param[in] flow
1549  *   Pointer to the flow.
1550  */
1551 static void
1552 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1553 {
1554         struct mlx5_priv *priv = dev->data->dev_private;
1555         uint32_t handle_idx;
1556         struct mlx5_flow_handle *dev_handle;
1557
1558         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1559                        handle_idx, dev_handle, next)
1560                 flow_drv_rxq_flags_trim(dev, dev_handle);
1561 }
1562
1563 /**
1564  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1565  *
1566  * @param dev
1567  *   Pointer to Ethernet device.
1568  */
1569 static void
1570 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1571 {
1572         struct mlx5_priv *priv = dev->data->dev_private;
1573         unsigned int i;
1574
1575         for (i = 0; i != priv->rxqs_n; ++i) {
1576                 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i);
1577                 unsigned int j;
1578
1579                 if (rxq == NULL || rxq->ctrl == NULL)
1580                         continue;
1581                 rxq->ctrl->rxq.mark = 0;
1582                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1583                         rxq->ctrl->flow_tunnels_n[j] = 0;
1584                 rxq->ctrl->rxq.tunnel = 0;
1585         }
1586         priv->mark_enabled = 0;
1587 }
1588
1589 /**
1590  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1591  *
1592  * @param[in] dev
1593  *   Pointer to the Ethernet device structure.
1594  */
1595 void
1596 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1597 {
1598         struct mlx5_priv *priv = dev->data->dev_private;
1599         unsigned int i;
1600
1601         for (i = 0; i != priv->rxqs_n; ++i) {
1602                 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i);
1603                 struct mlx5_rxq_data *data;
1604
1605                 if (rxq == NULL || rxq->ctrl == NULL)
1606                         continue;
1607                 data = &rxq->ctrl->rxq;
1608                 if (!rte_flow_dynf_metadata_avail()) {
1609                         data->dynf_meta = 0;
1610                         data->flow_meta_mask = 0;
1611                         data->flow_meta_offset = -1;
1612                         data->flow_meta_port_mask = 0;
1613                 } else {
1614                         data->dynf_meta = 1;
1615                         data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1616                         data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1617                         data->flow_meta_port_mask = priv->sh->dv_meta_mask;
1618                 }
1619         }
1620 }
1621
1622 /*
1623  * return a pointer to the desired action in the list of actions.
1624  *
1625  * @param[in] actions
1626  *   The list of actions to search the action in.
1627  * @param[in] action
1628  *   The action to find.
1629  *
1630  * @return
1631  *   Pointer to the action in the list, if found. NULL otherwise.
1632  */
1633 const struct rte_flow_action *
1634 mlx5_flow_find_action(const struct rte_flow_action *actions,
1635                       enum rte_flow_action_type action)
1636 {
1637         if (actions == NULL)
1638                 return NULL;
1639         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1640                 if (actions->type == action)
1641                         return actions;
1642         return NULL;
1643 }
1644
1645 /*
1646  * Validate the flag action.
1647  *
1648  * @param[in] action_flags
1649  *   Bit-fields that holds the actions detected until now.
1650  * @param[in] attr
1651  *   Attributes of flow that includes this action.
1652  * @param[out] error
1653  *   Pointer to error structure.
1654  *
1655  * @return
1656  *   0 on success, a negative errno value otherwise and rte_errno is set.
1657  */
1658 int
1659 mlx5_flow_validate_action_flag(uint64_t action_flags,
1660                                const struct rte_flow_attr *attr,
1661                                struct rte_flow_error *error)
1662 {
1663         if (action_flags & MLX5_FLOW_ACTION_MARK)
1664                 return rte_flow_error_set(error, EINVAL,
1665                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1666                                           "can't mark and flag in same flow");
1667         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1668                 return rte_flow_error_set(error, EINVAL,
1669                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1670                                           "can't have 2 flag"
1671                                           " actions in same flow");
1672         if (attr->egress)
1673                 return rte_flow_error_set(error, ENOTSUP,
1674                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1675                                           "flag action not supported for "
1676                                           "egress");
1677         return 0;
1678 }
1679
1680 /*
1681  * Validate the mark action.
1682  *
1683  * @param[in] action
1684  *   Pointer to the queue action.
1685  * @param[in] action_flags
1686  *   Bit-fields that holds the actions detected until now.
1687  * @param[in] attr
1688  *   Attributes of flow that includes this action.
1689  * @param[out] error
1690  *   Pointer to error structure.
1691  *
1692  * @return
1693  *   0 on success, a negative errno value otherwise and rte_errno is set.
1694  */
1695 int
1696 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1697                                uint64_t action_flags,
1698                                const struct rte_flow_attr *attr,
1699                                struct rte_flow_error *error)
1700 {
1701         const struct rte_flow_action_mark *mark = action->conf;
1702
1703         if (!mark)
1704                 return rte_flow_error_set(error, EINVAL,
1705                                           RTE_FLOW_ERROR_TYPE_ACTION,
1706                                           action,
1707                                           "configuration cannot be null");
1708         if (mark->id >= MLX5_FLOW_MARK_MAX)
1709                 return rte_flow_error_set(error, EINVAL,
1710                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1711                                           &mark->id,
1712                                           "mark id must in 0 <= id < "
1713                                           RTE_STR(MLX5_FLOW_MARK_MAX));
1714         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1715                 return rte_flow_error_set(error, EINVAL,
1716                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1717                                           "can't flag and mark in same flow");
1718         if (action_flags & MLX5_FLOW_ACTION_MARK)
1719                 return rte_flow_error_set(error, EINVAL,
1720                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1721                                           "can't have 2 mark actions in same"
1722                                           " flow");
1723         if (attr->egress)
1724                 return rte_flow_error_set(error, ENOTSUP,
1725                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1726                                           "mark action not supported for "
1727                                           "egress");
1728         return 0;
1729 }
1730
1731 /*
1732  * Validate the drop action.
1733  *
1734  * @param[in] action_flags
1735  *   Bit-fields that holds the actions detected until now.
1736  * @param[in] attr
1737  *   Attributes of flow that includes this action.
1738  * @param[out] error
1739  *   Pointer to error structure.
1740  *
1741  * @return
1742  *   0 on success, a negative errno value otherwise and rte_errno is set.
1743  */
1744 int
1745 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1746                                const struct rte_flow_attr *attr,
1747                                struct rte_flow_error *error)
1748 {
1749         if (attr->egress)
1750                 return rte_flow_error_set(error, ENOTSUP,
1751                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1752                                           "drop action not supported for "
1753                                           "egress");
1754         return 0;
1755 }
1756
1757 /*
1758  * Validate the queue action.
1759  *
1760  * @param[in] action
1761  *   Pointer to the queue action.
1762  * @param[in] action_flags
1763  *   Bit-fields that holds the actions detected until now.
1764  * @param[in] dev
1765  *   Pointer to the Ethernet device structure.
1766  * @param[in] attr
1767  *   Attributes of flow that includes this action.
1768  * @param[out] error
1769  *   Pointer to error structure.
1770  *
1771  * @return
1772  *   0 on success, a negative errno value otherwise and rte_errno is set.
1773  */
1774 int
1775 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1776                                 uint64_t action_flags,
1777                                 struct rte_eth_dev *dev,
1778                                 const struct rte_flow_attr *attr,
1779                                 struct rte_flow_error *error)
1780 {
1781         struct mlx5_priv *priv = dev->data->dev_private;
1782         const struct rte_flow_action_queue *queue = action->conf;
1783
1784         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1785                 return rte_flow_error_set(error, EINVAL,
1786                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1787                                           "can't have 2 fate actions in"
1788                                           " same flow");
1789         if (attr->egress)
1790                 return rte_flow_error_set(error, ENOTSUP,
1791                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1792                                           "queue action not supported for egress.");
1793         if (mlx5_is_external_rxq(dev, queue->index))
1794                 return 0;
1795         if (!priv->rxqs_n)
1796                 return rte_flow_error_set(error, EINVAL,
1797                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1798                                           NULL, "No Rx queues configured");
1799         if (queue->index >= priv->rxqs_n)
1800                 return rte_flow_error_set(error, EINVAL,
1801                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1802                                           &queue->index,
1803                                           "queue index out of range");
1804         if (mlx5_rxq_get(dev, queue->index) == NULL)
1805                 return rte_flow_error_set(error, EINVAL,
1806                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1807                                           &queue->index,
1808                                           "queue is not configured");
1809         return 0;
1810 }
1811
1812 /**
1813  * Validate queue numbers for device RSS.
1814  *
1815  * @param[in] dev
1816  *   Configured device.
1817  * @param[in] queues
1818  *   Array of queue numbers.
1819  * @param[in] queues_n
1820  *   Size of the @p queues array.
1821  * @param[out] error
1822  *   On error, filled with a textual error description.
1823  * @param[out] queue_idx
1824  *   On error, filled with an offending queue index in @p queues array.
1825  *
1826  * @return
1827  *   0 on success, a negative errno code on error.
1828  */
1829 static int
1830 mlx5_validate_rss_queues(struct rte_eth_dev *dev,
1831                          const uint16_t *queues, uint32_t queues_n,
1832                          const char **error, uint32_t *queue_idx)
1833 {
1834         const struct mlx5_priv *priv = dev->data->dev_private;
1835         bool is_hairpin = false;
1836         bool is_ext_rss = false;
1837         uint32_t i;
1838
1839         for (i = 0; i != queues_n; ++i) {
1840                 struct mlx5_rxq_ctrl *rxq_ctrl;
1841
1842                 if (mlx5_is_external_rxq(dev, queues[0])) {
1843                         is_ext_rss = true;
1844                         continue;
1845                 }
1846                 if (is_ext_rss) {
1847                         *error = "Combining external and regular RSS queues is not supported";
1848                         *queue_idx = i;
1849                         return -ENOTSUP;
1850                 }
1851                 if (queues[i] >= priv->rxqs_n) {
1852                         *error = "queue index out of range";
1853                         *queue_idx = i;
1854                         return -EINVAL;
1855                 }
1856                 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]);
1857                 if (rxq_ctrl == NULL) {
1858                         *error =  "queue is not configured";
1859                         *queue_idx = i;
1860                         return -EINVAL;
1861                 }
1862                 if (i == 0 && rxq_ctrl->is_hairpin)
1863                         is_hairpin = true;
1864                 if (is_hairpin != rxq_ctrl->is_hairpin) {
1865                         *error = "combining hairpin and regular RSS queues is not supported";
1866                         *queue_idx = i;
1867                         return -ENOTSUP;
1868                 }
1869         }
1870         return 0;
1871 }
1872
1873 /*
1874  * Validate the rss action.
1875  *
1876  * @param[in] dev
1877  *   Pointer to the Ethernet device structure.
1878  * @param[in] action
1879  *   Pointer to the queue action.
1880  * @param[out] error
1881  *   Pointer to error structure.
1882  *
1883  * @return
1884  *   0 on success, a negative errno value otherwise and rte_errno is set.
1885  */
1886 int
1887 mlx5_validate_action_rss(struct rte_eth_dev *dev,
1888                          const struct rte_flow_action *action,
1889                          struct rte_flow_error *error)
1890 {
1891         struct mlx5_priv *priv = dev->data->dev_private;
1892         const struct rte_flow_action_rss *rss = action->conf;
1893         int ret;
1894         const char *message;
1895         uint32_t queue_idx;
1896
1897         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1898             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1899                 return rte_flow_error_set(error, ENOTSUP,
1900                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1901                                           &rss->func,
1902                                           "RSS hash function not supported");
1903 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1904         if (rss->level > 2)
1905 #else
1906         if (rss->level > 1)
1907 #endif
1908                 return rte_flow_error_set(error, ENOTSUP,
1909                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1910                                           &rss->level,
1911                                           "tunnel RSS is not supported");
1912         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1913         if (rss->key_len == 0 && rss->key != NULL)
1914                 return rte_flow_error_set(error, ENOTSUP,
1915                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1916                                           &rss->key_len,
1917                                           "RSS hash key length 0");
1918         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1919                 return rte_flow_error_set(error, ENOTSUP,
1920                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1921                                           &rss->key_len,
1922                                           "RSS hash key too small");
1923         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1924                 return rte_flow_error_set(error, ENOTSUP,
1925                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1926                                           &rss->key_len,
1927                                           "RSS hash key too large");
1928         if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size)
1929                 return rte_flow_error_set(error, ENOTSUP,
1930                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1931                                           &rss->queue_num,
1932                                           "number of queues too large");
1933         if (rss->types & MLX5_RSS_HF_MASK)
1934                 return rte_flow_error_set(error, ENOTSUP,
1935                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1936                                           &rss->types,
1937                                           "some RSS protocols are not"
1938                                           " supported");
1939         if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) &&
1940             !(rss->types & RTE_ETH_RSS_IP))
1941                 return rte_flow_error_set(error, EINVAL,
1942                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1943                                           "L3 partial RSS requested but L3 RSS"
1944                                           " type not specified");
1945         if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) &&
1946             !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP)))
1947                 return rte_flow_error_set(error, EINVAL,
1948                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1949                                           "L4 partial RSS requested but L4 RSS"
1950                                           " type not specified");
1951         if (!priv->rxqs_n && priv->ext_rxqs == NULL)
1952                 return rte_flow_error_set(error, EINVAL,
1953                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1954                                           NULL, "No Rx queues configured");
1955         if (!rss->queue_num)
1956                 return rte_flow_error_set(error, EINVAL,
1957                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1958                                           NULL, "No queues configured");
1959         ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num,
1960                                        &message, &queue_idx);
1961         if (ret != 0) {
1962                 return rte_flow_error_set(error, -ret,
1963                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1964                                           &rss->queue[queue_idx], message);
1965         }
1966         return 0;
1967 }
1968
1969 /*
1970  * Validate the rss action.
1971  *
1972  * @param[in] action
1973  *   Pointer to the queue action.
1974  * @param[in] action_flags
1975  *   Bit-fields that holds the actions detected until now.
1976  * @param[in] dev
1977  *   Pointer to the Ethernet device structure.
1978  * @param[in] attr
1979  *   Attributes of flow that includes this action.
1980  * @param[in] item_flags
1981  *   Items that were detected.
1982  * @param[out] error
1983  *   Pointer to error structure.
1984  *
1985  * @return
1986  *   0 on success, a negative errno value otherwise and rte_errno is set.
1987  */
1988 int
1989 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1990                               uint64_t action_flags,
1991                               struct rte_eth_dev *dev,
1992                               const struct rte_flow_attr *attr,
1993                               uint64_t item_flags,
1994                               struct rte_flow_error *error)
1995 {
1996         const struct rte_flow_action_rss *rss = action->conf;
1997         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1998         int ret;
1999
2000         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
2001                 return rte_flow_error_set(error, EINVAL,
2002                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2003                                           "can't have 2 fate actions"
2004                                           " in same flow");
2005         ret = mlx5_validate_action_rss(dev, action, error);
2006         if (ret)
2007                 return ret;
2008         if (attr->egress)
2009                 return rte_flow_error_set(error, ENOTSUP,
2010                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
2011                                           "rss action not supported for "
2012                                           "egress");
2013         if (rss->level > 1 && !tunnel)
2014                 return rte_flow_error_set(error, EINVAL,
2015                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
2016                                           "inner RSS is not supported for "
2017                                           "non-tunnel flows");
2018         if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
2019             !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
2020                 return rte_flow_error_set(error, EINVAL,
2021                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
2022                                           "RSS on eCPRI is not supported now");
2023         }
2024         if ((item_flags & MLX5_FLOW_LAYER_MPLS) &&
2025             !(item_flags &
2026               (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) &&
2027             rss->level > 1)
2028                 return rte_flow_error_set(error, EINVAL,
2029                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2030                                           "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern");
2031         return 0;
2032 }
2033
2034 /*
2035  * Validate the default miss action.
2036  *
2037  * @param[in] action_flags
2038  *   Bit-fields that holds the actions detected until now.
2039  * @param[out] error
2040  *   Pointer to error structure.
2041  *
2042  * @return
2043  *   0 on success, a negative errno value otherwise and rte_errno is set.
2044  */
2045 int
2046 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
2047                                 const struct rte_flow_attr *attr,
2048                                 struct rte_flow_error *error)
2049 {
2050         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
2051                 return rte_flow_error_set(error, EINVAL,
2052                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2053                                           "can't have 2 fate actions in"
2054                                           " same flow");
2055         if (attr->egress)
2056                 return rte_flow_error_set(error, ENOTSUP,
2057                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
2058                                           "default miss action not supported "
2059                                           "for egress");
2060         if (attr->group)
2061                 return rte_flow_error_set(error, ENOTSUP,
2062                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
2063                                           "only group 0 is supported");
2064         if (attr->transfer)
2065                 return rte_flow_error_set(error, ENOTSUP,
2066                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
2067                                           NULL, "transfer is not supported");
2068         return 0;
2069 }
2070
2071 /*
2072  * Validate the count action.
2073  *
2074  * @param[in] dev
2075  *   Pointer to the Ethernet device structure.
2076  * @param[in] attr
2077  *   Attributes of flow that includes this action.
2078  * @param[out] error
2079  *   Pointer to error structure.
2080  *
2081  * @return
2082  *   0 on success, a negative errno value otherwise and rte_errno is set.
2083  */
2084 int
2085 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
2086                                 const struct rte_flow_attr *attr,
2087                                 struct rte_flow_error *error)
2088 {
2089         if (attr->egress)
2090                 return rte_flow_error_set(error, ENOTSUP,
2091                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
2092                                           "count action not supported for "
2093                                           "egress");
2094         return 0;
2095 }
2096
2097 /*
2098  * Validate the ASO CT action.
2099  *
2100  * @param[in] dev
2101  *   Pointer to the Ethernet device structure.
2102  * @param[in] conntrack
2103  *   Pointer to the CT action profile.
2104  * @param[out] error
2105  *   Pointer to error structure.
2106  *
2107  * @return
2108  *   0 on success, a negative errno value otherwise and rte_errno is set.
2109  */
2110 int
2111 mlx5_validate_action_ct(struct rte_eth_dev *dev,
2112                         const struct rte_flow_action_conntrack *conntrack,
2113                         struct rte_flow_error *error)
2114 {
2115         RTE_SET_USED(dev);
2116
2117         if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT)
2118                 return rte_flow_error_set(error, EINVAL,
2119                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2120                                           "Invalid CT state");
2121         if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST)
2122                 return rte_flow_error_set(error, EINVAL,
2123                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2124                                           "Invalid last TCP packet flag");
2125         return 0;
2126 }
2127
2128 /**
2129  * Verify the @p attributes will be correctly understood by the NIC and store
2130  * them in the @p flow if everything is correct.
2131  *
2132  * @param[in] dev
2133  *   Pointer to the Ethernet device structure.
2134  * @param[in] attributes
2135  *   Pointer to flow attributes
2136  * @param[out] error
2137  *   Pointer to error structure.
2138  *
2139  * @return
2140  *   0 on success, a negative errno value otherwise and rte_errno is set.
2141  */
2142 int
2143 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
2144                               const struct rte_flow_attr *attributes,
2145                               struct rte_flow_error *error)
2146 {
2147         struct mlx5_priv *priv = dev->data->dev_private;
2148         uint32_t priority_max = priv->sh->flow_max_priority - 1;
2149
2150         if (attributes->group)
2151                 return rte_flow_error_set(error, ENOTSUP,
2152                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
2153                                           NULL, "groups is not supported");
2154         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
2155             attributes->priority >= priority_max)
2156                 return rte_flow_error_set(error, ENOTSUP,
2157                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
2158                                           NULL, "priority out of range");
2159         if (attributes->egress)
2160                 return rte_flow_error_set(error, ENOTSUP,
2161                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
2162                                           "egress is not supported");
2163         if (attributes->transfer && !priv->sh->config.dv_esw_en)
2164                 return rte_flow_error_set(error, ENOTSUP,
2165                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
2166                                           NULL, "transfer is not supported");
2167         if (!attributes->ingress)
2168                 return rte_flow_error_set(error, EINVAL,
2169                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
2170                                           NULL,
2171                                           "ingress attribute is mandatory");
2172         return 0;
2173 }
2174
2175 /**
2176  * Validate ICMP6 item.
2177  *
2178  * @param[in] item
2179  *   Item specification.
2180  * @param[in] item_flags
2181  *   Bit-fields that holds the items detected until now.
2182  * @param[in] ext_vlan_sup
2183  *   Whether extended VLAN features are supported or not.
2184  * @param[out] error
2185  *   Pointer to error structure.
2186  *
2187  * @return
2188  *   0 on success, a negative errno value otherwise and rte_errno is set.
2189  */
2190 int
2191 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
2192                                uint64_t item_flags,
2193                                uint8_t target_protocol,
2194                                struct rte_flow_error *error)
2195 {
2196         const struct rte_flow_item_icmp6 *mask = item->mask;
2197         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2198         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
2199                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
2200         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2201                                       MLX5_FLOW_LAYER_OUTER_L4;
2202         int ret;
2203
2204         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
2205                 return rte_flow_error_set(error, EINVAL,
2206                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2207                                           "protocol filtering not compatible"
2208                                           " with ICMP6 layer");
2209         if (!(item_flags & l3m))
2210                 return rte_flow_error_set(error, EINVAL,
2211                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2212                                           "IPv6 is mandatory to filter on"
2213                                           " ICMP6");
2214         if (item_flags & l4m)
2215                 return rte_flow_error_set(error, EINVAL,
2216                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2217                                           "multiple L4 layers not supported");
2218         if (!mask)
2219                 mask = &rte_flow_item_icmp6_mask;
2220         ret = mlx5_flow_item_acceptable
2221                 (item, (const uint8_t *)mask,
2222                  (const uint8_t *)&rte_flow_item_icmp6_mask,
2223                  sizeof(struct rte_flow_item_icmp6),
2224                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2225         if (ret < 0)
2226                 return ret;
2227         return 0;
2228 }
2229
2230 /**
2231  * Validate ICMP item.
2232  *
2233  * @param[in] item
2234  *   Item specification.
2235  * @param[in] item_flags
2236  *   Bit-fields that holds the items detected until now.
2237  * @param[out] error
2238  *   Pointer to error structure.
2239  *
2240  * @return
2241  *   0 on success, a negative errno value otherwise and rte_errno is set.
2242  */
2243 int
2244 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
2245                              uint64_t item_flags,
2246                              uint8_t target_protocol,
2247                              struct rte_flow_error *error)
2248 {
2249         const struct rte_flow_item_icmp *mask = item->mask;
2250         const struct rte_flow_item_icmp nic_mask = {
2251                 .hdr.icmp_type = 0xff,
2252                 .hdr.icmp_code = 0xff,
2253                 .hdr.icmp_ident = RTE_BE16(0xffff),
2254                 .hdr.icmp_seq_nb = RTE_BE16(0xffff),
2255         };
2256         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2257         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
2258                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
2259         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2260                                       MLX5_FLOW_LAYER_OUTER_L4;
2261         int ret;
2262
2263         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
2264                 return rte_flow_error_set(error, EINVAL,
2265                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2266                                           "protocol filtering not compatible"
2267                                           " with ICMP layer");
2268         if (!(item_flags & l3m))
2269                 return rte_flow_error_set(error, EINVAL,
2270                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2271                                           "IPv4 is mandatory to filter"
2272                                           " on ICMP");
2273         if (item_flags & l4m)
2274                 return rte_flow_error_set(error, EINVAL,
2275                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2276                                           "multiple L4 layers not supported");
2277         if (!mask)
2278                 mask = &nic_mask;
2279         ret = mlx5_flow_item_acceptable
2280                 (item, (const uint8_t *)mask,
2281                  (const uint8_t *)&nic_mask,
2282                  sizeof(struct rte_flow_item_icmp),
2283                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2284         if (ret < 0)
2285                 return ret;
2286         return 0;
2287 }
2288
2289 /**
2290  * Validate Ethernet item.
2291  *
2292  * @param[in] item
2293  *   Item specification.
2294  * @param[in] item_flags
2295  *   Bit-fields that holds the items detected until now.
2296  * @param[out] error
2297  *   Pointer to error structure.
2298  *
2299  * @return
2300  *   0 on success, a negative errno value otherwise and rte_errno is set.
2301  */
2302 int
2303 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
2304                             uint64_t item_flags, bool ext_vlan_sup,
2305                             struct rte_flow_error *error)
2306 {
2307         const struct rte_flow_item_eth *mask = item->mask;
2308         const struct rte_flow_item_eth nic_mask = {
2309                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2310                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2311                 .type = RTE_BE16(0xffff),
2312                 .has_vlan = ext_vlan_sup ? 1 : 0,
2313         };
2314         int ret;
2315         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2316         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
2317                                        MLX5_FLOW_LAYER_OUTER_L2;
2318
2319         if (item_flags & ethm)
2320                 return rte_flow_error_set(error, ENOTSUP,
2321                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2322                                           "multiple L2 layers not supported");
2323         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
2324             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
2325                 return rte_flow_error_set(error, EINVAL,
2326                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2327                                           "L2 layer should not follow "
2328                                           "L3 layers");
2329         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
2330             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
2331                 return rte_flow_error_set(error, EINVAL,
2332                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2333                                           "L2 layer should not follow VLAN");
2334         if (item_flags & MLX5_FLOW_LAYER_GTP)
2335                 return rte_flow_error_set(error, EINVAL,
2336                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2337                                           "L2 layer should not follow GTP");
2338         if (!mask)
2339                 mask = &rte_flow_item_eth_mask;
2340         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2341                                         (const uint8_t *)&nic_mask,
2342                                         sizeof(struct rte_flow_item_eth),
2343                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2344         return ret;
2345 }
2346
2347 /**
2348  * Validate VLAN item.
2349  *
2350  * @param[in] item
2351  *   Item specification.
2352  * @param[in] item_flags
2353  *   Bit-fields that holds the items detected until now.
2354  * @param[in] dev
2355  *   Ethernet device flow is being created on.
2356  * @param[out] error
2357  *   Pointer to error structure.
2358  *
2359  * @return
2360  *   0 on success, a negative errno value otherwise and rte_errno is set.
2361  */
2362 int
2363 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
2364                              uint64_t item_flags,
2365                              struct rte_eth_dev *dev,
2366                              struct rte_flow_error *error)
2367 {
2368         const struct rte_flow_item_vlan *spec = item->spec;
2369         const struct rte_flow_item_vlan *mask = item->mask;
2370         const struct rte_flow_item_vlan nic_mask = {
2371                 .tci = RTE_BE16(UINT16_MAX),
2372                 .inner_type = RTE_BE16(UINT16_MAX),
2373         };
2374         uint16_t vlan_tag = 0;
2375         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2376         int ret;
2377         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2378                                         MLX5_FLOW_LAYER_INNER_L4) :
2379                                        (MLX5_FLOW_LAYER_OUTER_L3 |
2380                                         MLX5_FLOW_LAYER_OUTER_L4);
2381         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2382                                         MLX5_FLOW_LAYER_OUTER_VLAN;
2383
2384         if (item_flags & vlanm)
2385                 return rte_flow_error_set(error, EINVAL,
2386                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2387                                           "multiple VLAN layers not supported");
2388         else if ((item_flags & l34m) != 0)
2389                 return rte_flow_error_set(error, EINVAL,
2390                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2391                                           "VLAN cannot follow L3/L4 layer");
2392         if (!mask)
2393                 mask = &rte_flow_item_vlan_mask;
2394         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2395                                         (const uint8_t *)&nic_mask,
2396                                         sizeof(struct rte_flow_item_vlan),
2397                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2398         if (ret)
2399                 return ret;
2400         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2401                 struct mlx5_priv *priv = dev->data->dev_private;
2402
2403                 if (priv->vmwa_context) {
2404                         /*
2405                          * Non-NULL context means we have a virtual machine
2406                          * and SR-IOV enabled, we have to create VLAN interface
2407                          * to make hypervisor to setup E-Switch vport
2408                          * context correctly. We avoid creating the multiple
2409                          * VLAN interfaces, so we cannot support VLAN tag mask.
2410                          */
2411                         return rte_flow_error_set(error, EINVAL,
2412                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2413                                                   item,
2414                                                   "VLAN tag mask is not"
2415                                                   " supported in virtual"
2416                                                   " environment");
2417                 }
2418         }
2419         if (spec) {
2420                 vlan_tag = spec->tci;
2421                 vlan_tag &= mask->tci;
2422         }
2423         /*
2424          * From verbs perspective an empty VLAN is equivalent
2425          * to a packet without VLAN layer.
2426          */
2427         if (!vlan_tag)
2428                 return rte_flow_error_set(error, EINVAL,
2429                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2430                                           item->spec,
2431                                           "VLAN cannot be empty");
2432         return 0;
2433 }
2434
2435 /**
2436  * Validate IPV4 item.
2437  *
2438  * @param[in] item
2439  *   Item specification.
2440  * @param[in] item_flags
2441  *   Bit-fields that holds the items detected until now.
2442  * @param[in] last_item
2443  *   Previous validated item in the pattern items.
2444  * @param[in] ether_type
2445  *   Type in the ethernet layer header (including dot1q).
2446  * @param[in] acc_mask
2447  *   Acceptable mask, if NULL default internal default mask
2448  *   will be used to check whether item fields are supported.
2449  * @param[in] range_accepted
2450  *   True if range of values is accepted for specific fields, false otherwise.
2451  * @param[out] error
2452  *   Pointer to error structure.
2453  *
2454  * @return
2455  *   0 on success, a negative errno value otherwise and rte_errno is set.
2456  */
2457 int
2458 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
2459                              uint64_t item_flags,
2460                              uint64_t last_item,
2461                              uint16_t ether_type,
2462                              const struct rte_flow_item_ipv4 *acc_mask,
2463                              bool range_accepted,
2464                              struct rte_flow_error *error)
2465 {
2466         const struct rte_flow_item_ipv4 *mask = item->mask;
2467         const struct rte_flow_item_ipv4 *spec = item->spec;
2468         const struct rte_flow_item_ipv4 nic_mask = {
2469                 .hdr = {
2470                         .src_addr = RTE_BE32(0xffffffff),
2471                         .dst_addr = RTE_BE32(0xffffffff),
2472                         .type_of_service = 0xff,
2473                         .next_proto_id = 0xff,
2474                 },
2475         };
2476         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2477         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2478                                       MLX5_FLOW_LAYER_OUTER_L3;
2479         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2480                                       MLX5_FLOW_LAYER_OUTER_L4;
2481         int ret;
2482         uint8_t next_proto = 0xFF;
2483         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2484                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2485                                   MLX5_FLOW_LAYER_INNER_VLAN);
2486
2487         if ((last_item & l2_vlan) && ether_type &&
2488             ether_type != RTE_ETHER_TYPE_IPV4)
2489                 return rte_flow_error_set(error, EINVAL,
2490                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2491                                           "IPv4 cannot follow L2/VLAN layer "
2492                                           "which ether type is not IPv4");
2493         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2494                 if (mask && spec)
2495                         next_proto = mask->hdr.next_proto_id &
2496                                      spec->hdr.next_proto_id;
2497                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2498                         return rte_flow_error_set(error, EINVAL,
2499                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2500                                                   item,
2501                                                   "multiple tunnel "
2502                                                   "not supported");
2503         }
2504         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
2505                 return rte_flow_error_set(error, EINVAL,
2506                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2507                                           "wrong tunnel type - IPv6 specified "
2508                                           "but IPv4 item provided");
2509         if (item_flags & l3m)
2510                 return rte_flow_error_set(error, ENOTSUP,
2511                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2512                                           "multiple L3 layers not supported");
2513         else if (item_flags & l4m)
2514                 return rte_flow_error_set(error, EINVAL,
2515                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2516                                           "L3 cannot follow an L4 layer.");
2517         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2518                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2519                 return rte_flow_error_set(error, EINVAL,
2520                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2521                                           "L3 cannot follow an NVGRE layer.");
2522         if (!mask)
2523                 mask = &rte_flow_item_ipv4_mask;
2524         else if (mask->hdr.next_proto_id != 0 &&
2525                  mask->hdr.next_proto_id != 0xff)
2526                 return rte_flow_error_set(error, EINVAL,
2527                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2528                                           "partial mask is not supported"
2529                                           " for protocol");
2530         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2531                                         acc_mask ? (const uint8_t *)acc_mask
2532                                                  : (const uint8_t *)&nic_mask,
2533                                         sizeof(struct rte_flow_item_ipv4),
2534                                         range_accepted, error);
2535         if (ret < 0)
2536                 return ret;
2537         return 0;
2538 }
2539
2540 /**
2541  * Validate IPV6 item.
2542  *
2543  * @param[in] item
2544  *   Item specification.
2545  * @param[in] item_flags
2546  *   Bit-fields that holds the items detected until now.
2547  * @param[in] last_item
2548  *   Previous validated item in the pattern items.
2549  * @param[in] ether_type
2550  *   Type in the ethernet layer header (including dot1q).
2551  * @param[in] acc_mask
2552  *   Acceptable mask, if NULL default internal default mask
2553  *   will be used to check whether item fields are supported.
2554  * @param[out] error
2555  *   Pointer to error structure.
2556  *
2557  * @return
2558  *   0 on success, a negative errno value otherwise and rte_errno is set.
2559  */
2560 int
2561 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2562                              uint64_t item_flags,
2563                              uint64_t last_item,
2564                              uint16_t ether_type,
2565                              const struct rte_flow_item_ipv6 *acc_mask,
2566                              struct rte_flow_error *error)
2567 {
2568         const struct rte_flow_item_ipv6 *mask = item->mask;
2569         const struct rte_flow_item_ipv6 *spec = item->spec;
2570         const struct rte_flow_item_ipv6 nic_mask = {
2571                 .hdr = {
2572                         .src_addr =
2573                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2574                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2575                         .dst_addr =
2576                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2577                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2578                         .vtc_flow = RTE_BE32(0xffffffff),
2579                         .proto = 0xff,
2580                 },
2581         };
2582         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2583         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2584                                       MLX5_FLOW_LAYER_OUTER_L3;
2585         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2586                                       MLX5_FLOW_LAYER_OUTER_L4;
2587         int ret;
2588         uint8_t next_proto = 0xFF;
2589         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2590                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2591                                   MLX5_FLOW_LAYER_INNER_VLAN);
2592
2593         if ((last_item & l2_vlan) && ether_type &&
2594             ether_type != RTE_ETHER_TYPE_IPV6)
2595                 return rte_flow_error_set(error, EINVAL,
2596                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2597                                           "IPv6 cannot follow L2/VLAN layer "
2598                                           "which ether type is not IPv6");
2599         if (mask && mask->hdr.proto == UINT8_MAX && spec)
2600                 next_proto = spec->hdr.proto;
2601         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2602                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2603                         return rte_flow_error_set(error, EINVAL,
2604                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2605                                                   item,
2606                                                   "multiple tunnel "
2607                                                   "not supported");
2608         }
2609         if (next_proto == IPPROTO_HOPOPTS  ||
2610             next_proto == IPPROTO_ROUTING  ||
2611             next_proto == IPPROTO_FRAGMENT ||
2612             next_proto == IPPROTO_ESP      ||
2613             next_proto == IPPROTO_AH       ||
2614             next_proto == IPPROTO_DSTOPTS)
2615                 return rte_flow_error_set(error, EINVAL,
2616                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2617                                           "IPv6 proto (next header) should "
2618                                           "not be set as extension header");
2619         if (item_flags & MLX5_FLOW_LAYER_IPIP)
2620                 return rte_flow_error_set(error, EINVAL,
2621                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2622                                           "wrong tunnel type - IPv4 specified "
2623                                           "but IPv6 item provided");
2624         if (item_flags & l3m)
2625                 return rte_flow_error_set(error, ENOTSUP,
2626                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2627                                           "multiple L3 layers not supported");
2628         else if (item_flags & l4m)
2629                 return rte_flow_error_set(error, EINVAL,
2630                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2631                                           "L3 cannot follow an L4 layer.");
2632         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2633                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2634                 return rte_flow_error_set(error, EINVAL,
2635                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2636                                           "L3 cannot follow an NVGRE layer.");
2637         if (!mask)
2638                 mask = &rte_flow_item_ipv6_mask;
2639         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2640                                         acc_mask ? (const uint8_t *)acc_mask
2641                                                  : (const uint8_t *)&nic_mask,
2642                                         sizeof(struct rte_flow_item_ipv6),
2643                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2644         if (ret < 0)
2645                 return ret;
2646         return 0;
2647 }
2648
2649 /**
2650  * Validate UDP item.
2651  *
2652  * @param[in] item
2653  *   Item specification.
2654  * @param[in] item_flags
2655  *   Bit-fields that holds the items detected until now.
2656  * @param[in] target_protocol
2657  *   The next protocol in the previous item.
2658  * @param[in] flow_mask
2659  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2660  * @param[out] error
2661  *   Pointer to error structure.
2662  *
2663  * @return
2664  *   0 on success, a negative errno value otherwise and rte_errno is set.
2665  */
2666 int
2667 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2668                             uint64_t item_flags,
2669                             uint8_t target_protocol,
2670                             struct rte_flow_error *error)
2671 {
2672         const struct rte_flow_item_udp *mask = item->mask;
2673         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2674         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2675                                       MLX5_FLOW_LAYER_OUTER_L3;
2676         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2677                                       MLX5_FLOW_LAYER_OUTER_L4;
2678         int ret;
2679
2680         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2681                 return rte_flow_error_set(error, EINVAL,
2682                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2683                                           "protocol filtering not compatible"
2684                                           " with UDP layer");
2685         if (!(item_flags & l3m))
2686                 return rte_flow_error_set(error, EINVAL,
2687                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2688                                           "L3 is mandatory to filter on L4");
2689         if (item_flags & l4m)
2690                 return rte_flow_error_set(error, EINVAL,
2691                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2692                                           "multiple L4 layers not supported");
2693         if (!mask)
2694                 mask = &rte_flow_item_udp_mask;
2695         ret = mlx5_flow_item_acceptable
2696                 (item, (const uint8_t *)mask,
2697                  (const uint8_t *)&rte_flow_item_udp_mask,
2698                  sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2699                  error);
2700         if (ret < 0)
2701                 return ret;
2702         return 0;
2703 }
2704
2705 /**
2706  * Validate TCP item.
2707  *
2708  * @param[in] item
2709  *   Item specification.
2710  * @param[in] item_flags
2711  *   Bit-fields that holds the items detected until now.
2712  * @param[in] target_protocol
2713  *   The next protocol in the previous item.
2714  * @param[out] error
2715  *   Pointer to error structure.
2716  *
2717  * @return
2718  *   0 on success, a negative errno value otherwise and rte_errno is set.
2719  */
2720 int
2721 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2722                             uint64_t item_flags,
2723                             uint8_t target_protocol,
2724                             const struct rte_flow_item_tcp *flow_mask,
2725                             struct rte_flow_error *error)
2726 {
2727         const struct rte_flow_item_tcp *mask = item->mask;
2728         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2729         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2730                                       MLX5_FLOW_LAYER_OUTER_L3;
2731         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2732                                       MLX5_FLOW_LAYER_OUTER_L4;
2733         int ret;
2734
2735         MLX5_ASSERT(flow_mask);
2736         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2737                 return rte_flow_error_set(error, EINVAL,
2738                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2739                                           "protocol filtering not compatible"
2740                                           " with TCP layer");
2741         if (!(item_flags & l3m))
2742                 return rte_flow_error_set(error, EINVAL,
2743                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2744                                           "L3 is mandatory to filter on L4");
2745         if (item_flags & l4m)
2746                 return rte_flow_error_set(error, EINVAL,
2747                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2748                                           "multiple L4 layers not supported");
2749         if (!mask)
2750                 mask = &rte_flow_item_tcp_mask;
2751         ret = mlx5_flow_item_acceptable
2752                 (item, (const uint8_t *)mask,
2753                  (const uint8_t *)flow_mask,
2754                  sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2755                  error);
2756         if (ret < 0)
2757                 return ret;
2758         return 0;
2759 }
2760
2761 /**
2762  * Validate VXLAN item.
2763  *
2764  * @param[in] dev
2765  *   Pointer to the Ethernet device structure.
2766  * @param[in] udp_dport
2767  *   UDP destination port
2768  * @param[in] item
2769  *   Item specification.
2770  * @param[in] item_flags
2771  *   Bit-fields that holds the items detected until now.
2772  * @param[in] attr
2773  *   Flow rule attributes.
2774  * @param[out] error
2775  *   Pointer to error structure.
2776  *
2777  * @return
2778  *   0 on success, a negative errno value otherwise and rte_errno is set.
2779  */
2780 int
2781 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev,
2782                               uint16_t udp_dport,
2783                               const struct rte_flow_item *item,
2784                               uint64_t item_flags,
2785                               const struct rte_flow_attr *attr,
2786                               struct rte_flow_error *error)
2787 {
2788         const struct rte_flow_item_vxlan *spec = item->spec;
2789         const struct rte_flow_item_vxlan *mask = item->mask;
2790         int ret;
2791         struct mlx5_priv *priv = dev->data->dev_private;
2792         union vni {
2793                 uint32_t vlan_id;
2794                 uint8_t vni[4];
2795         } id = { .vlan_id = 0, };
2796         const struct rte_flow_item_vxlan nic_mask = {
2797                 .vni = "\xff\xff\xff",
2798                 .rsvd1 = 0xff,
2799         };
2800         const struct rte_flow_item_vxlan *valid_mask;
2801
2802         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2803                 return rte_flow_error_set(error, ENOTSUP,
2804                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2805                                           "multiple tunnel layers not"
2806                                           " supported");
2807         valid_mask = &rte_flow_item_vxlan_mask;
2808         /*
2809          * Verify only UDPv4 is present as defined in
2810          * https://tools.ietf.org/html/rfc7348
2811          */
2812         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2813                 return rte_flow_error_set(error, EINVAL,
2814                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2815                                           "no outer UDP layer found");
2816         if (!mask)
2817                 mask = &rte_flow_item_vxlan_mask;
2818
2819         if (priv->sh->steering_format_version !=
2820             MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 ||
2821             !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) {
2822                 /* FDB domain & NIC domain non-zero group */
2823                 if ((attr->transfer || attr->group) && priv->sh->misc5_cap)
2824                         valid_mask = &nic_mask;
2825                 /* Group zero in NIC domain */
2826                 if (!attr->group && !attr->transfer &&
2827                     priv->sh->tunnel_header_0_1)
2828                         valid_mask = &nic_mask;
2829         }
2830         ret = mlx5_flow_item_acceptable
2831                 (item, (const uint8_t *)mask,
2832                  (const uint8_t *)valid_mask,
2833                  sizeof(struct rte_flow_item_vxlan),
2834                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2835         if (ret < 0)
2836                 return ret;
2837         if (spec) {
2838                 memcpy(&id.vni[1], spec->vni, 3);
2839                 memcpy(&id.vni[1], mask->vni, 3);
2840         }
2841         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2842                 return rte_flow_error_set(error, ENOTSUP,
2843                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2844                                           "VXLAN tunnel must be fully defined");
2845         return 0;
2846 }
2847
2848 /**
2849  * Validate VXLAN_GPE item.
2850  *
2851  * @param[in] item
2852  *   Item specification.
2853  * @param[in] item_flags
2854  *   Bit-fields that holds the items detected until now.
2855  * @param[in] priv
2856  *   Pointer to the private data structure.
2857  * @param[in] target_protocol
2858  *   The next protocol in the previous item.
2859  * @param[out] error
2860  *   Pointer to error structure.
2861  *
2862  * @return
2863  *   0 on success, a negative errno value otherwise and rte_errno is set.
2864  */
2865 int
2866 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2867                                   uint64_t item_flags,
2868                                   struct rte_eth_dev *dev,
2869                                   struct rte_flow_error *error)
2870 {
2871         struct mlx5_priv *priv = dev->data->dev_private;
2872         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2873         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2874         int ret;
2875         union vni {
2876                 uint32_t vlan_id;
2877                 uint8_t vni[4];
2878         } id = { .vlan_id = 0, };
2879
2880         if (!priv->sh->config.l3_vxlan_en)
2881                 return rte_flow_error_set(error, ENOTSUP,
2882                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2883                                           "L3 VXLAN is not enabled by device"
2884                                           " parameter and/or not configured in"
2885                                           " firmware");
2886         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2887                 return rte_flow_error_set(error, ENOTSUP,
2888                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2889                                           "multiple tunnel layers not"
2890                                           " supported");
2891         /*
2892          * Verify only UDPv4 is present as defined in
2893          * https://tools.ietf.org/html/rfc7348
2894          */
2895         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2896                 return rte_flow_error_set(error, EINVAL,
2897                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2898                                           "no outer UDP layer found");
2899         if (!mask)
2900                 mask = &rte_flow_item_vxlan_gpe_mask;
2901         ret = mlx5_flow_item_acceptable
2902                 (item, (const uint8_t *)mask,
2903                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2904                  sizeof(struct rte_flow_item_vxlan_gpe),
2905                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2906         if (ret < 0)
2907                 return ret;
2908         if (spec) {
2909                 if (spec->protocol)
2910                         return rte_flow_error_set(error, ENOTSUP,
2911                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2912                                                   item,
2913                                                   "VxLAN-GPE protocol"
2914                                                   " not supported");
2915                 memcpy(&id.vni[1], spec->vni, 3);
2916                 memcpy(&id.vni[1], mask->vni, 3);
2917         }
2918         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2919                 return rte_flow_error_set(error, ENOTSUP,
2920                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2921                                           "VXLAN-GPE tunnel must be fully"
2922                                           " defined");
2923         return 0;
2924 }
2925 /**
2926  * Validate GRE Key item.
2927  *
2928  * @param[in] item
2929  *   Item specification.
2930  * @param[in] item_flags
2931  *   Bit flags to mark detected items.
2932  * @param[in] gre_item
2933  *   Pointer to gre_item
2934  * @param[out] error
2935  *   Pointer to error structure.
2936  *
2937  * @return
2938  *   0 on success, a negative errno value otherwise and rte_errno is set.
2939  */
2940 int
2941 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2942                                 uint64_t item_flags,
2943                                 const struct rte_flow_item *gre_item,
2944                                 struct rte_flow_error *error)
2945 {
2946         const rte_be32_t *mask = item->mask;
2947         int ret = 0;
2948         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2949         const struct rte_flow_item_gre *gre_spec;
2950         const struct rte_flow_item_gre *gre_mask;
2951
2952         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2953                 return rte_flow_error_set(error, ENOTSUP,
2954                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2955                                           "Multiple GRE key not support");
2956         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2957                 return rte_flow_error_set(error, ENOTSUP,
2958                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2959                                           "No preceding GRE header");
2960         if (item_flags & MLX5_FLOW_LAYER_INNER)
2961                 return rte_flow_error_set(error, ENOTSUP,
2962                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2963                                           "GRE key following a wrong item");
2964         gre_mask = gre_item->mask;
2965         if (!gre_mask)
2966                 gre_mask = &rte_flow_item_gre_mask;
2967         gre_spec = gre_item->spec;
2968         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2969                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2970                 return rte_flow_error_set(error, EINVAL,
2971                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2972                                           "Key bit must be on");
2973
2974         if (!mask)
2975                 mask = &gre_key_default_mask;
2976         ret = mlx5_flow_item_acceptable
2977                 (item, (const uint8_t *)mask,
2978                  (const uint8_t *)&gre_key_default_mask,
2979                  sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2980         return ret;
2981 }
2982
2983 /**
2984  * Validate GRE optional item.
2985  *
2986  * @param[in] dev
2987  *   Pointer to the Ethernet device structure.
2988  * @param[in] item
2989  *   Item specification.
2990  * @param[in] item_flags
2991  *   Bit flags to mark detected items.
2992  * @param[in] attr
2993  *   Flow rule attributes.
2994  * @param[in] gre_item
2995  *   Pointer to gre_item
2996  * @param[out] error
2997  *   Pointer to error structure.
2998  *
2999  * @return
3000  *   0 on success, a negative errno value otherwise and rte_errno is set.
3001  */
3002 int
3003 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev,
3004                                    const struct rte_flow_item *item,
3005                                    uint64_t item_flags,
3006                                    const struct rte_flow_attr *attr,
3007                                    const struct rte_flow_item *gre_item,
3008                                    struct rte_flow_error *error)
3009 {
3010         const struct rte_flow_item_gre *gre_spec = gre_item->spec;
3011         const struct rte_flow_item_gre *gre_mask = gre_item->mask;
3012         const struct rte_flow_item_gre_opt *spec = item->spec;
3013         const struct rte_flow_item_gre_opt *mask = item->mask;
3014         struct mlx5_priv *priv = dev->data->dev_private;
3015         int ret = 0;
3016         struct rte_flow_item_gre_opt nic_mask = {
3017                 .checksum_rsvd = {
3018                         .checksum = RTE_BE16(UINT16_MAX),
3019                         .reserved1 = 0x0,
3020                 },
3021                 .key = {
3022                         .key = RTE_BE32(UINT32_MAX),
3023                 },
3024                 .sequence = {
3025                         .sequence = RTE_BE32(UINT32_MAX),
3026                 },
3027         };
3028
3029         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
3030                 return rte_flow_error_set(error, ENOTSUP,
3031                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3032                                           "No preceding GRE header");
3033         if (item_flags & MLX5_FLOW_LAYER_INNER)
3034                 return rte_flow_error_set(error, ENOTSUP,
3035                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3036                                           "GRE option following a wrong item");
3037         if (!spec || !mask)
3038                 return rte_flow_error_set(error, EINVAL,
3039                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3040                                           "At least one field gre_option(checksum/key/sequence) must be specified");
3041         if (!gre_mask)
3042                 gre_mask = &rte_flow_item_gre_mask;
3043         if (mask->checksum_rsvd.checksum)
3044                 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) &&
3045                                  !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000)))
3046                         return rte_flow_error_set(error, EINVAL,
3047                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3048                                                   item,
3049                                                   "Checksum bit must be on");
3050         if (mask->key.key)
3051                 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
3052                                  !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
3053                         return rte_flow_error_set(error, EINVAL,
3054                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3055                                                   item, "Key bit must be on");
3056         if (mask->sequence.sequence)
3057                 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) &&
3058                                  !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000)))
3059                         return rte_flow_error_set(error, EINVAL,
3060                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3061                                                   item,
3062                                                   "Sequence bit must be on");
3063         if (mask->checksum_rsvd.checksum || mask->sequence.sequence) {
3064                 if (priv->sh->steering_format_version ==
3065                     MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 ||
3066                     ((attr->group || attr->transfer) &&
3067                      !priv->sh->misc5_cap) ||
3068                     (!(priv->sh->tunnel_header_0_1 &&
3069                        priv->sh->tunnel_header_2_3) &&
3070                     !attr->group && !attr->transfer))
3071                         return rte_flow_error_set(error, EINVAL,
3072                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3073                                                   item,
3074                                                   "Checksum/Sequence not supported");
3075         }
3076         ret = mlx5_flow_item_acceptable
3077                 (item, (const uint8_t *)mask,
3078                  (const uint8_t *)&nic_mask,
3079                  sizeof(struct rte_flow_item_gre_opt),
3080                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3081         return ret;
3082 }
3083
3084 /**
3085  * Validate GRE item.
3086  *
3087  * @param[in] item
3088  *   Item specification.
3089  * @param[in] item_flags
3090  *   Bit flags to mark detected items.
3091  * @param[in] target_protocol
3092  *   The next protocol in the previous item.
3093  * @param[out] error
3094  *   Pointer to error structure.
3095  *
3096  * @return
3097  *   0 on success, a negative errno value otherwise and rte_errno is set.
3098  */
3099 int
3100 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
3101                             uint64_t item_flags,
3102                             uint8_t target_protocol,
3103                             struct rte_flow_error *error)
3104 {
3105         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
3106         const struct rte_flow_item_gre *mask = item->mask;
3107         int ret;
3108         const struct rte_flow_item_gre nic_mask = {
3109                 .c_rsvd0_ver = RTE_BE16(0xB000),
3110                 .protocol = RTE_BE16(UINT16_MAX),
3111         };
3112
3113         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
3114                 return rte_flow_error_set(error, EINVAL,
3115                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3116                                           "protocol filtering not compatible"
3117                                           " with this GRE layer");
3118         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3119                 return rte_flow_error_set(error, ENOTSUP,
3120                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3121                                           "multiple tunnel layers not"
3122                                           " supported");
3123         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
3124                 return rte_flow_error_set(error, ENOTSUP,
3125                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3126                                           "L3 Layer is missing");
3127         if (!mask)
3128                 mask = &rte_flow_item_gre_mask;
3129         ret = mlx5_flow_item_acceptable
3130                 (item, (const uint8_t *)mask,
3131                  (const uint8_t *)&nic_mask,
3132                  sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
3133                  error);
3134         if (ret < 0)
3135                 return ret;
3136 #ifndef HAVE_MLX5DV_DR
3137 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
3138         if (spec && (spec->protocol & mask->protocol))
3139                 return rte_flow_error_set(error, ENOTSUP,
3140                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3141                                           "without MPLS support the"
3142                                           " specification cannot be used for"
3143                                           " filtering");
3144 #endif
3145 #endif
3146         return 0;
3147 }
3148
3149 /**
3150  * Validate Geneve item.
3151  *
3152  * @param[in] item
3153  *   Item specification.
3154  * @param[in] itemFlags
3155  *   Bit-fields that holds the items detected until now.
3156  * @param[in] enPriv
3157  *   Pointer to the private data structure.
3158  * @param[out] error
3159  *   Pointer to error structure.
3160  *
3161  * @return
3162  *   0 on success, a negative errno value otherwise and rte_errno is set.
3163  */
3164
3165 int
3166 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
3167                                uint64_t item_flags,
3168                                struct rte_eth_dev *dev,
3169                                struct rte_flow_error *error)
3170 {
3171         struct mlx5_priv *priv = dev->data->dev_private;
3172         const struct rte_flow_item_geneve *spec = item->spec;
3173         const struct rte_flow_item_geneve *mask = item->mask;
3174         int ret;
3175         uint16_t gbhdr;
3176         uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ?
3177                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
3178         const struct rte_flow_item_geneve nic_mask = {
3179                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
3180                 .vni = "\xff\xff\xff",
3181                 .protocol = RTE_BE16(UINT16_MAX),
3182         };
3183
3184         if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx)
3185                 return rte_flow_error_set(error, ENOTSUP,
3186                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3187                                           "L3 Geneve is not enabled by device"
3188                                           " parameter and/or not configured in"
3189                                           " firmware");
3190         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3191                 return rte_flow_error_set(error, ENOTSUP,
3192                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3193                                           "multiple tunnel layers not"
3194                                           " supported");
3195         /*
3196          * Verify only UDPv4 is present as defined in
3197          * https://tools.ietf.org/html/rfc7348
3198          */
3199         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
3200                 return rte_flow_error_set(error, EINVAL,
3201                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3202                                           "no outer UDP layer found");
3203         if (!mask)
3204                 mask = &rte_flow_item_geneve_mask;
3205         ret = mlx5_flow_item_acceptable
3206                                   (item, (const uint8_t *)mask,
3207                                    (const uint8_t *)&nic_mask,
3208                                    sizeof(struct rte_flow_item_geneve),
3209                                    MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3210         if (ret)
3211                 return ret;
3212         if (spec) {
3213                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
3214                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
3215                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
3216                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
3217                         return rte_flow_error_set(error, ENOTSUP,
3218                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3219                                                   item,
3220                                                   "Geneve protocol unsupported"
3221                                                   " fields are being used");
3222                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
3223                         return rte_flow_error_set
3224                                         (error, ENOTSUP,
3225                                          RTE_FLOW_ERROR_TYPE_ITEM,
3226                                          item,
3227                                          "Unsupported Geneve options length");
3228         }
3229         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
3230                 return rte_flow_error_set
3231                                     (error, ENOTSUP,
3232                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
3233                                      "Geneve tunnel must be fully defined");
3234         return 0;
3235 }
3236
3237 /**
3238  * Validate Geneve TLV option item.
3239  *
3240  * @param[in] item
3241  *   Item specification.
3242  * @param[in] last_item
3243  *   Previous validated item in the pattern items.
3244  * @param[in] geneve_item
3245  *   Previous GENEVE item specification.
3246  * @param[in] dev
3247  *   Pointer to the rte_eth_dev structure.
3248  * @param[out] error
3249  *   Pointer to error structure.
3250  *
3251  * @return
3252  *   0 on success, a negative errno value otherwise and rte_errno is set.
3253  */
3254 int
3255 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item,
3256                                    uint64_t last_item,
3257                                    const struct rte_flow_item *geneve_item,
3258                                    struct rte_eth_dev *dev,
3259                                    struct rte_flow_error *error)
3260 {
3261         struct mlx5_priv *priv = dev->data->dev_private;
3262         struct mlx5_dev_ctx_shared *sh = priv->sh;
3263         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource;
3264         struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr;
3265         uint8_t data_max_supported =
3266                         hca_attr->max_geneve_tlv_option_data_len * 4;
3267         const struct rte_flow_item_geneve *geneve_spec;
3268         const struct rte_flow_item_geneve *geneve_mask;
3269         const struct rte_flow_item_geneve_opt *spec = item->spec;
3270         const struct rte_flow_item_geneve_opt *mask = item->mask;
3271         unsigned int i;
3272         unsigned int data_len;
3273         uint8_t tlv_option_len;
3274         uint16_t optlen_m, optlen_v;
3275         const struct rte_flow_item_geneve_opt full_mask = {
3276                 .option_class = RTE_BE16(0xffff),
3277                 .option_type = 0xff,
3278                 .option_len = 0x1f,
3279         };
3280
3281         if (!mask)
3282                 mask = &rte_flow_item_geneve_opt_mask;
3283         if (!spec)
3284                 return rte_flow_error_set
3285                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3286                         "Geneve TLV opt class/type/length must be specified");
3287         if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK)
3288                 return rte_flow_error_set
3289                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3290                         "Geneve TLV opt length exceeds the limit (31)");
3291         /* Check if class type and length masks are full. */
3292         if (full_mask.option_class != mask->option_class ||
3293             full_mask.option_type != mask->option_type ||
3294             full_mask.option_len != (mask->option_len & full_mask.option_len))
3295                 return rte_flow_error_set
3296                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3297                         "Geneve TLV opt class/type/length masks must be full");
3298         /* Check if length is supported */
3299         if ((uint32_t)spec->option_len >
3300                         hca_attr->max_geneve_tlv_option_data_len)
3301                 return rte_flow_error_set
3302                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3303                         "Geneve TLV opt length not supported");
3304         if (hca_attr->max_geneve_tlv_options > 1)
3305                 DRV_LOG(DEBUG,
3306                         "max_geneve_tlv_options supports more than 1 option");
3307         /* Check GENEVE item preceding. */
3308         if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE))
3309                 return rte_flow_error_set
3310                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3311                         "Geneve opt item must be preceded with Geneve item");
3312         geneve_spec = geneve_item->spec;
3313         geneve_mask = geneve_item->mask ? geneve_item->mask :
3314                                           &rte_flow_item_geneve_mask;
3315         /* Check if GENEVE TLV option size doesn't exceed option length */
3316         if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 ||
3317                             geneve_spec->ver_opt_len_o_c_rsvd0)) {
3318                 tlv_option_len = spec->option_len & mask->option_len;
3319                 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0);
3320                 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v);
3321                 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0);
3322                 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m);
3323                 if ((optlen_v & optlen_m) <= tlv_option_len)
3324                         return rte_flow_error_set
3325                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3326                                  "GENEVE TLV option length exceeds optlen");
3327         }
3328         /* Check if length is 0 or data is 0. */
3329         if (spec->data == NULL || spec->option_len == 0)
3330                 return rte_flow_error_set
3331                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3332                         "Geneve TLV opt with zero data/length not supported");
3333         /* Check not all data & mask are 0. */
3334         data_len = spec->option_len * 4;
3335         if (mask->data == NULL) {
3336                 for (i = 0; i < data_len; i++)
3337                         if (spec->data[i])
3338                                 break;
3339                 if (i == data_len)
3340                         return rte_flow_error_set(error, ENOTSUP,
3341                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
3342                                 "Can't match on Geneve option data 0");
3343         } else {
3344                 for (i = 0; i < data_len; i++)
3345                         if (spec->data[i] & mask->data[i])
3346                                 break;
3347                 if (i == data_len)
3348                         return rte_flow_error_set(error, ENOTSUP,
3349                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
3350                                 "Can't match on Geneve option data and mask 0");
3351                 /* Check data mask supported. */
3352                 for (i = data_max_supported; i < data_len ; i++)
3353                         if (mask->data[i])
3354                                 return rte_flow_error_set(error, ENOTSUP,
3355                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
3356                                         "Data mask is of unsupported size");
3357         }
3358         /* Check GENEVE option is supported in NIC. */
3359         if (!hca_attr->geneve_tlv_opt)
3360                 return rte_flow_error_set
3361                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3362                         "Geneve TLV opt not supported");
3363         /* Check if we already have geneve option with different type/class. */
3364         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
3365         geneve_opt_resource = sh->geneve_tlv_option_resource;
3366         if (geneve_opt_resource != NULL)
3367                 if (geneve_opt_resource->option_class != spec->option_class ||
3368                     geneve_opt_resource->option_type != spec->option_type ||
3369                     geneve_opt_resource->length != spec->option_len) {
3370                         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
3371                         return rte_flow_error_set(error, ENOTSUP,
3372                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
3373                                 "Only one Geneve TLV option supported");
3374                 }
3375         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
3376         return 0;
3377 }
3378
3379 /**
3380  * Validate MPLS item.
3381  *
3382  * @param[in] dev
3383  *   Pointer to the rte_eth_dev structure.
3384  * @param[in] item
3385  *   Item specification.
3386  * @param[in] item_flags
3387  *   Bit-fields that holds the items detected until now.
3388  * @param[in] prev_layer
3389  *   The protocol layer indicated in previous item.
3390  * @param[out] error
3391  *   Pointer to error structure.
3392  *
3393  * @return
3394  *   0 on success, a negative errno value otherwise and rte_errno is set.
3395  */
3396 int
3397 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
3398                              const struct rte_flow_item *item __rte_unused,
3399                              uint64_t item_flags __rte_unused,
3400                              uint64_t prev_layer __rte_unused,
3401                              struct rte_flow_error *error)
3402 {
3403 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
3404         const struct rte_flow_item_mpls *mask = item->mask;
3405         struct mlx5_priv *priv = dev->data->dev_private;
3406         int ret;
3407
3408         if (!priv->sh->dev_cap.mpls_en)
3409                 return rte_flow_error_set(error, ENOTSUP,
3410                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3411                                           "MPLS not supported or"
3412                                           " disabled in firmware"
3413                                           " configuration.");
3414         /* MPLS over UDP, GRE is allowed */
3415         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP |
3416                             MLX5_FLOW_LAYER_GRE |
3417                             MLX5_FLOW_LAYER_GRE_KEY)))
3418                 return rte_flow_error_set(error, EINVAL,
3419                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3420                                           "protocol filtering not compatible"
3421                                           " with MPLS layer");
3422         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
3423         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
3424             !(item_flags & MLX5_FLOW_LAYER_GRE))
3425                 return rte_flow_error_set(error, ENOTSUP,
3426                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3427                                           "multiple tunnel layers not"
3428                                           " supported");
3429         if (!mask)
3430                 mask = &rte_flow_item_mpls_mask;
3431         ret = mlx5_flow_item_acceptable
3432                 (item, (const uint8_t *)mask,
3433                  (const uint8_t *)&rte_flow_item_mpls_mask,
3434                  sizeof(struct rte_flow_item_mpls),
3435                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3436         if (ret < 0)
3437                 return ret;
3438         return 0;
3439 #else
3440         return rte_flow_error_set(error, ENOTSUP,
3441                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
3442                                   "MPLS is not supported by Verbs, please"
3443                                   " update.");
3444 #endif
3445 }
3446
3447 /**
3448  * Validate NVGRE item.
3449  *
3450  * @param[in] item
3451  *   Item specification.
3452  * @param[in] item_flags
3453  *   Bit flags to mark detected items.
3454  * @param[in] target_protocol
3455  *   The next protocol in the previous item.
3456  * @param[out] error
3457  *   Pointer to error structure.
3458  *
3459  * @return
3460  *   0 on success, a negative errno value otherwise and rte_errno is set.
3461  */
3462 int
3463 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
3464                               uint64_t item_flags,
3465                               uint8_t target_protocol,
3466                               struct rte_flow_error *error)
3467 {
3468         const struct rte_flow_item_nvgre *mask = item->mask;
3469         int ret;
3470
3471         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
3472                 return rte_flow_error_set(error, EINVAL,
3473                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3474                                           "protocol filtering not compatible"
3475                                           " with this GRE layer");
3476         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3477                 return rte_flow_error_set(error, ENOTSUP,
3478                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3479                                           "multiple tunnel layers not"
3480                                           " supported");
3481         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
3482                 return rte_flow_error_set(error, ENOTSUP,
3483                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3484                                           "L3 Layer is missing");
3485         if (!mask)
3486                 mask = &rte_flow_item_nvgre_mask;
3487         ret = mlx5_flow_item_acceptable
3488                 (item, (const uint8_t *)mask,
3489                  (const uint8_t *)&rte_flow_item_nvgre_mask,
3490                  sizeof(struct rte_flow_item_nvgre),
3491                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3492         if (ret < 0)
3493                 return ret;
3494         return 0;
3495 }
3496
3497 /**
3498  * Validate eCPRI item.
3499  *
3500  * @param[in] item
3501  *   Item specification.
3502  * @param[in] item_flags
3503  *   Bit-fields that holds the items detected until now.
3504  * @param[in] last_item
3505  *   Previous validated item in the pattern items.
3506  * @param[in] ether_type
3507  *   Type in the ethernet layer header (including dot1q).
3508  * @param[in] acc_mask
3509  *   Acceptable mask, if NULL default internal default mask
3510  *   will be used to check whether item fields are supported.
3511  * @param[out] error
3512  *   Pointer to error structure.
3513  *
3514  * @return
3515  *   0 on success, a negative errno value otherwise and rte_errno is set.
3516  */
3517 int
3518 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
3519                               uint64_t item_flags,
3520                               uint64_t last_item,
3521                               uint16_t ether_type,
3522                               const struct rte_flow_item_ecpri *acc_mask,
3523                               struct rte_flow_error *error)
3524 {
3525         const struct rte_flow_item_ecpri *mask = item->mask;
3526         const struct rte_flow_item_ecpri nic_mask = {
3527                 .hdr = {
3528                         .common = {
3529                                 .u32 =
3530                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
3531                                         .type = 0xFF,
3532                                         }).u32),
3533                         },
3534                         .dummy[0] = 0xFFFFFFFF,
3535                 },
3536         };
3537         const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
3538                                         MLX5_FLOW_LAYER_OUTER_VLAN);
3539         struct rte_flow_item_ecpri mask_lo;
3540
3541         if (!(last_item & outer_l2_vlan) &&
3542             last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP)
3543                 return rte_flow_error_set(error, EINVAL,
3544                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3545                                           "eCPRI can only follow L2/VLAN layer or UDP layer");
3546         if ((last_item & outer_l2_vlan) && ether_type &&
3547             ether_type != RTE_ETHER_TYPE_ECPRI)
3548                 return rte_flow_error_set(error, EINVAL,
3549                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3550                                           "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE");
3551         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3552                 return rte_flow_error_set(error, EINVAL,
3553                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3554                                           "eCPRI with tunnel is not supported right now");
3555         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
3556                 return rte_flow_error_set(error, ENOTSUP,
3557                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3558                                           "multiple L3 layers not supported");
3559         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
3560                 return rte_flow_error_set(error, EINVAL,
3561                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3562                                           "eCPRI cannot coexist with a TCP layer");
3563         /* In specification, eCPRI could be over UDP layer. */
3564         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
3565                 return rte_flow_error_set(error, EINVAL,
3566                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3567                                           "eCPRI over UDP layer is not yet supported right now");
3568         /* Mask for type field in common header could be zero. */
3569         if (!mask)
3570                 mask = &rte_flow_item_ecpri_mask;
3571         mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
3572         /* Input mask is in big-endian format. */
3573         if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
3574                 return rte_flow_error_set(error, EINVAL,
3575                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3576                                           "partial mask is not supported for protocol");
3577         else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
3578                 return rte_flow_error_set(error, EINVAL,
3579                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3580                                           "message header mask must be after a type mask");
3581         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
3582                                          acc_mask ? (const uint8_t *)acc_mask
3583                                                   : (const uint8_t *)&nic_mask,
3584                                          sizeof(struct rte_flow_item_ecpri),
3585                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3586 }
3587
3588 static int
3589 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
3590                    const struct rte_flow_attr *attr __rte_unused,
3591                    const struct rte_flow_item items[] __rte_unused,
3592                    const struct rte_flow_action actions[] __rte_unused,
3593                    bool external __rte_unused,
3594                    int hairpin __rte_unused,
3595                    struct rte_flow_error *error)
3596 {
3597         return rte_flow_error_set(error, ENOTSUP,
3598                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3599 }
3600
3601 static struct mlx5_flow *
3602 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
3603                   const struct rte_flow_attr *attr __rte_unused,
3604                   const struct rte_flow_item items[] __rte_unused,
3605                   const struct rte_flow_action actions[] __rte_unused,
3606                   struct rte_flow_error *error)
3607 {
3608         rte_flow_error_set(error, ENOTSUP,
3609                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3610         return NULL;
3611 }
3612
3613 static int
3614 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
3615                     struct mlx5_flow *dev_flow __rte_unused,
3616                     const struct rte_flow_attr *attr __rte_unused,
3617                     const struct rte_flow_item items[] __rte_unused,
3618                     const struct rte_flow_action actions[] __rte_unused,
3619                     struct rte_flow_error *error)
3620 {
3621         return rte_flow_error_set(error, ENOTSUP,
3622                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3623 }
3624
3625 static int
3626 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
3627                 struct rte_flow *flow __rte_unused,
3628                 struct rte_flow_error *error)
3629 {
3630         return rte_flow_error_set(error, ENOTSUP,
3631                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3632 }
3633
3634 static void
3635 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
3636                  struct rte_flow *flow __rte_unused)
3637 {
3638 }
3639
3640 static void
3641 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
3642                   struct rte_flow *flow __rte_unused)
3643 {
3644 }
3645
3646 static int
3647 flow_null_query(struct rte_eth_dev *dev __rte_unused,
3648                 struct rte_flow *flow __rte_unused,
3649                 const struct rte_flow_action *actions __rte_unused,
3650                 void *data __rte_unused,
3651                 struct rte_flow_error *error)
3652 {
3653         return rte_flow_error_set(error, ENOTSUP,
3654                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3655 }
3656
3657 static int
3658 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused,
3659                       uint32_t domains __rte_unused,
3660                       uint32_t flags __rte_unused)
3661 {
3662         return 0;
3663 }
3664
3665 /* Void driver to protect from null pointer reference. */
3666 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
3667         .validate = flow_null_validate,
3668         .prepare = flow_null_prepare,
3669         .translate = flow_null_translate,
3670         .apply = flow_null_apply,
3671         .remove = flow_null_remove,
3672         .destroy = flow_null_destroy,
3673         .query = flow_null_query,
3674         .sync_domain = flow_null_sync_domain,
3675 };
3676
3677 /**
3678  * Select flow driver type according to flow attributes and device
3679  * configuration.
3680  *
3681  * @param[in] dev
3682  *   Pointer to the dev structure.
3683  * @param[in] attr
3684  *   Pointer to the flow attributes.
3685  *
3686  * @return
3687  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
3688  */
3689 static enum mlx5_flow_drv_type
3690 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
3691 {
3692         struct mlx5_priv *priv = dev->data->dev_private;
3693         /* The OS can determine first a specific flow type (DV, VERBS) */
3694         enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
3695
3696         if (type != MLX5_FLOW_TYPE_MAX)
3697                 return type;
3698         /*
3699          * Currently when dv_flow_en == 2, only HW steering engine is
3700          * supported. New engines can also be chosen here if ready.
3701          */
3702         if (priv->sh->config.dv_flow_en == 2)
3703                 return MLX5_FLOW_TYPE_HW;
3704         /* If no OS specific type - continue with DV/VERBS selection */
3705         if (attr->transfer && priv->sh->config.dv_esw_en)
3706                 type = MLX5_FLOW_TYPE_DV;
3707         if (!attr->transfer)
3708                 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
3709                                                      MLX5_FLOW_TYPE_VERBS;
3710         return type;
3711 }
3712
3713 #define flow_get_drv_ops(type) flow_drv_ops[type]
3714
3715 /**
3716  * Flow driver validation API. This abstracts calling driver specific functions.
3717  * The type of flow driver is determined according to flow attributes.
3718  *
3719  * @param[in] dev
3720  *   Pointer to the dev structure.
3721  * @param[in] attr
3722  *   Pointer to the flow attributes.
3723  * @param[in] items
3724  *   Pointer to the list of items.
3725  * @param[in] actions
3726  *   Pointer to the list of actions.
3727  * @param[in] external
3728  *   This flow rule is created by request external to PMD.
3729  * @param[in] hairpin
3730  *   Number of hairpin TX actions, 0 means classic flow.
3731  * @param[out] error
3732  *   Pointer to the error structure.
3733  *
3734  * @return
3735  *   0 on success, a negative errno value otherwise and rte_errno is set.
3736  */
3737 static inline int
3738 flow_drv_validate(struct rte_eth_dev *dev,
3739                   const struct rte_flow_attr *attr,
3740                   const struct rte_flow_item items[],
3741                   const struct rte_flow_action actions[],
3742                   bool external, int hairpin, struct rte_flow_error *error)
3743 {
3744         const struct mlx5_flow_driver_ops *fops;
3745         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
3746
3747         fops = flow_get_drv_ops(type);
3748         return fops->validate(dev, attr, items, actions, external,
3749                               hairpin, error);
3750 }
3751
3752 /**
3753  * Flow driver preparation API. This abstracts calling driver specific
3754  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3755  * calculates the size of memory required for device flow, allocates the memory,
3756  * initializes the device flow and returns the pointer.
3757  *
3758  * @note
3759  *   This function initializes device flow structure such as dv or verbs in
3760  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
3761  *   rest. For example, adding returning device flow to flow->dev_flow list and
3762  *   setting backward reference to the flow should be done out of this function.
3763  *   layers field is not filled either.
3764  *
3765  * @param[in] dev
3766  *   Pointer to the dev structure.
3767  * @param[in] attr
3768  *   Pointer to the flow attributes.
3769  * @param[in] items
3770  *   Pointer to the list of items.
3771  * @param[in] actions
3772  *   Pointer to the list of actions.
3773  * @param[in] flow_idx
3774  *   This memory pool index to the flow.
3775  * @param[out] error
3776  *   Pointer to the error structure.
3777  *
3778  * @return
3779  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
3780  */
3781 static inline struct mlx5_flow *
3782 flow_drv_prepare(struct rte_eth_dev *dev,
3783                  const struct rte_flow *flow,
3784                  const struct rte_flow_attr *attr,
3785                  const struct rte_flow_item items[],
3786                  const struct rte_flow_action actions[],
3787                  uint32_t flow_idx,
3788                  struct rte_flow_error *error)
3789 {
3790         const struct mlx5_flow_driver_ops *fops;
3791         enum mlx5_flow_drv_type type = flow->drv_type;
3792         struct mlx5_flow *mlx5_flow = NULL;
3793
3794         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3795         fops = flow_get_drv_ops(type);
3796         mlx5_flow = fops->prepare(dev, attr, items, actions, error);
3797         if (mlx5_flow)
3798                 mlx5_flow->flow_idx = flow_idx;
3799         return mlx5_flow;
3800 }
3801
3802 /**
3803  * Flow driver translation API. This abstracts calling driver specific
3804  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3805  * translates a generic flow into a driver flow. flow_drv_prepare() must
3806  * precede.
3807  *
3808  * @note
3809  *   dev_flow->layers could be filled as a result of parsing during translation
3810  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
3811  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
3812  *   flow->actions could be overwritten even though all the expanded dev_flows
3813  *   have the same actions.
3814  *
3815  * @param[in] dev
3816  *   Pointer to the rte dev structure.
3817  * @param[in, out] dev_flow
3818  *   Pointer to the mlx5 flow.
3819  * @param[in] attr
3820  *   Pointer to the flow attributes.
3821  * @param[in] items
3822  *   Pointer to the list of items.
3823  * @param[in] actions
3824  *   Pointer to the list of actions.
3825  * @param[out] error
3826  *   Pointer to the error structure.
3827  *
3828  * @return
3829  *   0 on success, a negative errno value otherwise and rte_errno is set.
3830  */
3831 static inline int
3832 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3833                    const struct rte_flow_attr *attr,
3834                    const struct rte_flow_item items[],
3835                    const struct rte_flow_action actions[],
3836                    struct rte_flow_error *error)
3837 {
3838         const struct mlx5_flow_driver_ops *fops;
3839         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
3840
3841         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3842         fops = flow_get_drv_ops(type);
3843         return fops->translate(dev, dev_flow, attr, items, actions, error);
3844 }
3845
3846 /**
3847  * Flow driver apply API. This abstracts calling driver specific functions.
3848  * Parent flow (rte_flow) should have driver type (drv_type). It applies
3849  * translated driver flows on to device. flow_drv_translate() must precede.
3850  *
3851  * @param[in] dev
3852  *   Pointer to Ethernet device structure.
3853  * @param[in, out] flow
3854  *   Pointer to flow structure.
3855  * @param[out] error
3856  *   Pointer to error structure.
3857  *
3858  * @return
3859  *   0 on success, a negative errno value otherwise and rte_errno is set.
3860  */
3861 static inline int
3862 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3863                struct rte_flow_error *error)
3864 {
3865         const struct mlx5_flow_driver_ops *fops;
3866         enum mlx5_flow_drv_type type = flow->drv_type;
3867
3868         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3869         fops = flow_get_drv_ops(type);
3870         return fops->apply(dev, flow, error);
3871 }
3872
3873 /**
3874  * Flow driver destroy API. This abstracts calling driver specific functions.
3875  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3876  * on device and releases resources of the flow.
3877  *
3878  * @param[in] dev
3879  *   Pointer to Ethernet device.
3880  * @param[in, out] flow
3881  *   Pointer to flow structure.
3882  */
3883 static inline void
3884 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3885 {
3886         const struct mlx5_flow_driver_ops *fops;
3887         enum mlx5_flow_drv_type type = flow->drv_type;
3888
3889         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3890         fops = flow_get_drv_ops(type);
3891         fops->destroy(dev, flow);
3892 }
3893
3894 /**
3895  * Flow driver find RSS policy tbl API. This abstracts calling driver
3896  * specific functions. Parent flow (rte_flow) should have driver
3897  * type (drv_type). It will find the RSS policy table that has the rss_desc.
3898  *
3899  * @param[in] dev
3900  *   Pointer to Ethernet device.
3901  * @param[in, out] flow
3902  *   Pointer to flow structure.
3903  * @param[in] policy
3904  *   Pointer to meter policy table.
3905  * @param[in] rss_desc
3906  *   Pointer to rss_desc
3907  */
3908 static struct mlx5_flow_meter_sub_policy *
3909 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
3910                 struct rte_flow *flow,
3911                 struct mlx5_flow_meter_policy *policy,
3912                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
3913 {
3914         const struct mlx5_flow_driver_ops *fops;
3915         enum mlx5_flow_drv_type type = flow->drv_type;
3916
3917         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3918         fops = flow_get_drv_ops(type);
3919         return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc);
3920 }
3921
3922 /**
3923  * Flow driver color tag rule API. This abstracts calling driver
3924  * specific functions. Parent flow (rte_flow) should have driver
3925  * type (drv_type). It will create the color tag rules in hierarchy meter.
3926  *
3927  * @param[in] dev
3928  *   Pointer to Ethernet device.
3929  * @param[in, out] flow
3930  *   Pointer to flow structure.
3931  * @param[in] fm
3932  *   Pointer to flow meter structure.
3933  * @param[in] src_port
3934  *   The src port this extra rule should use.
3935  * @param[in] item
3936  *   The src port id match item.
3937  * @param[out] error
3938  *   Pointer to error structure.
3939  */
3940 static int
3941 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev,
3942                 struct rte_flow *flow,
3943                 struct mlx5_flow_meter_info *fm,
3944                 int32_t src_port,
3945                 const struct rte_flow_item *item,
3946                 struct rte_flow_error *error)
3947 {
3948         const struct mlx5_flow_driver_ops *fops;
3949         enum mlx5_flow_drv_type type = flow->drv_type;
3950
3951         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3952         fops = flow_get_drv_ops(type);
3953         return fops->meter_hierarchy_rule_create(dev, fm,
3954                                                 src_port, item, error);
3955 }
3956
3957 /**
3958  * Get RSS action from the action list.
3959  *
3960  * @param[in] dev
3961  *   Pointer to Ethernet device.
3962  * @param[in] actions
3963  *   Pointer to the list of actions.
3964  * @param[in] flow
3965  *   Parent flow structure pointer.
3966  *
3967  * @return
3968  *   Pointer to the RSS action if exist, else return NULL.
3969  */
3970 static const struct rte_flow_action_rss*
3971 flow_get_rss_action(struct rte_eth_dev *dev,
3972                     const struct rte_flow_action actions[])
3973 {
3974         struct mlx5_priv *priv = dev->data->dev_private;
3975         const struct rte_flow_action_rss *rss = NULL;
3976         struct mlx5_meter_policy_action_container *acg;
3977         struct mlx5_meter_policy_action_container *acy;
3978
3979         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3980                 switch (actions->type) {
3981                 case RTE_FLOW_ACTION_TYPE_RSS:
3982                         rss = actions->conf;
3983                         break;
3984                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
3985                 {
3986                         const struct rte_flow_action_sample *sample =
3987                                                                 actions->conf;
3988                         const struct rte_flow_action *act = sample->actions;
3989                         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++)
3990                                 if (act->type == RTE_FLOW_ACTION_TYPE_RSS)
3991                                         rss = act->conf;
3992                         break;
3993                 }
3994                 case RTE_FLOW_ACTION_TYPE_METER:
3995                 {
3996                         uint32_t mtr_idx;
3997                         struct mlx5_flow_meter_info *fm;
3998                         struct mlx5_flow_meter_policy *policy;
3999                         const struct rte_flow_action_meter *mtr = actions->conf;
4000
4001                         fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx);
4002                         if (fm && !fm->def_policy) {
4003                                 policy = mlx5_flow_meter_policy_find(dev,
4004                                                 fm->policy_id, NULL);
4005                                 MLX5_ASSERT(policy);
4006                                 if (policy->is_hierarchy) {
4007                                         policy =
4008                                 mlx5_flow_meter_hierarchy_get_final_policy(dev,
4009                                                                         policy);
4010                                         if (!policy)
4011                                                 return NULL;
4012                                 }
4013                                 if (policy->is_rss) {
4014                                         acg =
4015                                         &policy->act_cnt[RTE_COLOR_GREEN];
4016                                         acy =
4017                                         &policy->act_cnt[RTE_COLOR_YELLOW];
4018                                         if (acg->fate_action ==
4019                                             MLX5_FLOW_FATE_SHARED_RSS)
4020                                                 rss = acg->rss->conf;
4021                                         else if (acy->fate_action ==
4022                                                  MLX5_FLOW_FATE_SHARED_RSS)
4023                                                 rss = acy->rss->conf;
4024                                 }
4025                         }
4026                         break;
4027                 }
4028                 default:
4029                         break;
4030                 }
4031         }
4032         return rss;
4033 }
4034
4035 /**
4036  * Get ASO age action by index.
4037  *
4038  * @param[in] dev
4039  *   Pointer to the Ethernet device structure.
4040  * @param[in] age_idx
4041  *   Index to the ASO age action.
4042  *
4043  * @return
4044  *   The specified ASO age action.
4045  */
4046 struct mlx5_aso_age_action*
4047 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx)
4048 {
4049         uint16_t pool_idx = age_idx & UINT16_MAX;
4050         uint16_t offset = (age_idx >> 16) & UINT16_MAX;
4051         struct mlx5_priv *priv = dev->data->dev_private;
4052         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
4053         struct mlx5_aso_age_pool *pool;
4054
4055         rte_rwlock_read_lock(&mng->resize_rwl);
4056         pool = mng->pools[pool_idx];
4057         rte_rwlock_read_unlock(&mng->resize_rwl);
4058         return &pool->actions[offset - 1];
4059 }
4060
4061 /* maps indirect action to translated direct in some actions array */
4062 struct mlx5_translated_action_handle {
4063         struct rte_flow_action_handle *action; /**< Indirect action handle. */
4064         int index; /**< Index in related array of rte_flow_action. */
4065 };
4066
4067 /**
4068  * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related
4069  * direct action if translation possible.
4070  * This functionality used to run same execution path for both direct and
4071  * indirect actions on flow create. All necessary preparations for indirect
4072  * action handling should be performed on *handle* actions list returned
4073  * from this call.
4074  *
4075  * @param[in] dev
4076  *   Pointer to Ethernet device.
4077  * @param[in] actions
4078  *   List of actions to translate.
4079  * @param[out] handle
4080  *   List to store translated indirect action object handles.
4081  * @param[in, out] indir_n
4082  *   Size of *handle* array. On return should be updated with number of
4083  *   indirect actions retrieved from the *actions* list.
4084  * @param[out] translated_actions
4085  *   List of actions where all indirect actions were translated to direct
4086  *   if possible. NULL if no translation took place.
4087  * @param[out] error
4088  *   Pointer to the error structure.
4089  *
4090  * @return
4091  *   0 on success, a negative errno value otherwise and rte_errno is set.
4092  */
4093 static int
4094 flow_action_handles_translate(struct rte_eth_dev *dev,
4095                               const struct rte_flow_action actions[],
4096                               struct mlx5_translated_action_handle *handle,
4097                               int *indir_n,
4098                               struct rte_flow_action **translated_actions,
4099                               struct rte_flow_error *error)
4100 {
4101         struct mlx5_priv *priv = dev->data->dev_private;
4102         struct rte_flow_action *translated = NULL;
4103         size_t actions_size;
4104         int n;
4105         int copied_n = 0;
4106         struct mlx5_translated_action_handle *handle_end = NULL;
4107
4108         for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
4109                 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT)
4110                         continue;
4111                 if (copied_n == *indir_n) {
4112                         return rte_flow_error_set
4113                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
4114                                  NULL, "too many shared actions");
4115                 }
4116                 rte_memcpy(&handle[copied_n].action, &actions[n].conf,
4117                            sizeof(actions[n].conf));
4118                 handle[copied_n].index = n;
4119                 copied_n++;
4120         }
4121         n++;
4122         *indir_n = copied_n;
4123         if (!copied_n)
4124                 return 0;
4125         actions_size = sizeof(struct rte_flow_action) * n;
4126         translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
4127         if (!translated) {
4128                 rte_errno = ENOMEM;
4129                 return -ENOMEM;
4130         }
4131         memcpy(translated, actions, actions_size);
4132         for (handle_end = handle + copied_n; handle < handle_end; handle++) {
4133                 struct mlx5_shared_action_rss *shared_rss;
4134                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
4135                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
4136                 uint32_t idx = act_idx &
4137                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
4138
4139                 switch (type) {
4140                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
4141                         shared_rss = mlx5_ipool_get
4142                           (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
4143                         translated[handle->index].type =
4144                                 RTE_FLOW_ACTION_TYPE_RSS;
4145                         translated[handle->index].conf =
4146                                 &shared_rss->origin;
4147                         break;
4148                 case MLX5_INDIRECT_ACTION_TYPE_COUNT:
4149                         translated[handle->index].type =
4150                                                 (enum rte_flow_action_type)
4151                                                 MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
4152                         translated[handle->index].conf = (void *)(uintptr_t)idx;
4153                         break;
4154                 case MLX5_INDIRECT_ACTION_TYPE_AGE:
4155                         if (priv->sh->flow_hit_aso_en) {
4156                                 translated[handle->index].type =
4157                                         (enum rte_flow_action_type)
4158                                         MLX5_RTE_FLOW_ACTION_TYPE_AGE;
4159                                 translated[handle->index].conf =
4160                                                          (void *)(uintptr_t)idx;
4161                                 break;
4162                         }
4163                         /* Fall-through */
4164                 case MLX5_INDIRECT_ACTION_TYPE_CT:
4165                         if (priv->sh->ct_aso_en) {
4166                                 translated[handle->index].type =
4167                                         RTE_FLOW_ACTION_TYPE_CONNTRACK;
4168                                 translated[handle->index].conf =
4169                                                          (void *)(uintptr_t)idx;
4170                                 break;
4171                         }
4172                         /* Fall-through */
4173                 default:
4174                         mlx5_free(translated);
4175                         return rte_flow_error_set
4176                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
4177                                  NULL, "invalid indirect action type");
4178                 }
4179         }
4180         *translated_actions = translated;
4181         return 0;
4182 }
4183
4184 /**
4185  * Get Shared RSS action from the action list.
4186  *
4187  * @param[in] dev
4188  *   Pointer to Ethernet device.
4189  * @param[in] shared
4190  *   Pointer to the list of actions.
4191  * @param[in] shared_n
4192  *   Actions list length.
4193  *
4194  * @return
4195  *   The MLX5 RSS action ID if exists, otherwise return 0.
4196  */
4197 static uint32_t
4198 flow_get_shared_rss_action(struct rte_eth_dev *dev,
4199                            struct mlx5_translated_action_handle *handle,
4200                            int shared_n)
4201 {
4202         struct mlx5_translated_action_handle *handle_end;
4203         struct mlx5_priv *priv = dev->data->dev_private;
4204         struct mlx5_shared_action_rss *shared_rss;
4205
4206
4207         for (handle_end = handle + shared_n; handle < handle_end; handle++) {
4208                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
4209                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
4210                 uint32_t idx = act_idx &
4211                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
4212                 switch (type) {
4213                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
4214                         shared_rss = mlx5_ipool_get
4215                                 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
4216                                                                            idx);
4217                         __atomic_add_fetch(&shared_rss->refcnt, 1,
4218                                            __ATOMIC_RELAXED);
4219                         return idx;
4220                 default:
4221                         break;
4222                 }
4223         }
4224         return 0;
4225 }
4226
4227 static unsigned int
4228 find_graph_root(uint32_t rss_level)
4229 {
4230         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
4231                                MLX5_EXPANSION_ROOT_OUTER;
4232 }
4233
4234 /**
4235  *  Get layer flags from the prefix flow.
4236  *
4237  *  Some flows may be split to several subflows, the prefix subflow gets the
4238  *  match items and the suffix sub flow gets the actions.
4239  *  Some actions need the user defined match item flags to get the detail for
4240  *  the action.
4241  *  This function helps the suffix flow to get the item layer flags from prefix
4242  *  subflow.
4243  *
4244  * @param[in] dev_flow
4245  *   Pointer the created prefix subflow.
4246  *
4247  * @return
4248  *   The layers get from prefix subflow.
4249  */
4250 static inline uint64_t
4251 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
4252 {
4253         uint64_t layers = 0;
4254
4255         /*
4256          * Layers bits could be localization, but usually the compiler will
4257          * help to do the optimization work for source code.
4258          * If no decap actions, use the layers directly.
4259          */
4260         if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
4261                 return dev_flow->handle->layers;
4262         /* Convert L3 layers with decap action. */
4263         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
4264                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
4265         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
4266                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
4267         /* Convert L4 layers with decap action.  */
4268         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
4269                 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
4270         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
4271                 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
4272         return layers;
4273 }
4274
4275 /**
4276  * Get metadata split action information.
4277  *
4278  * @param[in] actions
4279  *   Pointer to the list of actions.
4280  * @param[out] qrss
4281  *   Pointer to the return pointer.
4282  * @param[out] qrss_type
4283  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
4284  *   if no QUEUE/RSS is found.
4285  * @param[out] encap_idx
4286  *   Pointer to the index of the encap action if exists, otherwise the last
4287  *   action index.
4288  *
4289  * @return
4290  *   Total number of actions.
4291  */
4292 static int
4293 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
4294                                        const struct rte_flow_action **qrss,
4295                                        int *encap_idx)
4296 {
4297         const struct rte_flow_action_raw_encap *raw_encap;
4298         int actions_n = 0;
4299         int raw_decap_idx = -1;
4300
4301         *encap_idx = -1;
4302         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4303                 switch (actions->type) {
4304                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4305                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4306                         *encap_idx = actions_n;
4307                         break;
4308                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4309                         raw_decap_idx = actions_n;
4310                         break;
4311                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4312                         raw_encap = actions->conf;
4313                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4314                                 *encap_idx = raw_decap_idx != -1 ?
4315                                                       raw_decap_idx : actions_n;
4316                         break;
4317                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4318                 case RTE_FLOW_ACTION_TYPE_RSS:
4319                         *qrss = actions;
4320                         break;
4321                 default:
4322                         break;
4323                 }
4324                 actions_n++;
4325         }
4326         if (*encap_idx == -1)
4327                 *encap_idx = actions_n;
4328         /* Count RTE_FLOW_ACTION_TYPE_END. */
4329         return actions_n + 1;
4330 }
4331
4332 /**
4333  * Check if the action will change packet.
4334  *
4335  * @param dev
4336  *   Pointer to Ethernet device.
4337  * @param[in] type
4338  *   action type.
4339  *
4340  * @return
4341  *   true if action will change packet, false otherwise.
4342  */
4343 static bool flow_check_modify_action_type(struct rte_eth_dev *dev,
4344                                           enum rte_flow_action_type type)
4345 {
4346         struct mlx5_priv *priv = dev->data->dev_private;
4347
4348         switch (type) {
4349         case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
4350         case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
4351         case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
4352         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
4353         case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
4354         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
4355         case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
4356         case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
4357         case RTE_FLOW_ACTION_TYPE_DEC_TTL:
4358         case RTE_FLOW_ACTION_TYPE_SET_TTL:
4359         case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
4360         case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
4361         case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
4362         case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
4363         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
4364         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
4365         case RTE_FLOW_ACTION_TYPE_SET_META:
4366         case RTE_FLOW_ACTION_TYPE_SET_TAG:
4367         case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
4368         case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4369         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4370         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4371         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4372         case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4373         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4374         case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4375         case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4376         case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4377         case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
4378                 return true;
4379         case RTE_FLOW_ACTION_TYPE_FLAG:
4380         case RTE_FLOW_ACTION_TYPE_MARK:
4381                 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
4382                         return true;
4383                 else
4384                         return false;
4385         default:
4386                 return false;
4387         }
4388 }
4389
4390 /**
4391  * Check meter action from the action list.
4392  *
4393  * @param dev
4394  *   Pointer to Ethernet device.
4395  * @param[in] actions
4396  *   Pointer to the list of actions.
4397  * @param[out] has_mtr
4398  *   Pointer to the meter exist flag.
4399  * @param[out] has_modify
4400  *   Pointer to the flag showing there's packet change action.
4401  * @param[out] meter_id
4402  *   Pointer to the meter id.
4403  *
4404  * @return
4405  *   Total number of actions.
4406  */
4407 static int
4408 flow_check_meter_action(struct rte_eth_dev *dev,
4409                         const struct rte_flow_action actions[],
4410                         bool *has_mtr, bool *has_modify, uint32_t *meter_id)
4411 {
4412         const struct rte_flow_action_meter *mtr = NULL;
4413         int actions_n = 0;
4414
4415         MLX5_ASSERT(has_mtr);
4416         *has_mtr = false;
4417         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4418                 switch (actions->type) {
4419                 case RTE_FLOW_ACTION_TYPE_METER:
4420                         mtr = actions->conf;
4421                         *meter_id = mtr->mtr_id;
4422                         *has_mtr = true;
4423                         break;
4424                 default:
4425                         break;
4426                 }
4427                 if (!*has_mtr)
4428                         *has_modify |= flow_check_modify_action_type(dev,
4429                                                                 actions->type);
4430                 actions_n++;
4431         }
4432         /* Count RTE_FLOW_ACTION_TYPE_END. */
4433         return actions_n + 1;
4434 }
4435
4436 /**
4437  * Check if the flow should be split due to hairpin.
4438  * The reason for the split is that in current HW we can't
4439  * support encap and push-vlan on Rx, so if a flow contains
4440  * these actions we move it to Tx.
4441  *
4442  * @param dev
4443  *   Pointer to Ethernet device.
4444  * @param[in] attr
4445  *   Flow rule attributes.
4446  * @param[in] actions
4447  *   Associated actions (list terminated by the END action).
4448  *
4449  * @return
4450  *   > 0 the number of actions and the flow should be split,
4451  *   0 when no split required.
4452  */
4453 static int
4454 flow_check_hairpin_split(struct rte_eth_dev *dev,
4455                          const struct rte_flow_attr *attr,
4456                          const struct rte_flow_action actions[])
4457 {
4458         int queue_action = 0;
4459         int action_n = 0;
4460         int split = 0;
4461         const struct rte_flow_action_queue *queue;
4462         const struct rte_flow_action_rss *rss;
4463         const struct rte_flow_action_raw_encap *raw_encap;
4464         const struct rte_eth_hairpin_conf *conf;
4465
4466         if (!attr->ingress)
4467                 return 0;
4468         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4469                 switch (actions->type) {
4470                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4471                         queue = actions->conf;
4472                         if (queue == NULL)
4473                                 return 0;
4474                         conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
4475                         if (conf == NULL || conf->tx_explicit != 0)
4476                                 return 0;
4477                         queue_action = 1;
4478                         action_n++;
4479                         break;
4480                 case RTE_FLOW_ACTION_TYPE_RSS:
4481                         rss = actions->conf;
4482                         if (rss == NULL || rss->queue_num == 0)
4483                                 return 0;
4484                         conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
4485                         if (conf == NULL || conf->tx_explicit != 0)
4486                                 return 0;
4487                         queue_action = 1;
4488                         action_n++;
4489                         break;
4490                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4491                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4492                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4493                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4494                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4495                         split++;
4496                         action_n++;
4497                         break;
4498                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4499                         raw_encap = actions->conf;
4500                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4501                                 split++;
4502                         action_n++;
4503                         break;
4504                 default:
4505                         action_n++;
4506                         break;
4507                 }
4508         }
4509         if (split && queue_action)
4510                 return action_n;
4511         return 0;
4512 }
4513
4514 /* Declare flow create/destroy prototype in advance. */
4515 static uint32_t
4516 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
4517                  const struct rte_flow_attr *attr,
4518                  const struct rte_flow_item items[],
4519                  const struct rte_flow_action actions[],
4520                  bool external, struct rte_flow_error *error);
4521
4522 static void
4523 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
4524                   uint32_t flow_idx);
4525
4526 int
4527 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused,
4528                       struct mlx5_list_entry *entry, void *cb_ctx)
4529 {
4530         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4531         struct mlx5_flow_mreg_copy_resource *mcp_res =
4532                                container_of(entry, typeof(*mcp_res), hlist_ent);
4533
4534         return mcp_res->mark_id != *(uint32_t *)(ctx->data);
4535 }
4536
4537 struct mlx5_list_entry *
4538 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx)
4539 {
4540         struct rte_eth_dev *dev = tool_ctx;
4541         struct mlx5_priv *priv = dev->data->dev_private;
4542         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4543         struct mlx5_flow_mreg_copy_resource *mcp_res;
4544         struct rte_flow_error *error = ctx->error;
4545         uint32_t idx = 0;
4546         int ret;
4547         uint32_t mark_id = *(uint32_t *)(ctx->data);
4548         struct rte_flow_attr attr = {
4549                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4550                 .ingress = 1,
4551         };
4552         struct mlx5_rte_flow_item_tag tag_spec = {
4553                 .data = mark_id,
4554         };
4555         struct rte_flow_item items[] = {
4556                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
4557         };
4558         struct rte_flow_action_mark ftag = {
4559                 .id = mark_id,
4560         };
4561         struct mlx5_flow_action_copy_mreg cp_mreg = {
4562                 .dst = REG_B,
4563                 .src = REG_NON,
4564         };
4565         struct rte_flow_action_jump jump = {
4566                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4567         };
4568         struct rte_flow_action actions[] = {
4569                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
4570         };
4571
4572         /* Fill the register fields in the flow. */
4573         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
4574         if (ret < 0)
4575                 return NULL;
4576         tag_spec.id = ret;
4577         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4578         if (ret < 0)
4579                 return NULL;
4580         cp_mreg.src = ret;
4581         /* Provide the full width of FLAG specific value. */
4582         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
4583                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
4584         /* Build a new flow. */
4585         if (mark_id != MLX5_DEFAULT_COPY_ID) {
4586                 items[0] = (struct rte_flow_item){
4587                         .type = (enum rte_flow_item_type)
4588                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4589                         .spec = &tag_spec,
4590                 };
4591                 items[1] = (struct rte_flow_item){
4592                         .type = RTE_FLOW_ITEM_TYPE_END,
4593                 };
4594                 actions[0] = (struct rte_flow_action){
4595                         .type = (enum rte_flow_action_type)
4596                                 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
4597                         .conf = &ftag,
4598                 };
4599                 actions[1] = (struct rte_flow_action){
4600                         .type = (enum rte_flow_action_type)
4601                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4602                         .conf = &cp_mreg,
4603                 };
4604                 actions[2] = (struct rte_flow_action){
4605                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4606                         .conf = &jump,
4607                 };
4608                 actions[3] = (struct rte_flow_action){
4609                         .type = RTE_FLOW_ACTION_TYPE_END,
4610                 };
4611         } else {
4612                 /* Default rule, wildcard match. */
4613                 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR;
4614                 items[0] = (struct rte_flow_item){
4615                         .type = RTE_FLOW_ITEM_TYPE_END,
4616                 };
4617                 actions[0] = (struct rte_flow_action){
4618                         .type = (enum rte_flow_action_type)
4619                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4620                         .conf = &cp_mreg,
4621                 };
4622                 actions[1] = (struct rte_flow_action){
4623                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4624                         .conf = &jump,
4625                 };
4626                 actions[2] = (struct rte_flow_action){
4627                         .type = RTE_FLOW_ACTION_TYPE_END,
4628                 };
4629         }
4630         /* Build a new entry. */
4631         mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
4632         if (!mcp_res) {
4633                 rte_errno = ENOMEM;
4634                 return NULL;
4635         }
4636         mcp_res->idx = idx;
4637         mcp_res->mark_id = mark_id;
4638         /*
4639          * The copy Flows are not included in any list. There
4640          * ones are referenced from other Flows and can not
4641          * be applied, removed, deleted in arbitrary order
4642          * by list traversing.
4643          */
4644         mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP,
4645                                         &attr, items, actions, false, error);
4646         if (!mcp_res->rix_flow) {
4647                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
4648                 return NULL;
4649         }
4650         return &mcp_res->hlist_ent;
4651 }
4652
4653 struct mlx5_list_entry *
4654 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
4655                       void *cb_ctx __rte_unused)
4656 {
4657         struct rte_eth_dev *dev = tool_ctx;
4658         struct mlx5_priv *priv = dev->data->dev_private;
4659         struct mlx5_flow_mreg_copy_resource *mcp_res;
4660         uint32_t idx = 0;
4661
4662         mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
4663         if (!mcp_res) {
4664                 rte_errno = ENOMEM;
4665                 return NULL;
4666         }
4667         memcpy(mcp_res, oentry, sizeof(*mcp_res));
4668         mcp_res->idx = idx;
4669         return &mcp_res->hlist_ent;
4670 }
4671
4672 void
4673 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
4674 {
4675         struct mlx5_flow_mreg_copy_resource *mcp_res =
4676                                container_of(entry, typeof(*mcp_res), hlist_ent);
4677         struct rte_eth_dev *dev = tool_ctx;
4678         struct mlx5_priv *priv = dev->data->dev_private;
4679
4680         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
4681 }
4682
4683 /**
4684  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4685  *
4686  * As mark_id is unique, if there's already a registered flow for the mark_id,
4687  * return by increasing the reference counter of the resource. Otherwise, create
4688  * the resource (mcp_res) and flow.
4689  *
4690  * Flow looks like,
4691  *   - If ingress port is ANY and reg_c[1] is mark_id,
4692  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4693  *
4694  * For default flow (zero mark_id), flow is like,
4695  *   - If ingress port is ANY,
4696  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
4697  *
4698  * @param dev
4699  *   Pointer to Ethernet device.
4700  * @param mark_id
4701  *   ID of MARK action, zero means default flow for META.
4702  * @param[out] error
4703  *   Perform verbose error reporting if not NULL.
4704  *
4705  * @return
4706  *   Associated resource on success, NULL otherwise and rte_errno is set.
4707  */
4708 static struct mlx5_flow_mreg_copy_resource *
4709 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
4710                           struct rte_flow_error *error)
4711 {
4712         struct mlx5_priv *priv = dev->data->dev_private;
4713         struct mlx5_list_entry *entry;
4714         struct mlx5_flow_cb_ctx ctx = {
4715                 .dev = dev,
4716                 .error = error,
4717                 .data = &mark_id,
4718         };
4719
4720         /* Check if already registered. */
4721         MLX5_ASSERT(priv->mreg_cp_tbl);
4722         entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx);
4723         if (!entry)
4724                 return NULL;
4725         return container_of(entry, struct mlx5_flow_mreg_copy_resource,
4726                             hlist_ent);
4727 }
4728
4729 void
4730 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
4731 {
4732         struct mlx5_flow_mreg_copy_resource *mcp_res =
4733                                container_of(entry, typeof(*mcp_res), hlist_ent);
4734         struct rte_eth_dev *dev = tool_ctx;
4735         struct mlx5_priv *priv = dev->data->dev_private;
4736
4737         MLX5_ASSERT(mcp_res->rix_flow);
4738         flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow);
4739         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
4740 }
4741
4742 /**
4743  * Release flow in RX_CP_TBL.
4744  *
4745  * @param dev
4746  *   Pointer to Ethernet device.
4747  * @flow
4748  *   Parent flow for wich copying is provided.
4749  */
4750 static void
4751 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
4752                           struct rte_flow *flow)
4753 {
4754         struct mlx5_flow_mreg_copy_resource *mcp_res;
4755         struct mlx5_priv *priv = dev->data->dev_private;
4756
4757         if (!flow->rix_mreg_copy)
4758                 return;
4759         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
4760                                  flow->rix_mreg_copy);
4761         if (!mcp_res || !priv->mreg_cp_tbl)
4762                 return;
4763         MLX5_ASSERT(mcp_res->rix_flow);
4764         mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
4765         flow->rix_mreg_copy = 0;
4766 }
4767
4768 /**
4769  * Remove the default copy action from RX_CP_TBL.
4770  *
4771  * This functions is called in the mlx5_dev_start(). No thread safe
4772  * is guaranteed.
4773  *
4774  * @param dev
4775  *   Pointer to Ethernet device.
4776  */
4777 static void
4778 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
4779 {
4780         struct mlx5_list_entry *entry;
4781         struct mlx5_priv *priv = dev->data->dev_private;
4782         struct mlx5_flow_cb_ctx ctx;
4783         uint32_t mark_id;
4784
4785         /* Check if default flow is registered. */
4786         if (!priv->mreg_cp_tbl)
4787                 return;
4788         mark_id = MLX5_DEFAULT_COPY_ID;
4789         ctx.data = &mark_id;
4790         entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx);
4791         if (!entry)
4792                 return;
4793         mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
4794 }
4795
4796 /**
4797  * Add the default copy action in in RX_CP_TBL.
4798  *
4799  * This functions is called in the mlx5_dev_start(). No thread safe
4800  * is guaranteed.
4801  *
4802  * @param dev
4803  *   Pointer to Ethernet device.
4804  * @param[out] error
4805  *   Perform verbose error reporting if not NULL.
4806  *
4807  * @return
4808  *   0 for success, negative value otherwise and rte_errno is set.
4809  */
4810 static int
4811 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
4812                                   struct rte_flow_error *error)
4813 {
4814         struct mlx5_priv *priv = dev->data->dev_private;
4815         struct mlx5_flow_mreg_copy_resource *mcp_res;
4816         struct mlx5_flow_cb_ctx ctx;
4817         uint32_t mark_id;
4818
4819         /* Check whether extensive metadata feature is engaged. */
4820         if (!priv->sh->config.dv_flow_en ||
4821             priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4822             !mlx5_flow_ext_mreg_supported(dev) ||
4823             !priv->sh->dv_regc0_mask)
4824                 return 0;
4825         /*
4826          * Add default mreg copy flow may be called multiple time, but
4827          * only be called once in stop. Avoid register it twice.
4828          */
4829         mark_id = MLX5_DEFAULT_COPY_ID;
4830         ctx.data = &mark_id;
4831         if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx))
4832                 return 0;
4833         mcp_res = flow_mreg_add_copy_action(dev, mark_id, error);
4834         if (!mcp_res)
4835                 return -rte_errno;
4836         return 0;
4837 }
4838
4839 /**
4840  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4841  *
4842  * All the flow having Q/RSS action should be split by
4843  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
4844  * performs the following,
4845  *   - CQE->flow_tag := reg_c[1] (MARK)
4846  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4847  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
4848  * but there should be a flow per each MARK ID set by MARK action.
4849  *
4850  * For the aforementioned reason, if there's a MARK action in flow's action
4851  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
4852  * the MARK ID to CQE's flow_tag like,
4853  *   - If reg_c[1] is mark_id,
4854  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4855  *
4856  * For SET_META action which stores value in reg_c[0], as the destination is
4857  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
4858  * MARK ID means the default flow. The default flow looks like,
4859  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4860  *
4861  * @param dev
4862  *   Pointer to Ethernet device.
4863  * @param flow
4864  *   Pointer to flow structure.
4865  * @param[in] actions
4866  *   Pointer to the list of actions.
4867  * @param[out] error
4868  *   Perform verbose error reporting if not NULL.
4869  *
4870  * @return
4871  *   0 on success, negative value otherwise and rte_errno is set.
4872  */
4873 static int
4874 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
4875                             struct rte_flow *flow,
4876                             const struct rte_flow_action *actions,
4877                             struct rte_flow_error *error)
4878 {
4879         struct mlx5_priv *priv = dev->data->dev_private;
4880         struct mlx5_sh_config *config = &priv->sh->config;
4881         struct mlx5_flow_mreg_copy_resource *mcp_res;
4882         const struct rte_flow_action_mark *mark;
4883
4884         /* Check whether extensive metadata feature is engaged. */
4885         if (!config->dv_flow_en ||
4886             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4887             !mlx5_flow_ext_mreg_supported(dev) ||
4888             !priv->sh->dv_regc0_mask)
4889                 return 0;
4890         /* Find MARK action. */
4891         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4892                 switch (actions->type) {
4893                 case RTE_FLOW_ACTION_TYPE_FLAG:
4894                         mcp_res = flow_mreg_add_copy_action
4895                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
4896                         if (!mcp_res)
4897                                 return -rte_errno;
4898                         flow->rix_mreg_copy = mcp_res->idx;
4899                         return 0;
4900                 case RTE_FLOW_ACTION_TYPE_MARK:
4901                         mark = (const struct rte_flow_action_mark *)
4902                                 actions->conf;
4903                         mcp_res =
4904                                 flow_mreg_add_copy_action(dev, mark->id, error);
4905                         if (!mcp_res)
4906                                 return -rte_errno;
4907                         flow->rix_mreg_copy = mcp_res->idx;
4908                         return 0;
4909                 default:
4910                         break;
4911                 }
4912         }
4913         return 0;
4914 }
4915
4916 #define MLX5_MAX_SPLIT_ACTIONS 24
4917 #define MLX5_MAX_SPLIT_ITEMS 24
4918
4919 /**
4920  * Split the hairpin flow.
4921  * Since HW can't support encap and push-vlan on Rx, we move these
4922  * actions to Tx.
4923  * If the count action is after the encap then we also
4924  * move the count action. in this case the count will also measure
4925  * the outer bytes.
4926  *
4927  * @param dev
4928  *   Pointer to Ethernet device.
4929  * @param[in] actions
4930  *   Associated actions (list terminated by the END action).
4931  * @param[out] actions_rx
4932  *   Rx flow actions.
4933  * @param[out] actions_tx
4934  *   Tx flow actions..
4935  * @param[out] pattern_tx
4936  *   The pattern items for the Tx flow.
4937  * @param[out] flow_id
4938  *   The flow ID connected to this flow.
4939  *
4940  * @return
4941  *   0 on success.
4942  */
4943 static int
4944 flow_hairpin_split(struct rte_eth_dev *dev,
4945                    const struct rte_flow_action actions[],
4946                    struct rte_flow_action actions_rx[],
4947                    struct rte_flow_action actions_tx[],
4948                    struct rte_flow_item pattern_tx[],
4949                    uint32_t flow_id)
4950 {
4951         const struct rte_flow_action_raw_encap *raw_encap;
4952         const struct rte_flow_action_raw_decap *raw_decap;
4953         struct mlx5_rte_flow_action_set_tag *set_tag;
4954         struct rte_flow_action *tag_action;
4955         struct mlx5_rte_flow_item_tag *tag_item;
4956         struct rte_flow_item *item;
4957         char *addr;
4958         int encap = 0;
4959
4960         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4961                 switch (actions->type) {
4962                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4963                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4964                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4965                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4966                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4967                         rte_memcpy(actions_tx, actions,
4968                                sizeof(struct rte_flow_action));
4969                         actions_tx++;
4970                         break;
4971                 case RTE_FLOW_ACTION_TYPE_COUNT:
4972                         if (encap) {
4973                                 rte_memcpy(actions_tx, actions,
4974                                            sizeof(struct rte_flow_action));
4975                                 actions_tx++;
4976                         } else {
4977                                 rte_memcpy(actions_rx, actions,
4978                                            sizeof(struct rte_flow_action));
4979                                 actions_rx++;
4980                         }
4981                         break;
4982                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4983                         raw_encap = actions->conf;
4984                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) {
4985                                 memcpy(actions_tx, actions,
4986                                        sizeof(struct rte_flow_action));
4987                                 actions_tx++;
4988                                 encap = 1;
4989                         } else {
4990                                 rte_memcpy(actions_rx, actions,
4991                                            sizeof(struct rte_flow_action));
4992                                 actions_rx++;
4993                         }
4994                         break;
4995                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4996                         raw_decap = actions->conf;
4997                         if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) {
4998                                 memcpy(actions_tx, actions,
4999                                        sizeof(struct rte_flow_action));
5000                                 actions_tx++;
5001                         } else {
5002                                 rte_memcpy(actions_rx, actions,
5003                                            sizeof(struct rte_flow_action));
5004                                 actions_rx++;
5005                         }
5006                         break;
5007                 default:
5008                         rte_memcpy(actions_rx, actions,
5009                                    sizeof(struct rte_flow_action));
5010                         actions_rx++;
5011                         break;
5012                 }
5013         }
5014         /* Add set meta action and end action for the Rx flow. */
5015         tag_action = actions_rx;
5016         tag_action->type = (enum rte_flow_action_type)
5017                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5018         actions_rx++;
5019         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
5020         actions_rx++;
5021         set_tag = (void *)actions_rx;
5022         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
5023                 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL),
5024                 .data = flow_id,
5025         };
5026         MLX5_ASSERT(set_tag->id > REG_NON);
5027         tag_action->conf = set_tag;
5028         /* Create Tx item list. */
5029         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
5030         addr = (void *)&pattern_tx[2];
5031         item = pattern_tx;
5032         item->type = (enum rte_flow_item_type)
5033                      MLX5_RTE_FLOW_ITEM_TYPE_TAG;
5034         tag_item = (void *)addr;
5035         tag_item->data = flow_id;
5036         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
5037         MLX5_ASSERT(set_tag->id > REG_NON);
5038         item->spec = tag_item;
5039         addr += sizeof(struct mlx5_rte_flow_item_tag);
5040         tag_item = (void *)addr;
5041         tag_item->data = UINT32_MAX;
5042         tag_item->id = UINT16_MAX;
5043         item->mask = tag_item;
5044         item->last = NULL;
5045         item++;
5046         item->type = RTE_FLOW_ITEM_TYPE_END;
5047         return 0;
5048 }
5049
5050 /**
5051  * The last stage of splitting chain, just creates the subflow
5052  * without any modification.
5053  *
5054  * @param[in] dev
5055  *   Pointer to Ethernet device.
5056  * @param[in] flow
5057  *   Parent flow structure pointer.
5058  * @param[in, out] sub_flow
5059  *   Pointer to return the created subflow, may be NULL.
5060  * @param[in] attr
5061  *   Flow rule attributes.
5062  * @param[in] items
5063  *   Pattern specification (list terminated by the END pattern item).
5064  * @param[in] actions
5065  *   Associated actions (list terminated by the END action).
5066  * @param[in] flow_split_info
5067  *   Pointer to flow split info structure.
5068  * @param[out] error
5069  *   Perform verbose error reporting if not NULL.
5070  * @return
5071  *   0 on success, negative value otherwise
5072  */
5073 static int
5074 flow_create_split_inner(struct rte_eth_dev *dev,
5075                         struct rte_flow *flow,
5076                         struct mlx5_flow **sub_flow,
5077                         const struct rte_flow_attr *attr,
5078                         const struct rte_flow_item items[],
5079                         const struct rte_flow_action actions[],
5080                         struct mlx5_flow_split_info *flow_split_info,
5081                         struct rte_flow_error *error)
5082 {
5083         struct mlx5_flow *dev_flow;
5084         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
5085
5086         dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
5087                                     flow_split_info->flow_idx, error);
5088         if (!dev_flow)
5089                 return -rte_errno;
5090         dev_flow->flow = flow;
5091         dev_flow->external = flow_split_info->external;
5092         dev_flow->skip_scale = flow_split_info->skip_scale;
5093         /* Subflow object was created, we must include one in the list. */
5094         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
5095                       dev_flow->handle, next);
5096         /*
5097          * If dev_flow is as one of the suffix flow, some actions in suffix
5098          * flow may need some user defined item layer flags, and pass the
5099          * Metadata rxq mark flag to suffix flow as well.
5100          */
5101         if (flow_split_info->prefix_layers)
5102                 dev_flow->handle->layers = flow_split_info->prefix_layers;
5103         if (flow_split_info->prefix_mark) {
5104                 MLX5_ASSERT(wks);
5105                 wks->mark = 1;
5106         }
5107         if (sub_flow)
5108                 *sub_flow = dev_flow;
5109 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5110         dev_flow->dv.table_id = flow_split_info->table_id;
5111 #endif
5112         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
5113 }
5114
5115 /**
5116  * Get the sub policy of a meter.
5117  *
5118  * @param[in] dev
5119  *   Pointer to Ethernet device.
5120  * @param[in] flow
5121  *   Parent flow structure pointer.
5122  * @param wks
5123  *   Pointer to thread flow work space.
5124  * @param[in] attr
5125  *   Flow rule attributes.
5126  * @param[in] items
5127  *   Pattern specification (list terminated by the END pattern item).
5128  * @param[out] error
5129  *   Perform verbose error reporting if not NULL.
5130  *
5131  * @return
5132  *   Pointer to the meter sub policy, NULL otherwise and rte_errno is set.
5133  */
5134 static struct mlx5_flow_meter_sub_policy *
5135 get_meter_sub_policy(struct rte_eth_dev *dev,
5136                      struct rte_flow *flow,
5137                      struct mlx5_flow_workspace *wks,
5138                      const struct rte_flow_attr *attr,
5139                      const struct rte_flow_item items[],
5140                      struct rte_flow_error *error)
5141 {
5142         struct mlx5_flow_meter_policy *policy;
5143         struct mlx5_flow_meter_policy *final_policy;
5144         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
5145
5146         policy = wks->policy;
5147         final_policy = policy->is_hierarchy ? wks->final_policy : policy;
5148         if (final_policy->is_rss || final_policy->is_queue) {
5149                 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS];
5150                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0};
5151                 uint32_t i;
5152
5153                 /*
5154                  * This is a tmp dev_flow,
5155                  * no need to register any matcher for it in translate.
5156                  */
5157                 wks->skip_matcher_reg = 1;
5158                 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
5159                         struct mlx5_flow dev_flow = {0};
5160                         struct mlx5_flow_handle dev_handle = { {0} };
5161                         uint8_t fate = final_policy->act_cnt[i].fate_action;
5162
5163                         if (fate == MLX5_FLOW_FATE_SHARED_RSS) {
5164                                 const struct rte_flow_action_rss *rss_act =
5165                                         final_policy->act_cnt[i].rss->conf;
5166                                 struct rte_flow_action rss_actions[2] = {
5167                                         [0] = {
5168                                         .type = RTE_FLOW_ACTION_TYPE_RSS,
5169                                         .conf = rss_act,
5170                                         },
5171                                         [1] = {
5172                                         .type = RTE_FLOW_ACTION_TYPE_END,
5173                                         .conf = NULL,
5174                                         }
5175                                 };
5176
5177                                 dev_flow.handle = &dev_handle;
5178                                 dev_flow.ingress = attr->ingress;
5179                                 dev_flow.flow = flow;
5180                                 dev_flow.external = 0;
5181 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5182                                 dev_flow.dv.transfer = attr->transfer;
5183 #endif
5184                                 /**
5185                                  * Translate RSS action to get rss hash fields.
5186                                  */
5187                                 if (flow_drv_translate(dev, &dev_flow, attr,
5188                                                 items, rss_actions, error))
5189                                         goto exit;
5190                                 rss_desc_v[i] = wks->rss_desc;
5191                                 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN;
5192                                 rss_desc_v[i].hash_fields =
5193                                                 dev_flow.hash_fields;
5194                                 rss_desc_v[i].queue_num =
5195                                                 rss_desc_v[i].hash_fields ?
5196                                                 rss_desc_v[i].queue_num : 1;
5197                                 rss_desc_v[i].tunnel =
5198                                                 !!(dev_flow.handle->layers &
5199                                                    MLX5_FLOW_LAYER_TUNNEL);
5200                                 /* Use the RSS queues in the containers. */
5201                                 rss_desc_v[i].queue =
5202                                         (uint16_t *)(uintptr_t)rss_act->queue;
5203                                 rss_desc[i] = &rss_desc_v[i];
5204                         } else if (fate == MLX5_FLOW_FATE_QUEUE) {
5205                                 /* This is queue action. */
5206                                 rss_desc_v[i] = wks->rss_desc;
5207                                 rss_desc_v[i].key_len = 0;
5208                                 rss_desc_v[i].hash_fields = 0;
5209                                 rss_desc_v[i].queue =
5210                                         &final_policy->act_cnt[i].queue;
5211                                 rss_desc_v[i].queue_num = 1;
5212                                 rss_desc[i] = &rss_desc_v[i];
5213                         } else {
5214                                 rss_desc[i] = NULL;
5215                         }
5216                 }
5217                 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev,
5218                                                 flow, policy, rss_desc);
5219         } else {
5220                 enum mlx5_meter_domain mtr_domain =
5221                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
5222                                 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
5223                                                 MLX5_MTR_DOMAIN_INGRESS);
5224                 sub_policy = policy->sub_policys[mtr_domain][0];
5225         }
5226         if (!sub_policy)
5227                 rte_flow_error_set(error, EINVAL,
5228                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5229                                    "Failed to get meter sub-policy.");
5230 exit:
5231         return sub_policy;
5232 }
5233
5234 /**
5235  * Split the meter flow.
5236  *
5237  * As meter flow will split to three sub flow, other than meter
5238  * action, the other actions make sense to only meter accepts
5239  * the packet. If it need to be dropped, no other additional
5240  * actions should be take.
5241  *
5242  * One kind of special action which decapsulates the L3 tunnel
5243  * header will be in the prefix sub flow, as not to take the
5244  * L3 tunnel header into account.
5245  *
5246  * @param[in] dev
5247  *   Pointer to Ethernet device.
5248  * @param[in] flow
5249  *   Parent flow structure pointer.
5250  * @param wks
5251  *   Pointer to thread flow work space.
5252  * @param[in] attr
5253  *   Flow rule attributes.
5254  * @param[in] items
5255  *   Pattern specification (list terminated by the END pattern item).
5256  * @param[out] sfx_items
5257  *   Suffix flow match items (list terminated by the END pattern item).
5258  * @param[in] actions
5259  *   Associated actions (list terminated by the END action).
5260  * @param[out] actions_sfx
5261  *   Suffix flow actions.
5262  * @param[out] actions_pre
5263  *   Prefix flow actions.
5264  * @param[out] mtr_flow_id
5265  *   Pointer to meter flow id.
5266  * @param[out] error
5267  *   Perform verbose error reporting if not NULL.
5268  *
5269  * @return
5270  *   0 on success, a negative errno value otherwise and rte_errno is set.
5271  */
5272 static int
5273 flow_meter_split_prep(struct rte_eth_dev *dev,
5274                       struct rte_flow *flow,
5275                       struct mlx5_flow_workspace *wks,
5276                       const struct rte_flow_attr *attr,
5277                       const struct rte_flow_item items[],
5278                       struct rte_flow_item sfx_items[],
5279                       const struct rte_flow_action actions[],
5280                       struct rte_flow_action actions_sfx[],
5281                       struct rte_flow_action actions_pre[],
5282                       uint32_t *mtr_flow_id,
5283                       struct rte_flow_error *error)
5284 {
5285         struct mlx5_priv *priv = dev->data->dev_private;
5286         struct mlx5_flow_meter_info *fm = wks->fm;
5287         struct rte_flow_action *tag_action = NULL;
5288         struct rte_flow_item *tag_item;
5289         struct mlx5_rte_flow_action_set_tag *set_tag;
5290         const struct rte_flow_action_raw_encap *raw_encap;
5291         const struct rte_flow_action_raw_decap *raw_decap;
5292         struct mlx5_rte_flow_item_tag *tag_item_spec;
5293         struct mlx5_rte_flow_item_tag *tag_item_mask;
5294         uint32_t tag_id = 0;
5295         struct rte_flow_item *vlan_item_dst = NULL;
5296         const struct rte_flow_item *vlan_item_src = NULL;
5297         const struct rte_flow_item *orig_items = items;
5298         struct rte_flow_action *hw_mtr_action;
5299         struct rte_flow_action *action_pre_head = NULL;
5300         int32_t flow_src_port = priv->representor_id;
5301         bool mtr_first;
5302         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
5303         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
5304                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
5305         uint32_t flow_id = 0;
5306         uint32_t flow_id_reversed = 0;
5307         uint8_t flow_id_bits = 0;
5308         bool after_meter = false;
5309         int shift;
5310
5311         /* Prepare the suffix subflow items. */
5312         tag_item = sfx_items++;
5313         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5314                 struct mlx5_priv *port_priv;
5315                 const struct rte_flow_item_port_id *pid_v;
5316                 int item_type = items->type;
5317
5318                 switch (item_type) {
5319                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
5320                         pid_v = items->spec;
5321                         MLX5_ASSERT(pid_v);
5322                         port_priv = mlx5_port_to_eswitch_info(pid_v->id, false);
5323                         if (!port_priv)
5324                                 return rte_flow_error_set(error,
5325                                                 rte_errno,
5326                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5327                                                 pid_v,
5328                                                 "Failed to get port info.");
5329                         flow_src_port = port_priv->representor_id;
5330                         if (!fm->def_policy && wks->policy->is_hierarchy &&
5331                             flow_src_port != priv->representor_id) {
5332                                 if (flow_drv_mtr_hierarchy_rule_create(dev,
5333                                                                 flow, fm,
5334                                                                 flow_src_port,
5335                                                                 items,
5336                                                                 error))
5337                                         return -rte_errno;
5338                         }
5339                         memcpy(sfx_items, items, sizeof(*sfx_items));
5340                         sfx_items++;
5341                         break;
5342                 case RTE_FLOW_ITEM_TYPE_VLAN:
5343                         /* Determine if copy vlan item below. */
5344                         vlan_item_src = items;
5345                         vlan_item_dst = sfx_items++;
5346                         vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID;
5347                         break;
5348                 default:
5349                         break;
5350                 }
5351         }
5352         sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
5353         sfx_items++;
5354         mtr_first = priv->sh->meter_aso_en &&
5355                 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX));
5356         /* For ASO meter, meter must be before tag in TX direction. */
5357         if (mtr_first) {
5358                 action_pre_head = actions_pre++;
5359                 /* Leave space for tag action. */
5360                 tag_action = actions_pre++;
5361         }
5362         /* Prepare the actions for prefix and suffix flow. */
5363         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5364                 struct rte_flow_action *action_cur = NULL;
5365
5366                 switch (actions->type) {
5367                 case RTE_FLOW_ACTION_TYPE_METER:
5368                         if (mtr_first) {
5369                                 action_cur = action_pre_head;
5370                         } else {
5371                                 /* Leave space for tag action. */
5372                                 tag_action = actions_pre++;
5373                                 action_cur = actions_pre++;
5374                         }
5375                         after_meter = true;
5376                         break;
5377                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5378                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5379                         action_cur = actions_pre++;
5380                         break;
5381                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5382                         raw_encap = actions->conf;
5383                         if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
5384                                 action_cur = actions_pre++;
5385                         break;
5386                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5387                         raw_decap = actions->conf;
5388                         if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
5389                                 action_cur = actions_pre++;
5390                         break;
5391                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5392                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5393                         if (vlan_item_dst && vlan_item_src) {
5394                                 memcpy(vlan_item_dst, vlan_item_src,
5395                                         sizeof(*vlan_item_dst));
5396                                 /*
5397                                  * Convert to internal match item, it is used
5398                                  * for vlan push and set vid.
5399                                  */
5400                                 vlan_item_dst->type = (enum rte_flow_item_type)
5401                                                 MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
5402                         }
5403                         break;
5404                 case RTE_FLOW_ACTION_TYPE_COUNT:
5405                         if (fm->def_policy)
5406                                 action_cur = after_meter ?
5407                                                 actions_sfx++ : actions_pre++;
5408                         break;
5409                 default:
5410                         break;
5411                 }
5412                 if (!action_cur)
5413                         action_cur = (fm->def_policy) ?
5414                                         actions_sfx++ : actions_pre++;
5415                 memcpy(action_cur, actions, sizeof(struct rte_flow_action));
5416         }
5417         /* Add end action to the actions. */
5418         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
5419         if (priv->sh->meter_aso_en) {
5420                 /**
5421                  * For ASO meter, need to add an extra jump action explicitly,
5422                  * to jump from meter to policer table.
5423                  */
5424                 struct mlx5_flow_meter_sub_policy *sub_policy;
5425                 struct mlx5_flow_tbl_data_entry *tbl_data;
5426
5427                 if (!fm->def_policy) {
5428                         sub_policy = get_meter_sub_policy(dev, flow, wks,
5429                                                           attr, orig_items,
5430                                                           error);
5431                         if (!sub_policy)
5432                                 return -rte_errno;
5433                 } else {
5434                         enum mlx5_meter_domain mtr_domain =
5435                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
5436                                 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
5437                                                 MLX5_MTR_DOMAIN_INGRESS);
5438
5439                         sub_policy =
5440                         &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy;
5441                 }
5442                 tbl_data = container_of(sub_policy->tbl_rsc,
5443                                         struct mlx5_flow_tbl_data_entry, tbl);
5444                 hw_mtr_action = actions_pre++;
5445                 hw_mtr_action->type = (enum rte_flow_action_type)
5446                                       MLX5_RTE_FLOW_ACTION_TYPE_JUMP;
5447                 hw_mtr_action->conf = tbl_data->jump.action;
5448         }
5449         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
5450         actions_pre++;
5451         if (!tag_action)
5452                 return rte_flow_error_set(error, ENOMEM,
5453                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5454                                           NULL, "No tag action space.");
5455         if (!mtr_flow_id) {
5456                 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID;
5457                 goto exit;
5458         }
5459         /* Only default-policy Meter creates mtr flow id. */
5460         if (fm->def_policy) {
5461                 mlx5_ipool_malloc(fm->flow_ipool, &tag_id);
5462                 if (!tag_id)
5463                         return rte_flow_error_set(error, ENOMEM,
5464                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5465                                         "Failed to allocate meter flow id.");
5466                 flow_id = tag_id - 1;
5467                 flow_id_bits = (!flow_id) ? 1 :
5468                                 (MLX5_REG_BITS - __builtin_clz(flow_id));
5469                 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) >
5470                     mtr_reg_bits) {
5471                         mlx5_ipool_free(fm->flow_ipool, tag_id);
5472                         return rte_flow_error_set(error, EINVAL,
5473                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5474                                         "Meter flow id exceeds max limit.");
5475                 }
5476                 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits)
5477                         priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits;
5478         }
5479         /* Build tag actions and items for meter_id/meter flow_id. */
5480         set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre;
5481         tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
5482         tag_item_mask = tag_item_spec + 1;
5483         /* Both flow_id and meter_id share the same register. */
5484         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
5485                 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
5486                                                             0, error),
5487                 .offset = mtr_id_offset,
5488                 .length = mtr_reg_bits,
5489                 .data = flow->meter,
5490         };
5491         /*
5492          * The color Reg bits used by flow_id are growing from
5493          * msb to lsb, so must do bit reverse for flow_id val in RegC.
5494          */
5495         for (shift = 0; shift < flow_id_bits; shift++)
5496                 flow_id_reversed = (flow_id_reversed << 1) |
5497                                 ((flow_id >> shift) & 0x1);
5498         set_tag->data |=
5499                 flow_id_reversed << (mtr_reg_bits - flow_id_bits);
5500         tag_item_spec->id = set_tag->id;
5501         tag_item_spec->data = set_tag->data << mtr_id_offset;
5502         tag_item_mask->data = UINT32_MAX << mtr_id_offset;
5503         tag_action->type = (enum rte_flow_action_type)
5504                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5505         tag_action->conf = set_tag;
5506         tag_item->type = (enum rte_flow_item_type)
5507                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG;
5508         tag_item->spec = tag_item_spec;
5509         tag_item->last = NULL;
5510         tag_item->mask = tag_item_mask;
5511 exit:
5512         if (mtr_flow_id)
5513                 *mtr_flow_id = tag_id;
5514         return 0;
5515 }
5516
5517 /**
5518  * Split action list having QUEUE/RSS for metadata register copy.
5519  *
5520  * Once Q/RSS action is detected in user's action list, the flow action
5521  * should be split in order to copy metadata registers, which will happen in
5522  * RX_CP_TBL like,
5523  *   - CQE->flow_tag := reg_c[1] (MARK)
5524  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
5525  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
5526  * This is because the last action of each flow must be a terminal action
5527  * (QUEUE, RSS or DROP).
5528  *
5529  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
5530  * stored and kept in the mlx5_flow structure per each sub_flow.
5531  *
5532  * The Q/RSS action is replaced with,
5533  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
5534  * And the following JUMP action is added at the end,
5535  *   - JUMP, to RX_CP_TBL.
5536  *
5537  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
5538  * flow_create_split_metadata() routine. The flow will look like,
5539  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
5540  *
5541  * @param dev
5542  *   Pointer to Ethernet device.
5543  * @param[out] split_actions
5544  *   Pointer to store split actions to jump to CP_TBL.
5545  * @param[in] actions
5546  *   Pointer to the list of original flow actions.
5547  * @param[in] qrss
5548  *   Pointer to the Q/RSS action.
5549  * @param[in] actions_n
5550  *   Number of original actions.
5551  * @param[in] mtr_sfx
5552  *   Check if it is in meter suffix table.
5553  * @param[out] error
5554  *   Perform verbose error reporting if not NULL.
5555  *
5556  * @return
5557  *   non-zero unique flow_id on success, otherwise 0 and
5558  *   error/rte_error are set.
5559  */
5560 static uint32_t
5561 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
5562                           struct rte_flow_action *split_actions,
5563                           const struct rte_flow_action *actions,
5564                           const struct rte_flow_action *qrss,
5565                           int actions_n, int mtr_sfx,
5566                           struct rte_flow_error *error)
5567 {
5568         struct mlx5_priv *priv = dev->data->dev_private;
5569         struct mlx5_rte_flow_action_set_tag *set_tag;
5570         struct rte_flow_action_jump *jump;
5571         const int qrss_idx = qrss - actions;
5572         uint32_t flow_id = 0;
5573         int ret = 0;
5574
5575         /*
5576          * Given actions will be split
5577          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
5578          * - Add jump to mreg CP_TBL.
5579          * As a result, there will be one more action.
5580          */
5581         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
5582         /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */
5583         ++actions_n;
5584         set_tag = (void *)(split_actions + actions_n);
5585         /*
5586          * If we are not the meter suffix flow, add the tag action.
5587          * Since meter suffix flow already has the tag added.
5588          */
5589         if (!mtr_sfx) {
5590                 /*
5591                  * Allocate the new subflow ID. This one is unique within
5592                  * device and not shared with representors. Otherwise,
5593                  * we would have to resolve multi-thread access synch
5594                  * issue. Each flow on the shared device is appended
5595                  * with source vport identifier, so the resulting
5596                  * flows will be unique in the shared (by master and
5597                  * representors) domain even if they have coinciding
5598                  * IDs.
5599                  */
5600                 mlx5_ipool_malloc(priv->sh->ipool
5601                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id);
5602                 if (!flow_id)
5603                         return rte_flow_error_set(error, ENOMEM,
5604                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5605                                                   NULL, "can't allocate id "
5606                                                   "for split Q/RSS subflow");
5607                 /* Internal SET_TAG action to set flow ID. */
5608                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
5609                         .data = flow_id,
5610                 };
5611                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
5612                 if (ret < 0)
5613                         return ret;
5614                 set_tag->id = ret;
5615                 /* Construct new actions array. */
5616                 /* Replace QUEUE/RSS action. */
5617                 split_actions[qrss_idx] = (struct rte_flow_action){
5618                         .type = (enum rte_flow_action_type)
5619                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
5620                         .conf = set_tag,
5621                 };
5622         } else {
5623                 /*
5624                  * If we are the suffix flow of meter, tag already exist.
5625                  * Set the QUEUE/RSS action to void.
5626                  */
5627                 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID;
5628         }
5629         /* JUMP action to jump to mreg copy table (CP_TBL). */
5630         jump = (void *)(set_tag + 1);
5631         *jump = (struct rte_flow_action_jump){
5632                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
5633         };
5634         split_actions[actions_n - 2] = (struct rte_flow_action){
5635                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
5636                 .conf = jump,
5637         };
5638         split_actions[actions_n - 1] = (struct rte_flow_action){
5639                 .type = RTE_FLOW_ACTION_TYPE_END,
5640         };
5641         return flow_id;
5642 }
5643
5644 /**
5645  * Extend the given action list for Tx metadata copy.
5646  *
5647  * Copy the given action list to the ext_actions and add flow metadata register
5648  * copy action in order to copy reg_a set by WQE to reg_c[0].
5649  *
5650  * @param[out] ext_actions
5651  *   Pointer to the extended action list.
5652  * @param[in] actions
5653  *   Pointer to the list of actions.
5654  * @param[in] actions_n
5655  *   Number of actions in the list.
5656  * @param[out] error
5657  *   Perform verbose error reporting if not NULL.
5658  * @param[in] encap_idx
5659  *   The encap action index.
5660  *
5661  * @return
5662  *   0 on success, negative value otherwise
5663  */
5664 static int
5665 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
5666                        struct rte_flow_action *ext_actions,
5667                        const struct rte_flow_action *actions,
5668                        int actions_n, struct rte_flow_error *error,
5669                        int encap_idx)
5670 {
5671         struct mlx5_flow_action_copy_mreg *cp_mreg =
5672                 (struct mlx5_flow_action_copy_mreg *)
5673                         (ext_actions + actions_n + 1);
5674         int ret;
5675
5676         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
5677         if (ret < 0)
5678                 return ret;
5679         cp_mreg->dst = ret;
5680         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
5681         if (ret < 0)
5682                 return ret;
5683         cp_mreg->src = ret;
5684         if (encap_idx != 0)
5685                 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
5686         if (encap_idx == actions_n - 1) {
5687                 ext_actions[actions_n - 1] = (struct rte_flow_action){
5688                         .type = (enum rte_flow_action_type)
5689                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5690                         .conf = cp_mreg,
5691                 };
5692                 ext_actions[actions_n] = (struct rte_flow_action){
5693                         .type = RTE_FLOW_ACTION_TYPE_END,
5694                 };
5695         } else {
5696                 ext_actions[encap_idx] = (struct rte_flow_action){
5697                         .type = (enum rte_flow_action_type)
5698                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5699                         .conf = cp_mreg,
5700                 };
5701                 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
5702                                 sizeof(*ext_actions) * (actions_n - encap_idx));
5703         }
5704         return 0;
5705 }
5706
5707 /**
5708  * Check the match action from the action list.
5709  *
5710  * @param[in] actions
5711  *   Pointer to the list of actions.
5712  * @param[in] attr
5713  *   Flow rule attributes.
5714  * @param[in] action
5715  *   The action to be check if exist.
5716  * @param[out] match_action_pos
5717  *   Pointer to the position of the matched action if exists, otherwise is -1.
5718  * @param[out] qrss_action_pos
5719  *   Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
5720  * @param[out] modify_after_mirror
5721  *   Pointer to the flag of modify action after FDB mirroring.
5722  *
5723  * @return
5724  *   > 0 the total number of actions.
5725  *   0 if not found match action in action list.
5726  */
5727 static int
5728 flow_check_match_action(const struct rte_flow_action actions[],
5729                         const struct rte_flow_attr *attr,
5730                         enum rte_flow_action_type action,
5731                         int *match_action_pos, int *qrss_action_pos,
5732                         int *modify_after_mirror)
5733 {
5734         const struct rte_flow_action_sample *sample;
5735         const struct rte_flow_action_raw_decap *decap;
5736         int actions_n = 0;
5737         uint32_t ratio = 0;
5738         int sub_type = 0;
5739         int flag = 0;
5740         int fdb_mirror = 0;
5741
5742         *match_action_pos = -1;
5743         *qrss_action_pos = -1;
5744         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5745                 if (actions->type == action) {
5746                         flag = 1;
5747                         *match_action_pos = actions_n;
5748                 }
5749                 switch (actions->type) {
5750                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5751                 case RTE_FLOW_ACTION_TYPE_RSS:
5752                         *qrss_action_pos = actions_n;
5753                         break;
5754                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
5755                         sample = actions->conf;
5756                         ratio = sample->ratio;
5757                         sub_type = ((const struct rte_flow_action *)
5758                                         (sample->actions))->type;
5759                         if (ratio == 1 && attr->transfer)
5760                                 fdb_mirror = 1;
5761                         break;
5762                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5763                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5764                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5765                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5766                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5767                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5768                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5769                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5770                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5771                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5772                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5773                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5774                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5775                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5776                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5777                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5778                 case RTE_FLOW_ACTION_TYPE_FLAG:
5779                 case RTE_FLOW_ACTION_TYPE_MARK:
5780                 case RTE_FLOW_ACTION_TYPE_SET_META:
5781                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5782                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5783                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5784                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5785                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5786                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5787                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5788                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
5789                 case RTE_FLOW_ACTION_TYPE_METER:
5790                         if (fdb_mirror)
5791                                 *modify_after_mirror = 1;
5792                         break;
5793                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5794                         decap = actions->conf;
5795                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5796                                 ;
5797                         actions_n++;
5798                         if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5799                                 const struct rte_flow_action_raw_encap *encap =
5800                                                                 actions->conf;
5801                                 if (decap->size <=
5802                                         MLX5_ENCAPSULATION_DECISION_SIZE &&
5803                                     encap->size >
5804                                         MLX5_ENCAPSULATION_DECISION_SIZE)
5805                                         /* L3 encap. */
5806                                         break;
5807                         }
5808                         if (fdb_mirror)
5809                                 *modify_after_mirror = 1;
5810                         break;
5811                 default:
5812                         break;
5813                 }
5814                 actions_n++;
5815         }
5816         if (flag && fdb_mirror && !*modify_after_mirror) {
5817                 /* FDB mirroring uses the destination array to implement
5818                  * instead of FLOW_SAMPLER object.
5819                  */
5820                 if (sub_type != RTE_FLOW_ACTION_TYPE_END)
5821                         flag = 0;
5822         }
5823         /* Count RTE_FLOW_ACTION_TYPE_END. */
5824         return flag ? actions_n + 1 : 0;
5825 }
5826
5827 #define SAMPLE_SUFFIX_ITEM 3
5828
5829 /**
5830  * Split the sample flow.
5831  *
5832  * As sample flow will split to two sub flow, sample flow with
5833  * sample action, the other actions will move to new suffix flow.
5834  *
5835  * Also add unique tag id with tag action in the sample flow,
5836  * the same tag id will be as match in the suffix flow.
5837  *
5838  * @param dev
5839  *   Pointer to Ethernet device.
5840  * @param[in] add_tag
5841  *   Add extra tag action flag.
5842  * @param[out] sfx_items
5843  *   Suffix flow match items (list terminated by the END pattern item).
5844  * @param[in] actions
5845  *   Associated actions (list terminated by the END action).
5846  * @param[out] actions_sfx
5847  *   Suffix flow actions.
5848  * @param[out] actions_pre
5849  *   Prefix flow actions.
5850  * @param[in] actions_n
5851  *  The total number of actions.
5852  * @param[in] sample_action_pos
5853  *   The sample action position.
5854  * @param[in] qrss_action_pos
5855  *   The Queue/RSS action position.
5856  * @param[in] jump_table
5857  *   Add extra jump action flag.
5858  * @param[out] error
5859  *   Perform verbose error reporting if not NULL.
5860  *
5861  * @return
5862  *   0 on success, or unique flow_id, a negative errno value
5863  *   otherwise and rte_errno is set.
5864  */
5865 static int
5866 flow_sample_split_prep(struct rte_eth_dev *dev,
5867                        int add_tag,
5868                        const struct rte_flow_item items[],
5869                        struct rte_flow_item sfx_items[],
5870                        const struct rte_flow_action actions[],
5871                        struct rte_flow_action actions_sfx[],
5872                        struct rte_flow_action actions_pre[],
5873                        int actions_n,
5874                        int sample_action_pos,
5875                        int qrss_action_pos,
5876                        int jump_table,
5877                        struct rte_flow_error *error)
5878 {
5879         struct mlx5_priv *priv = dev->data->dev_private;
5880         struct mlx5_rte_flow_action_set_tag *set_tag;
5881         struct mlx5_rte_flow_item_tag *tag_spec;
5882         struct mlx5_rte_flow_item_tag *tag_mask;
5883         struct rte_flow_action_jump *jump_action;
5884         uint32_t tag_id = 0;
5885         int append_index = 0;
5886         int set_tag_idx = -1;
5887         int index;
5888         int ret;
5889
5890         if (sample_action_pos < 0)
5891                 return rte_flow_error_set(error, EINVAL,
5892                                           RTE_FLOW_ERROR_TYPE_ACTION,
5893                                           NULL, "invalid position of sample "
5894                                           "action in list");
5895         /* Prepare the actions for prefix and suffix flow. */
5896         if (add_tag) {
5897                 /* Update the new added tag action index preceding
5898                  * the PUSH_VLAN or ENCAP action.
5899                  */
5900                 const struct rte_flow_action_raw_encap *raw_encap;
5901                 const struct rte_flow_action *action = actions;
5902                 int encap_idx;
5903                 int action_idx = 0;
5904                 int raw_decap_idx = -1;
5905                 int push_vlan_idx = -1;
5906                 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) {
5907                         switch (action->type) {
5908                         case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5909                                 raw_decap_idx = action_idx;
5910                                 break;
5911                         case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5912                                 raw_encap = action->conf;
5913                                 if (raw_encap->size >
5914                                         MLX5_ENCAPSULATION_DECISION_SIZE) {
5915                                         encap_idx = raw_decap_idx != -1 ?
5916                                                     raw_decap_idx : action_idx;
5917                                         if (encap_idx < sample_action_pos &&
5918                                             push_vlan_idx == -1)
5919                                                 set_tag_idx = encap_idx;
5920                                 }
5921                                 break;
5922                         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5923                         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5924                                 encap_idx = action_idx;
5925                                 if (encap_idx < sample_action_pos &&
5926                                     push_vlan_idx == -1)
5927                                         set_tag_idx = encap_idx;
5928                                 break;
5929                         case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5930                         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5931                                 push_vlan_idx = action_idx;
5932                                 if (push_vlan_idx < sample_action_pos)
5933                                         set_tag_idx = action_idx;
5934                                 break;
5935                         default:
5936                                 break;
5937                         }
5938                         action_idx++;
5939                 }
5940         }
5941         /* Prepare the actions for prefix and suffix flow. */
5942         if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
5943                 index = qrss_action_pos;
5944                 /* Put the preceding the Queue/RSS action into prefix flow. */
5945                 if (index != 0)
5946                         memcpy(actions_pre, actions,
5947                                sizeof(struct rte_flow_action) * index);
5948                 /* Put others preceding the sample action into prefix flow. */
5949                 if (sample_action_pos > index + 1)
5950                         memcpy(actions_pre + index, actions + index + 1,
5951                                sizeof(struct rte_flow_action) *
5952                                (sample_action_pos - index - 1));
5953                 index = sample_action_pos - 1;
5954                 /* Put Queue/RSS action into Suffix flow. */
5955                 memcpy(actions_sfx, actions + qrss_action_pos,
5956                        sizeof(struct rte_flow_action));
5957                 actions_sfx++;
5958         } else if (add_tag && set_tag_idx >= 0) {
5959                 if (set_tag_idx > 0)
5960                         memcpy(actions_pre, actions,
5961                                sizeof(struct rte_flow_action) * set_tag_idx);
5962                 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx,
5963                        sizeof(struct rte_flow_action) *
5964                        (sample_action_pos - set_tag_idx));
5965                 index = sample_action_pos;
5966         } else {
5967                 index = sample_action_pos;
5968                 if (index != 0)
5969                         memcpy(actions_pre, actions,
5970                                sizeof(struct rte_flow_action) * index);
5971         }
5972         /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress.
5973          * For CX6DX and above, metadata registers Cx preserve their value,
5974          * add an extra tag action for NIC-RX and E-Switch Domain.
5975          */
5976         if (add_tag) {
5977                 /* Prepare the prefix tag action. */
5978                 append_index++;
5979                 set_tag = (void *)(actions_pre + actions_n + append_index);
5980                 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error);
5981                 /* Trust VF/SF on CX5 not supported meter so that the reserved
5982                  * metadata regC is REG_NON, back to use application tag
5983                  * index 0.
5984                  */
5985                 if (unlikely(ret == REG_NON))
5986                         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
5987                 if (ret < 0)
5988                         return ret;
5989                 mlx5_ipool_malloc(priv->sh->ipool
5990                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id);
5991                 *set_tag = (struct mlx5_rte_flow_action_set_tag) {
5992                         .id = ret,
5993                         .data = tag_id,
5994                 };
5995                 /* Prepare the suffix subflow items. */
5996                 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5997                         if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID) {
5998                                 memcpy(sfx_items, items, sizeof(*sfx_items));
5999                                 sfx_items++;
6000                         }
6001                 }
6002                 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
6003                 tag_spec->data = tag_id;
6004                 tag_spec->id = set_tag->id;
6005                 tag_mask = tag_spec + 1;
6006                 tag_mask->data = UINT32_MAX;
6007                 sfx_items[0] = (struct rte_flow_item){
6008                         .type = (enum rte_flow_item_type)
6009                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
6010                         .spec = tag_spec,
6011                         .last = NULL,
6012                         .mask = tag_mask,
6013                 };
6014                 sfx_items[1] = (struct rte_flow_item){
6015                         .type = (enum rte_flow_item_type)
6016                                 RTE_FLOW_ITEM_TYPE_END,
6017                 };
6018                 /* Prepare the tag action in prefix subflow. */
6019                 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx;
6020                 actions_pre[set_tag_idx] =
6021                         (struct rte_flow_action){
6022                         .type = (enum rte_flow_action_type)
6023                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
6024                         .conf = set_tag,
6025                 };
6026                 /* Update next sample position due to add one tag action */
6027                 index += 1;
6028         }
6029         /* Copy the sample action into prefix flow. */
6030         memcpy(actions_pre + index, actions + sample_action_pos,
6031                sizeof(struct rte_flow_action));
6032         index += 1;
6033         /* For the modify action after the sample action in E-Switch mirroring,
6034          * Add the extra jump action in prefix subflow and jump into the next
6035          * table, then do the modify action in the new table.
6036          */
6037         if (jump_table) {
6038                 /* Prepare the prefix jump action. */
6039                 append_index++;
6040                 jump_action = (void *)(actions_pre + actions_n + append_index);
6041                 jump_action->group = jump_table;
6042                 actions_pre[index++] =
6043                         (struct rte_flow_action){
6044                         .type = (enum rte_flow_action_type)
6045                                 RTE_FLOW_ACTION_TYPE_JUMP,
6046                         .conf = jump_action,
6047                 };
6048         }
6049         actions_pre[index] = (struct rte_flow_action){
6050                 .type = (enum rte_flow_action_type)
6051                         RTE_FLOW_ACTION_TYPE_END,
6052         };
6053         /* Put the actions after sample into Suffix flow. */
6054         memcpy(actions_sfx, actions + sample_action_pos + 1,
6055                sizeof(struct rte_flow_action) *
6056                (actions_n - sample_action_pos - 1));
6057         return tag_id;
6058 }
6059
6060 /**
6061  * The splitting for metadata feature.
6062  *
6063  * - Q/RSS action on NIC Rx should be split in order to pass by
6064  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
6065  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
6066  *
6067  * - All the actions on NIC Tx should have a mreg copy action to
6068  *   copy reg_a from WQE to reg_c[0].
6069  *
6070  * @param dev
6071  *   Pointer to Ethernet device.
6072  * @param[in] flow
6073  *   Parent flow structure pointer.
6074  * @param[in] attr
6075  *   Flow rule attributes.
6076  * @param[in] items
6077  *   Pattern specification (list terminated by the END pattern item).
6078  * @param[in] actions
6079  *   Associated actions (list terminated by the END action).
6080  * @param[in] flow_split_info
6081  *   Pointer to flow split info structure.
6082  * @param[out] error
6083  *   Perform verbose error reporting if not NULL.
6084  * @return
6085  *   0 on success, negative value otherwise
6086  */
6087 static int
6088 flow_create_split_metadata(struct rte_eth_dev *dev,
6089                            struct rte_flow *flow,
6090                            const struct rte_flow_attr *attr,
6091                            const struct rte_flow_item items[],
6092                            const struct rte_flow_action actions[],
6093                            struct mlx5_flow_split_info *flow_split_info,
6094                            struct rte_flow_error *error)
6095 {
6096         struct mlx5_priv *priv = dev->data->dev_private;
6097         struct mlx5_sh_config *config = &priv->sh->config;
6098         const struct rte_flow_action *qrss = NULL;
6099         struct rte_flow_action *ext_actions = NULL;
6100         struct mlx5_flow *dev_flow = NULL;
6101         uint32_t qrss_id = 0;
6102         int mtr_sfx = 0;
6103         size_t act_size;
6104         int actions_n;
6105         int encap_idx;
6106         int ret;
6107
6108         /* Check whether extensive metadata feature is engaged. */
6109         if (!config->dv_flow_en ||
6110             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
6111             !mlx5_flow_ext_mreg_supported(dev))
6112                 return flow_create_split_inner(dev, flow, NULL, attr, items,
6113                                                actions, flow_split_info, error);
6114         actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
6115                                                            &encap_idx);
6116         if (qrss) {
6117                 /* Exclude hairpin flows from splitting. */
6118                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
6119                         const struct rte_flow_action_queue *queue;
6120
6121                         queue = qrss->conf;
6122                         if (mlx5_rxq_is_hairpin(dev, queue->index))
6123                                 qrss = NULL;
6124                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
6125                         const struct rte_flow_action_rss *rss;
6126
6127                         rss = qrss->conf;
6128                         if (mlx5_rxq_is_hairpin(dev, rss->queue[0]))
6129                                 qrss = NULL;
6130                 }
6131         }
6132         if (qrss) {
6133                 /* Check if it is in meter suffix table. */
6134                 mtr_sfx = attr->group == (attr->transfer ?
6135                           (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
6136                           MLX5_FLOW_TABLE_LEVEL_METER);
6137                 /*
6138                  * Q/RSS action on NIC Rx should be split in order to pass by
6139                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
6140                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
6141                  */
6142                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
6143                            sizeof(struct rte_flow_action_set_tag) +
6144                            sizeof(struct rte_flow_action_jump);
6145                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
6146                                           SOCKET_ID_ANY);
6147                 if (!ext_actions)
6148                         return rte_flow_error_set(error, ENOMEM,
6149                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6150                                                   NULL, "no memory to split "
6151                                                   "metadata flow");
6152                 /*
6153                  * Create the new actions list with removed Q/RSS action
6154                  * and appended set tag and jump to register copy table
6155                  * (RX_CP_TBL). We should preallocate unique tag ID here
6156                  * in advance, because it is needed for set tag action.
6157                  */
6158                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
6159                                                     qrss, actions_n,
6160                                                     mtr_sfx, error);
6161                 if (!mtr_sfx && !qrss_id) {
6162                         ret = -rte_errno;
6163                         goto exit;
6164                 }
6165         } else if (attr->egress && !attr->transfer) {
6166                 /*
6167                  * All the actions on NIC Tx should have a metadata register
6168                  * copy action to copy reg_a from WQE to reg_c[meta]
6169                  */
6170                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
6171                            sizeof(struct mlx5_flow_action_copy_mreg);
6172                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
6173                                           SOCKET_ID_ANY);
6174                 if (!ext_actions)
6175                         return rte_flow_error_set(error, ENOMEM,
6176                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6177                                                   NULL, "no memory to split "
6178                                                   "metadata flow");
6179                 /* Create the action list appended with copy register. */
6180                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
6181                                              actions_n, error, encap_idx);
6182                 if (ret < 0)
6183                         goto exit;
6184         }
6185         /* Add the unmodified original or prefix subflow. */
6186         ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
6187                                       items, ext_actions ? ext_actions :
6188                                       actions, flow_split_info, error);
6189         if (ret < 0)
6190                 goto exit;
6191         MLX5_ASSERT(dev_flow);
6192         if (qrss) {
6193                 const struct rte_flow_attr q_attr = {
6194                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
6195                         .ingress = 1,
6196                 };
6197                 /* Internal PMD action to set register. */
6198                 struct mlx5_rte_flow_item_tag q_tag_spec = {
6199                         .data = qrss_id,
6200                         .id = REG_NON,
6201                 };
6202                 struct rte_flow_item q_items[] = {
6203                         {
6204                                 .type = (enum rte_flow_item_type)
6205                                         MLX5_RTE_FLOW_ITEM_TYPE_TAG,
6206                                 .spec = &q_tag_spec,
6207                                 .last = NULL,
6208                                 .mask = NULL,
6209                         },
6210                         {
6211                                 .type = RTE_FLOW_ITEM_TYPE_END,
6212                         },
6213                 };
6214                 struct rte_flow_action q_actions[] = {
6215                         {
6216                                 .type = qrss->type,
6217                                 .conf = qrss->conf,
6218                         },
6219                         {
6220                                 .type = RTE_FLOW_ACTION_TYPE_END,
6221                         },
6222                 };
6223                 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
6224
6225                 /*
6226                  * Configure the tag item only if there is no meter subflow.
6227                  * Since tag is already marked in the meter suffix subflow
6228                  * we can just use the meter suffix items as is.
6229                  */
6230                 if (qrss_id) {
6231                         /* Not meter subflow. */
6232                         MLX5_ASSERT(!mtr_sfx);
6233                         /*
6234                          * Put unique id in prefix flow due to it is destroyed
6235                          * after suffix flow and id will be freed after there
6236                          * is no actual flows with this id and identifier
6237                          * reallocation becomes possible (for example, for
6238                          * other flows in other threads).
6239                          */
6240                         dev_flow->handle->split_flow_id = qrss_id;
6241                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
6242                                                    error);
6243                         if (ret < 0)
6244                                 goto exit;
6245                         q_tag_spec.id = ret;
6246                 }
6247                 dev_flow = NULL;
6248                 /* Add suffix subflow to execute Q/RSS. */
6249                 flow_split_info->prefix_layers = layers;
6250                 flow_split_info->prefix_mark = 0;
6251                 flow_split_info->table_id = 0;
6252                 ret = flow_create_split_inner(dev, flow, &dev_flow,
6253                                               &q_attr, mtr_sfx ? items :
6254                                               q_items, q_actions,
6255                                               flow_split_info, error);
6256                 if (ret < 0)
6257                         goto exit;
6258                 /* qrss ID should be freed if failed. */
6259                 qrss_id = 0;
6260                 MLX5_ASSERT(dev_flow);
6261         }
6262
6263 exit:
6264         /*
6265          * We do not destroy the partially created sub_flows in case of error.
6266          * These ones are included into parent flow list and will be destroyed
6267          * by flow_drv_destroy.
6268          */
6269         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
6270                         qrss_id);
6271         mlx5_free(ext_actions);
6272         return ret;
6273 }
6274
6275 /**
6276  * Create meter internal drop flow with the original pattern.
6277  *
6278  * @param dev
6279  *   Pointer to Ethernet device.
6280  * @param[in] flow
6281  *   Parent flow structure pointer.
6282  * @param[in] attr
6283  *   Flow rule attributes.
6284  * @param[in] items
6285  *   Pattern specification (list terminated by the END pattern item).
6286  * @param[in] flow_split_info
6287  *   Pointer to flow split info structure.
6288  * @param[in] fm
6289  *   Pointer to flow meter structure.
6290  * @param[out] error
6291  *   Perform verbose error reporting if not NULL.
6292  * @return
6293  *   0 on success, negative value otherwise
6294  */
6295 static uint32_t
6296 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev,
6297                         struct rte_flow *flow,
6298                         const struct rte_flow_attr *attr,
6299                         const struct rte_flow_item items[],
6300                         struct mlx5_flow_split_info *flow_split_info,
6301                         struct mlx5_flow_meter_info *fm,
6302                         struct rte_flow_error *error)
6303 {
6304         struct mlx5_flow *dev_flow = NULL;
6305         struct rte_flow_attr drop_attr = *attr;
6306         struct rte_flow_action drop_actions[3];
6307         struct mlx5_flow_split_info drop_split_info = *flow_split_info;
6308
6309         MLX5_ASSERT(fm->drop_cnt);
6310         drop_actions[0].type =
6311                 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
6312         drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt;
6313         drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP;
6314         drop_actions[1].conf = NULL;
6315         drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END;
6316         drop_actions[2].conf = NULL;
6317         drop_split_info.external = false;
6318         drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT;
6319         drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP;
6320         drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER;
6321         return flow_create_split_inner(dev, flow, &dev_flow,
6322                                 &drop_attr, items, drop_actions,
6323                                 &drop_split_info, error);
6324 }
6325
6326 /**
6327  * The splitting for meter feature.
6328  *
6329  * - The meter flow will be split to two flows as prefix and
6330  *   suffix flow. The packets make sense only it pass the prefix
6331  *   meter action.
6332  *
6333  * - Reg_C_5 is used for the packet to match betweend prefix and
6334  *   suffix flow.
6335  *
6336  * @param dev
6337  *   Pointer to Ethernet device.
6338  * @param[in] flow
6339  *   Parent flow structure pointer.
6340  * @param[in] attr
6341  *   Flow rule attributes.
6342  * @param[in] items
6343  *   Pattern specification (list terminated by the END pattern item).
6344  * @param[in] actions
6345  *   Associated actions (list terminated by the END action).
6346  * @param[in] flow_split_info
6347  *   Pointer to flow split info structure.
6348  * @param[out] error
6349  *   Perform verbose error reporting if not NULL.
6350  * @return
6351  *   0 on success, negative value otherwise
6352  */
6353 static int
6354 flow_create_split_meter(struct rte_eth_dev *dev,
6355                         struct rte_flow *flow,
6356                         const struct rte_flow_attr *attr,
6357                         const struct rte_flow_item items[],
6358                         const struct rte_flow_action actions[],
6359                         struct mlx5_flow_split_info *flow_split_info,
6360                         struct rte_flow_error *error)
6361 {
6362         struct mlx5_priv *priv = dev->data->dev_private;
6363         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
6364         struct rte_flow_action *sfx_actions = NULL;
6365         struct rte_flow_action *pre_actions = NULL;
6366         struct rte_flow_item *sfx_items = NULL;
6367         struct mlx5_flow *dev_flow = NULL;
6368         struct rte_flow_attr sfx_attr = *attr;
6369         struct mlx5_flow_meter_info *fm = NULL;
6370         uint8_t skip_scale_restore;
6371         bool has_mtr = false;
6372         bool has_modify = false;
6373         bool set_mtr_reg = true;
6374         bool is_mtr_hierarchy = false;
6375         uint32_t meter_id = 0;
6376         uint32_t mtr_idx = 0;
6377         uint32_t mtr_flow_id = 0;
6378         size_t act_size;
6379         size_t item_size;
6380         int actions_n = 0;
6381         int ret = 0;
6382
6383         if (priv->mtr_en)
6384                 actions_n = flow_check_meter_action(dev, actions, &has_mtr,
6385                                                     &has_modify, &meter_id);
6386         if (has_mtr) {
6387                 if (flow->meter) {
6388                         fm = flow_dv_meter_find_by_idx(priv, flow->meter);
6389                         if (!fm)
6390                                 return rte_flow_error_set(error, EINVAL,
6391                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6392                                                 NULL, "Meter not found.");
6393                 } else {
6394                         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
6395                         if (!fm)
6396                                 return rte_flow_error_set(error, EINVAL,
6397                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6398                                                 NULL, "Meter not found.");
6399                         ret = mlx5_flow_meter_attach(priv, fm,
6400                                                      &sfx_attr, error);
6401                         if (ret)
6402                                 return -rte_errno;
6403                         flow->meter = mtr_idx;
6404                 }
6405                 MLX5_ASSERT(wks);
6406                 wks->fm = fm;
6407                 if (!fm->def_policy) {
6408                         wks->policy = mlx5_flow_meter_policy_find(dev,
6409                                                                   fm->policy_id,
6410                                                                   NULL);
6411                         MLX5_ASSERT(wks->policy);
6412                         if (wks->policy->mark)
6413                                 wks->mark = 1;
6414                         if (wks->policy->is_hierarchy) {
6415                                 wks->final_policy =
6416                                 mlx5_flow_meter_hierarchy_get_final_policy(dev,
6417                                                                 wks->policy);
6418                                 if (!wks->final_policy)
6419                                         return rte_flow_error_set(error,
6420                                         EINVAL,
6421                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
6422                                 "Failed to find terminal policy of hierarchy.");
6423                                 is_mtr_hierarchy = true;
6424                         }
6425                 }
6426                 /*
6427                  * If it isn't default-policy Meter, and
6428                  * 1. There's no action in flow to change
6429                  *    packet (modify/encap/decap etc.), OR
6430                  * 2. No drop count needed for this meter.
6431                  * 3. It's not meter hierarchy.
6432                  * Then no need to use regC to save meter id anymore.
6433                  */
6434                 if (!fm->def_policy && !is_mtr_hierarchy &&
6435                     (!has_modify || !fm->drop_cnt))
6436                         set_mtr_reg = false;
6437                 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */
6438 #define METER_PREFIX_ACTION 7
6439                 act_size = (sizeof(struct rte_flow_action) *
6440                             (actions_n + METER_PREFIX_ACTION)) +
6441                            sizeof(struct mlx5_rte_flow_action_set_tag);
6442                 /* Suffix items: tag, vlan, port id, end. */
6443 #define METER_SUFFIX_ITEM 4
6444                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
6445                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
6446                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
6447                                           0, SOCKET_ID_ANY);
6448                 if (!sfx_actions)
6449                         return rte_flow_error_set(error, ENOMEM,
6450                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6451                                                   NULL, "no memory to split "
6452                                                   "meter flow");
6453                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
6454                              act_size);
6455                 /* There's no suffix flow for meter of non-default policy. */
6456                 if (!fm->def_policy)
6457                         pre_actions = sfx_actions + 1;
6458                 else
6459                         pre_actions = sfx_actions + actions_n;
6460                 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr,
6461                                             items, sfx_items, actions,
6462                                             sfx_actions, pre_actions,
6463                                             (set_mtr_reg ? &mtr_flow_id : NULL),
6464                                             error);
6465                 if (ret) {
6466                         ret = -rte_errno;
6467                         goto exit;
6468                 }
6469                 /* Add the prefix subflow. */
6470                 skip_scale_restore = flow_split_info->skip_scale;
6471                 flow_split_info->skip_scale |=
6472                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
6473                 ret = flow_create_split_inner(dev, flow, &dev_flow,
6474                                               attr, items, pre_actions,
6475                                               flow_split_info, error);
6476                 flow_split_info->skip_scale = skip_scale_restore;
6477                 if (ret) {
6478                         if (mtr_flow_id)
6479                                 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id);
6480                         ret = -rte_errno;
6481                         goto exit;
6482                 }
6483                 if (mtr_flow_id) {
6484                         dev_flow->handle->split_flow_id = mtr_flow_id;
6485                         dev_flow->handle->is_meter_flow_id = 1;
6486                 }
6487                 if (!fm->def_policy) {
6488                         if (!set_mtr_reg && fm->drop_cnt)
6489                                 ret =
6490                         flow_meter_create_drop_flow_with_org_pattern(dev, flow,
6491                                                         &sfx_attr, items,
6492                                                         flow_split_info,
6493                                                         fm, error);
6494                         goto exit;
6495                 }
6496                 /* Setting the sfx group atrr. */
6497                 sfx_attr.group = sfx_attr.transfer ?
6498                                 (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
6499                                  MLX5_FLOW_TABLE_LEVEL_METER;
6500                 flow_split_info->prefix_layers =
6501                                 flow_get_prefix_layer_flags(dev_flow);
6502                 flow_split_info->prefix_mark |= wks->mark;
6503                 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX;
6504         }
6505         /* Add the prefix subflow. */
6506         ret = flow_create_split_metadata(dev, flow,
6507                                          &sfx_attr, sfx_items ?
6508                                          sfx_items : items,
6509                                          sfx_actions ? sfx_actions : actions,
6510                                          flow_split_info, error);
6511 exit:
6512         if (sfx_actions)
6513                 mlx5_free(sfx_actions);
6514         return ret;
6515 }
6516
6517 /**
6518  * The splitting for sample feature.
6519  *
6520  * Once Sample action is detected in the action list, the flow actions should
6521  * be split into prefix sub flow and suffix sub flow.
6522  *
6523  * The original items remain in the prefix sub flow, all actions preceding the
6524  * sample action and the sample action itself will be copied to the prefix
6525  * sub flow, the actions following the sample action will be copied to the
6526  * suffix sub flow, Queue action always be located in the suffix sub flow.
6527  *
6528  * In order to make the packet from prefix sub flow matches with suffix sub
6529  * flow, an extra tag action be added into prefix sub flow, and the suffix sub
6530  * flow uses tag item with the unique flow id.
6531  *
6532  * @param dev
6533  *   Pointer to Ethernet device.
6534  * @param[in] flow
6535  *   Parent flow structure pointer.
6536  * @param[in] attr
6537  *   Flow rule attributes.
6538  * @param[in] items
6539  *   Pattern specification (list terminated by the END pattern item).
6540  * @param[in] actions
6541  *   Associated actions (list terminated by the END action).
6542  * @param[in] flow_split_info
6543  *   Pointer to flow split info structure.
6544  * @param[out] error
6545  *   Perform verbose error reporting if not NULL.
6546  * @return
6547  *   0 on success, negative value otherwise
6548  */
6549 static int
6550 flow_create_split_sample(struct rte_eth_dev *dev,
6551                          struct rte_flow *flow,
6552                          const struct rte_flow_attr *attr,
6553                          const struct rte_flow_item items[],
6554                          const struct rte_flow_action actions[],
6555                          struct mlx5_flow_split_info *flow_split_info,
6556                          struct rte_flow_error *error)
6557 {
6558         struct mlx5_priv *priv = dev->data->dev_private;
6559         struct rte_flow_action *sfx_actions = NULL;
6560         struct rte_flow_action *pre_actions = NULL;
6561         struct rte_flow_item *sfx_items = NULL;
6562         struct mlx5_flow *dev_flow = NULL;
6563         struct rte_flow_attr sfx_attr = *attr;
6564 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
6565         struct mlx5_flow_dv_sample_resource *sample_res;
6566         struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
6567         struct mlx5_flow_tbl_resource *sfx_tbl;
6568         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
6569 #endif
6570         size_t act_size;
6571         size_t item_size;
6572         uint32_t fdb_tx = 0;
6573         int32_t tag_id = 0;
6574         int actions_n = 0;
6575         int sample_action_pos;
6576         int qrss_action_pos;
6577         int add_tag = 0;
6578         int modify_after_mirror = 0;
6579         uint16_t jump_table = 0;
6580         const uint32_t next_ft_step = 1;
6581         int ret = 0;
6582
6583         if (priv->sampler_en)
6584                 actions_n = flow_check_match_action(actions, attr,
6585                                         RTE_FLOW_ACTION_TYPE_SAMPLE,
6586                                         &sample_action_pos, &qrss_action_pos,
6587                                         &modify_after_mirror);
6588         if (actions_n) {
6589                 /* The prefix actions must includes sample, tag, end. */
6590                 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
6591                            + sizeof(struct mlx5_rte_flow_action_set_tag);
6592                 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
6593                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
6594                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
6595                                           item_size), 0, SOCKET_ID_ANY);
6596                 if (!sfx_actions)
6597                         return rte_flow_error_set(error, ENOMEM,
6598                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6599                                                   NULL, "no memory to split "
6600                                                   "sample flow");
6601                 /* The representor_id is UINT16_MAX for uplink. */
6602                 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX);
6603                 /*
6604                  * When reg_c_preserve is set, metadata registers Cx preserve
6605                  * their value even through packet duplication.
6606                  */
6607                 add_tag = (!fdb_tx ||
6608                            priv->sh->cdev->config.hca_attr.reg_c_preserve);
6609                 if (add_tag)
6610                         sfx_items = (struct rte_flow_item *)((char *)sfx_actions
6611                                         + act_size);
6612                 if (modify_after_mirror)
6613                         jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR +
6614                                      next_ft_step;
6615                 pre_actions = sfx_actions + actions_n;
6616                 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items,
6617                                                 actions, sfx_actions,
6618                                                 pre_actions, actions_n,
6619                                                 sample_action_pos,
6620                                                 qrss_action_pos, jump_table,
6621                                                 error);
6622                 if (tag_id < 0 || (add_tag && !tag_id)) {
6623                         ret = -rte_errno;
6624                         goto exit;
6625                 }
6626                 if (modify_after_mirror)
6627                         flow_split_info->skip_scale =
6628                                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
6629                 /* Add the prefix subflow. */
6630                 ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
6631                                               items, pre_actions,
6632                                               flow_split_info, error);
6633                 if (ret) {
6634                         ret = -rte_errno;
6635                         goto exit;
6636                 }
6637                 dev_flow->handle->split_flow_id = tag_id;
6638 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
6639                 if (!modify_after_mirror) {
6640                         /* Set the sfx group attr. */
6641                         sample_res = (struct mlx5_flow_dv_sample_resource *)
6642                                                 dev_flow->dv.sample_res;
6643                         sfx_tbl = (struct mlx5_flow_tbl_resource *)
6644                                                 sample_res->normal_path_tbl;
6645                         sfx_tbl_data = container_of(sfx_tbl,
6646                                                 struct mlx5_flow_tbl_data_entry,
6647                                                 tbl);
6648                         sfx_attr.group = sfx_attr.transfer ?
6649                         (sfx_tbl_data->level - 1) : sfx_tbl_data->level;
6650                 } else {
6651                         MLX5_ASSERT(attr->transfer);
6652                         sfx_attr.group = jump_table;
6653                 }
6654                 flow_split_info->prefix_layers =
6655                                 flow_get_prefix_layer_flags(dev_flow);
6656                 MLX5_ASSERT(wks);
6657                 flow_split_info->prefix_mark |= wks->mark;
6658                 /* Suffix group level already be scaled with factor, set
6659                  * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale
6660                  * again in translation.
6661                  */
6662                 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT;
6663 #endif
6664         }
6665         /* Add the suffix subflow. */
6666         ret = flow_create_split_meter(dev, flow, &sfx_attr,
6667                                       sfx_items ? sfx_items : items,
6668                                       sfx_actions ? sfx_actions : actions,
6669                                       flow_split_info, error);
6670 exit:
6671         if (sfx_actions)
6672                 mlx5_free(sfx_actions);
6673         return ret;
6674 }
6675
6676 /**
6677  * Split the flow to subflow set. The splitters might be linked
6678  * in the chain, like this:
6679  * flow_create_split_outer() calls:
6680  *   flow_create_split_meter() calls:
6681  *     flow_create_split_metadata(meter_subflow_0) calls:
6682  *       flow_create_split_inner(metadata_subflow_0)
6683  *       flow_create_split_inner(metadata_subflow_1)
6684  *       flow_create_split_inner(metadata_subflow_2)
6685  *     flow_create_split_metadata(meter_subflow_1) calls:
6686  *       flow_create_split_inner(metadata_subflow_0)
6687  *       flow_create_split_inner(metadata_subflow_1)
6688  *       flow_create_split_inner(metadata_subflow_2)
6689  *
6690  * This provide flexible way to add new levels of flow splitting.
6691  * The all of successfully created subflows are included to the
6692  * parent flow dev_flow list.
6693  *
6694  * @param dev
6695  *   Pointer to Ethernet device.
6696  * @param[in] flow
6697  *   Parent flow structure pointer.
6698  * @param[in] attr
6699  *   Flow rule attributes.
6700  * @param[in] items
6701  *   Pattern specification (list terminated by the END pattern item).
6702  * @param[in] actions
6703  *   Associated actions (list terminated by the END action).
6704  * @param[in] flow_split_info
6705  *   Pointer to flow split info structure.
6706  * @param[out] error
6707  *   Perform verbose error reporting if not NULL.
6708  * @return
6709  *   0 on success, negative value otherwise
6710  */
6711 static int
6712 flow_create_split_outer(struct rte_eth_dev *dev,
6713                         struct rte_flow *flow,
6714                         const struct rte_flow_attr *attr,
6715                         const struct rte_flow_item items[],
6716                         const struct rte_flow_action actions[],
6717                         struct mlx5_flow_split_info *flow_split_info,
6718                         struct rte_flow_error *error)
6719 {
6720         int ret;
6721
6722         ret = flow_create_split_sample(dev, flow, attr, items,
6723                                        actions, flow_split_info, error);
6724         MLX5_ASSERT(ret <= 0);
6725         return ret;
6726 }
6727
6728 static inline struct mlx5_flow_tunnel *
6729 flow_tunnel_from_rule(const struct mlx5_flow *flow)
6730 {
6731         struct mlx5_flow_tunnel *tunnel;
6732
6733 #pragma GCC diagnostic push
6734 #pragma GCC diagnostic ignored "-Wcast-qual"
6735         tunnel = (typeof(tunnel))flow->tunnel;
6736 #pragma GCC diagnostic pop
6737
6738         return tunnel;
6739 }
6740
6741 /**
6742  * Adjust flow RSS workspace if needed.
6743  *
6744  * @param wks
6745  *   Pointer to thread flow work space.
6746  * @param rss_desc
6747  *   Pointer to RSS descriptor.
6748  * @param[in] nrssq_num
6749  *   New RSS queue number.
6750  *
6751  * @return
6752  *   0 on success, -1 otherwise and rte_errno is set.
6753  */
6754 static int
6755 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
6756                           struct mlx5_flow_rss_desc *rss_desc,
6757                           uint32_t nrssq_num)
6758 {
6759         if (likely(nrssq_num <= wks->rssq_num))
6760                 return 0;
6761         rss_desc->queue = realloc(rss_desc->queue,
6762                           sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2));
6763         if (!rss_desc->queue) {
6764                 rte_errno = ENOMEM;
6765                 return -1;
6766         }
6767         wks->rssq_num = RTE_ALIGN(nrssq_num, 2);
6768         return 0;
6769 }
6770
6771 /**
6772  * Create a flow and add it to @p list.
6773  *
6774  * @param dev
6775  *   Pointer to Ethernet device.
6776  * @param list
6777  *   Pointer to a TAILQ flow list. If this parameter NULL,
6778  *   no list insertion occurred, flow is just created,
6779  *   this is caller's responsibility to track the
6780  *   created flow.
6781  * @param[in] attr
6782  *   Flow rule attributes.
6783  * @param[in] items
6784  *   Pattern specification (list terminated by the END pattern item).
6785  * @param[in] actions
6786  *   Associated actions (list terminated by the END action).
6787  * @param[in] external
6788  *   This flow rule is created by request external to PMD.
6789  * @param[out] error
6790  *   Perform verbose error reporting if not NULL.
6791  *
6792  * @return
6793  *   A flow index on success, 0 otherwise and rte_errno is set.
6794  */
6795 static uint32_t
6796 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
6797                  const struct rte_flow_attr *attr,
6798                  const struct rte_flow_item items[],
6799                  const struct rte_flow_action original_actions[],
6800                  bool external, struct rte_flow_error *error)
6801 {
6802         struct mlx5_priv *priv = dev->data->dev_private;
6803         struct rte_flow *flow = NULL;
6804         struct mlx5_flow *dev_flow;
6805         const struct rte_flow_action_rss *rss = NULL;
6806         struct mlx5_translated_action_handle
6807                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
6808         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
6809         union {
6810                 struct mlx5_flow_expand_rss buf;
6811                 uint8_t buffer[4096];
6812         } expand_buffer;
6813         union {
6814                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6815                 uint8_t buffer[2048];
6816         } actions_rx;
6817         union {
6818                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6819                 uint8_t buffer[2048];
6820         } actions_hairpin_tx;
6821         union {
6822                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
6823                 uint8_t buffer[2048];
6824         } items_tx;
6825         struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
6826         struct mlx5_flow_rss_desc *rss_desc;
6827         const struct rte_flow_action *p_actions_rx;
6828         uint32_t i;
6829         uint32_t idx = 0;
6830         int hairpin_flow;
6831         struct rte_flow_attr attr_tx = { .priority = 0 };
6832         const struct rte_flow_action *actions;
6833         struct rte_flow_action *translated_actions = NULL;
6834         struct mlx5_flow_tunnel *tunnel;
6835         struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
6836         struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace();
6837         struct mlx5_flow_split_info flow_split_info = {
6838                 .external = !!external,
6839                 .skip_scale = 0,
6840                 .flow_idx = 0,
6841                 .prefix_mark = 0,
6842                 .prefix_layers = 0,
6843                 .table_id = 0
6844         };
6845         int ret;
6846
6847         MLX5_ASSERT(wks);
6848         rss_desc = &wks->rss_desc;
6849         ret = flow_action_handles_translate(dev, original_actions,
6850                                             indir_actions,
6851                                             &indir_actions_n,
6852                                             &translated_actions, error);
6853         if (ret < 0) {
6854                 MLX5_ASSERT(translated_actions == NULL);
6855                 return 0;
6856         }
6857         actions = translated_actions ? translated_actions : original_actions;
6858         p_actions_rx = actions;
6859         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
6860         ret = flow_drv_validate(dev, attr, items, p_actions_rx,
6861                                 external, hairpin_flow, error);
6862         if (ret < 0)
6863                 goto error_before_hairpin_split;
6864         flow = mlx5_ipool_zmalloc(priv->flows[type], &idx);
6865         if (!flow) {
6866                 rte_errno = ENOMEM;
6867                 goto error_before_hairpin_split;
6868         }
6869         if (hairpin_flow > 0) {
6870                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
6871                         rte_errno = EINVAL;
6872                         goto error_before_hairpin_split;
6873                 }
6874                 flow_hairpin_split(dev, actions, actions_rx.actions,
6875                                    actions_hairpin_tx.actions, items_tx.items,
6876                                    idx);
6877                 p_actions_rx = actions_rx.actions;
6878         }
6879         flow_split_info.flow_idx = idx;
6880         flow->drv_type = flow_get_drv_type(dev, attr);
6881         MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
6882                     flow->drv_type < MLX5_FLOW_TYPE_MAX);
6883         memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
6884         /* RSS Action only works on NIC RX domain */
6885         if (attr->ingress && !attr->transfer)
6886                 rss = flow_get_rss_action(dev, p_actions_rx);
6887         if (rss) {
6888                 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num))
6889                         return 0;
6890                 /*
6891                  * The following information is required by
6892                  * mlx5_flow_hashfields_adjust() in advance.
6893                  */
6894                 rss_desc->level = rss->level;
6895                 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */
6896                 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types;
6897         }
6898         flow->dev_handles = 0;
6899         if (rss && rss->types) {
6900                 unsigned int graph_root;
6901
6902                 graph_root = find_graph_root(rss->level);
6903                 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
6904                                            items, rss->types,
6905                                            mlx5_support_expansion, graph_root);
6906                 MLX5_ASSERT(ret > 0 &&
6907                        (unsigned int)ret < sizeof(expand_buffer.buffer));
6908                 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) {
6909                         for (i = 0; i < buf->entries; ++i)
6910                                 mlx5_dbg__print_pattern(buf->entry[i].pattern);
6911                 }
6912         } else {
6913                 buf->entries = 1;
6914                 buf->entry[0].pattern = (void *)(uintptr_t)items;
6915         }
6916         rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions,
6917                                                       indir_actions_n);
6918         for (i = 0; i < buf->entries; ++i) {
6919                 /* Initialize flow split data. */
6920                 flow_split_info.prefix_layers = 0;
6921                 flow_split_info.prefix_mark = 0;
6922                 flow_split_info.skip_scale = 0;
6923                 /*
6924                  * The splitter may create multiple dev_flows,
6925                  * depending on configuration. In the simplest
6926                  * case it just creates unmodified original flow.
6927                  */
6928                 ret = flow_create_split_outer(dev, flow, attr,
6929                                               buf->entry[i].pattern,
6930                                               p_actions_rx, &flow_split_info,
6931                                               error);
6932                 if (ret < 0)
6933                         goto error;
6934                 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) {
6935                         ret = flow_tunnel_add_default_miss(dev, flow, attr,
6936                                                            p_actions_rx,
6937                                                            idx,
6938                                                            wks->flows[0].tunnel,
6939                                                            &default_miss_ctx,
6940                                                            error);
6941                         if (ret < 0) {
6942                                 mlx5_free(default_miss_ctx.queue);
6943                                 goto error;
6944                         }
6945                 }
6946         }
6947         /* Create the tx flow. */
6948         if (hairpin_flow) {
6949                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
6950                 attr_tx.ingress = 0;
6951                 attr_tx.egress = 1;
6952                 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
6953                                          actions_hairpin_tx.actions,
6954                                          idx, error);
6955                 if (!dev_flow)
6956                         goto error;
6957                 dev_flow->flow = flow;
6958                 dev_flow->external = 0;
6959                 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
6960                               dev_flow->handle, next);
6961                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
6962                                          items_tx.items,
6963                                          actions_hairpin_tx.actions, error);
6964                 if (ret < 0)
6965                         goto error;
6966         }
6967         /*
6968          * Update the metadata register copy table. If extensive
6969          * metadata feature is enabled and registers are supported
6970          * we might create the extra rte_flow for each unique
6971          * MARK/FLAG action ID.
6972          *
6973          * The table is updated for ingress Flows only, because
6974          * the egress Flows belong to the different device and
6975          * copy table should be updated in peer NIC Rx domain.
6976          */
6977         if (attr->ingress &&
6978             (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
6979                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
6980                 if (ret)
6981                         goto error;
6982         }
6983         /*
6984          * If the flow is external (from application) OR device is started,
6985          * OR mreg discover, then apply immediately.
6986          */
6987         if (external || dev->data->dev_started ||
6988             (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP &&
6989              attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) {
6990                 ret = flow_drv_apply(dev, flow, error);
6991                 if (ret < 0)
6992                         goto error;
6993         }
6994         flow->type = type;
6995         flow_rxq_flags_set(dev, flow);
6996         rte_free(translated_actions);
6997         tunnel = flow_tunnel_from_rule(wks->flows);
6998         if (tunnel) {
6999                 flow->tunnel = 1;
7000                 flow->tunnel_id = tunnel->tunnel_id;
7001                 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED);
7002                 mlx5_free(default_miss_ctx.queue);
7003         }
7004         mlx5_flow_pop_thread_workspace();
7005         return idx;
7006 error:
7007         MLX5_ASSERT(flow);
7008         ret = rte_errno; /* Save rte_errno before cleanup. */
7009         flow_mreg_del_copy_action(dev, flow);
7010         flow_drv_destroy(dev, flow);
7011         if (rss_desc->shared_rss)
7012                 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *)
7013                         mlx5_ipool_get
7014                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
7015                         rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
7016         mlx5_ipool_free(priv->flows[type], idx);
7017         rte_errno = ret; /* Restore rte_errno. */
7018         ret = rte_errno;
7019         rte_errno = ret;
7020         mlx5_flow_pop_thread_workspace();
7021 error_before_hairpin_split:
7022         rte_free(translated_actions);
7023         return 0;
7024 }
7025
7026 /**
7027  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
7028  * incoming packets to table 1.
7029  *
7030  * Other flow rules, requested for group n, will be created in
7031  * e-switch table n+1.
7032  * Jump action to e-switch group n will be created to group n+1.
7033  *
7034  * Used when working in switchdev mode, to utilise advantages of table 1
7035  * and above.
7036  *
7037  * @param dev
7038  *   Pointer to Ethernet device.
7039  *
7040  * @return
7041  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
7042  */
7043 struct rte_flow *
7044 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
7045 {
7046         const struct rte_flow_attr attr = {
7047                 .group = 0,
7048                 .priority = 0,
7049                 .ingress = 1,
7050                 .egress = 0,
7051                 .transfer = 1,
7052         };
7053         const struct rte_flow_item pattern = {
7054                 .type = RTE_FLOW_ITEM_TYPE_END,
7055         };
7056         struct rte_flow_action_jump jump = {
7057                 .group = 1,
7058         };
7059         const struct rte_flow_action actions[] = {
7060                 {
7061                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
7062                         .conf = &jump,
7063                 },
7064                 {
7065                         .type = RTE_FLOW_ACTION_TYPE_END,
7066                 },
7067         };
7068         struct rte_flow_error error;
7069
7070         return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7071                                                    &attr, &pattern,
7072                                                    actions, false, &error);
7073 }
7074
7075 /**
7076  * Create a dedicated flow rule on e-switch table 1, matches ESW manager
7077  * and sq number, directs all packets to peer vport.
7078  *
7079  * @param dev
7080  *   Pointer to Ethernet device.
7081  * @param txq
7082  *   Txq index.
7083  *
7084  * @return
7085  *   Flow ID on success, 0 otherwise and rte_errno is set.
7086  */
7087 uint32_t
7088 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq)
7089 {
7090         struct rte_flow_attr attr = {
7091                 .group = 0,
7092                 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
7093                 .ingress = 1,
7094                 .egress = 0,
7095                 .transfer = 1,
7096         };
7097         struct rte_flow_item_port_id port_spec = {
7098                 .id = MLX5_PORT_ESW_MGR,
7099         };
7100         struct mlx5_rte_flow_item_tx_queue txq_spec = {
7101                 .queue = txq,
7102         };
7103         struct rte_flow_item pattern[] = {
7104                 {
7105                         .type = RTE_FLOW_ITEM_TYPE_PORT_ID,
7106                         .spec = &port_spec,
7107                 },
7108                 {
7109                         .type = (enum rte_flow_item_type)
7110                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
7111                         .spec = &txq_spec,
7112                 },
7113                 {
7114                         .type = RTE_FLOW_ITEM_TYPE_END,
7115                 },
7116         };
7117         struct rte_flow_action_jump jump = {
7118                 .group = 1,
7119         };
7120         struct rte_flow_action_port_id port = {
7121                 .id = dev->data->port_id,
7122         };
7123         struct rte_flow_action actions[] = {
7124                 {
7125                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
7126                         .conf = &jump,
7127                 },
7128                 {
7129                         .type = RTE_FLOW_ACTION_TYPE_END,
7130                 },
7131         };
7132         struct rte_flow_error error;
7133
7134         /*
7135          * Creates group 0, highest priority jump flow.
7136          * Matches txq to bypass kernel packets.
7137          */
7138         if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions,
7139                              false, &error) == 0)
7140                 return 0;
7141         /* Create group 1, lowest priority redirect flow for txq. */
7142         attr.group = 1;
7143         actions[0].conf = &port;
7144         actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID;
7145         return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern,
7146                                 actions, false, &error);
7147 }
7148
7149 /**
7150  * Validate a flow supported by the NIC.
7151  *
7152  * @see rte_flow_validate()
7153  * @see rte_flow_ops
7154  */
7155 int
7156 mlx5_flow_validate(struct rte_eth_dev *dev,
7157                    const struct rte_flow_attr *attr,
7158                    const struct rte_flow_item items[],
7159                    const struct rte_flow_action original_actions[],
7160                    struct rte_flow_error *error)
7161 {
7162         int hairpin_flow;
7163         struct mlx5_translated_action_handle
7164                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
7165         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
7166         const struct rte_flow_action *actions;
7167         struct rte_flow_action *translated_actions = NULL;
7168         int ret = flow_action_handles_translate(dev, original_actions,
7169                                                 indir_actions,
7170                                                 &indir_actions_n,
7171                                                 &translated_actions, error);
7172
7173         if (ret)
7174                 return ret;
7175         actions = translated_actions ? translated_actions : original_actions;
7176         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
7177         ret = flow_drv_validate(dev, attr, items, actions,
7178                                 true, hairpin_flow, error);
7179         rte_free(translated_actions);
7180         return ret;
7181 }
7182
7183 /**
7184  * Create a flow.
7185  *
7186  * @see rte_flow_create()
7187  * @see rte_flow_ops
7188  */
7189 struct rte_flow *
7190 mlx5_flow_create(struct rte_eth_dev *dev,
7191                  const struct rte_flow_attr *attr,
7192                  const struct rte_flow_item items[],
7193                  const struct rte_flow_action actions[],
7194                  struct rte_flow_error *error)
7195 {
7196         struct mlx5_priv *priv = dev->data->dev_private;
7197
7198         if (priv->sh->config.dv_flow_en == 2) {
7199                 rte_flow_error_set(error, ENOTSUP,
7200                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7201                           NULL,
7202                           "Flow non-Q creation not supported");
7203                 return NULL;
7204         }
7205         /*
7206          * If the device is not started yet, it is not allowed to created a
7207          * flow from application. PMD default flows and traffic control flows
7208          * are not affected.
7209          */
7210         if (unlikely(!dev->data->dev_started)) {
7211                 DRV_LOG(DEBUG, "port %u is not started when "
7212                         "inserting a flow", dev->data->port_id);
7213                 rte_flow_error_set(error, ENODEV,
7214                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7215                                    NULL,
7216                                    "port not started");
7217                 return NULL;
7218         }
7219
7220         return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN,
7221                                                    attr, items, actions,
7222                                                    true, error);
7223 }
7224
7225 /**
7226  * Destroy a flow in a list.
7227  *
7228  * @param dev
7229  *   Pointer to Ethernet device.
7230  * @param[in] flow_idx
7231  *   Index of flow to destroy.
7232  */
7233 static void
7234 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
7235                   uint32_t flow_idx)
7236 {
7237         struct mlx5_priv *priv = dev->data->dev_private;
7238         struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx);
7239
7240         if (!flow)
7241                 return;
7242         MLX5_ASSERT(flow->type == type);
7243         /*
7244          * Update RX queue flags only if port is started, otherwise it is
7245          * already clean.
7246          */
7247         if (dev->data->dev_started)
7248                 flow_rxq_flags_trim(dev, flow);
7249         flow_drv_destroy(dev, flow);
7250         if (flow->tunnel) {
7251                 struct mlx5_flow_tunnel *tunnel;
7252
7253                 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
7254                 RTE_VERIFY(tunnel);
7255                 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
7256                         mlx5_flow_tunnel_free(dev, tunnel);
7257         }
7258         flow_mreg_del_copy_action(dev, flow);
7259         mlx5_ipool_free(priv->flows[type], flow_idx);
7260 }
7261
7262 /**
7263  * Destroy all flows.
7264  *
7265  * @param dev
7266  *   Pointer to Ethernet device.
7267  * @param type
7268  *   Flow type to be flushed.
7269  * @param active
7270  *   If flushing is called actively.
7271  */
7272 void
7273 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
7274                      bool active)
7275 {
7276         struct mlx5_priv *priv = dev->data->dev_private;
7277         uint32_t num_flushed = 0, fidx = 1;
7278         struct rte_flow *flow;
7279
7280 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
7281         if (priv->sh->config.dv_flow_en == 2 &&
7282             type == MLX5_FLOW_TYPE_GEN) {
7283                 flow_hw_q_flow_flush(dev, NULL);
7284                 return;
7285         }
7286 #endif
7287
7288         MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) {
7289                 flow_list_destroy(dev, type, fidx);
7290                 num_flushed++;
7291         }
7292         if (active) {
7293                 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
7294                         dev->data->port_id, num_flushed);
7295         }
7296 }
7297
7298 /**
7299  * Stop all default actions for flows.
7300  *
7301  * @param dev
7302  *   Pointer to Ethernet device.
7303  */
7304 void
7305 mlx5_flow_stop_default(struct rte_eth_dev *dev)
7306 {
7307         flow_mreg_del_default_copy_action(dev);
7308         flow_rxq_flags_clear(dev);
7309 }
7310
7311 /**
7312  * Start all default actions for flows.
7313  *
7314  * @param dev
7315  *   Pointer to Ethernet device.
7316  * @return
7317  *   0 on success, a negative errno value otherwise and rte_errno is set.
7318  */
7319 int
7320 mlx5_flow_start_default(struct rte_eth_dev *dev)
7321 {
7322         struct rte_flow_error error;
7323
7324         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
7325         return flow_mreg_add_default_copy_action(dev, &error);
7326 }
7327
7328 /**
7329  * Release key of thread specific flow workspace data.
7330  */
7331 void
7332 flow_release_workspace(void *data)
7333 {
7334         struct mlx5_flow_workspace *wks = data;
7335         struct mlx5_flow_workspace *next;
7336
7337         while (wks) {
7338                 next = wks->next;
7339                 free(wks->rss_desc.queue);
7340                 free(wks);
7341                 wks = next;
7342         }
7343 }
7344
7345 /**
7346  * Get thread specific current flow workspace.
7347  *
7348  * @return pointer to thread specific flow workspace data, NULL on error.
7349  */
7350 struct mlx5_flow_workspace*
7351 mlx5_flow_get_thread_workspace(void)
7352 {
7353         struct mlx5_flow_workspace *data;
7354
7355         data = mlx5_flow_os_get_specific_workspace();
7356         MLX5_ASSERT(data && data->inuse);
7357         if (!data || !data->inuse)
7358                 DRV_LOG(ERR, "flow workspace not initialized.");
7359         return data;
7360 }
7361
7362 /**
7363  * Allocate and init new flow workspace.
7364  *
7365  * @return pointer to flow workspace data, NULL on error.
7366  */
7367 static struct mlx5_flow_workspace*
7368 flow_alloc_thread_workspace(void)
7369 {
7370         struct mlx5_flow_workspace *data = calloc(1, sizeof(*data));
7371
7372         if (!data) {
7373                 DRV_LOG(ERR, "Failed to allocate flow workspace "
7374                         "memory.");
7375                 return NULL;
7376         }
7377         data->rss_desc.queue = calloc(1,
7378                         sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM);
7379         if (!data->rss_desc.queue)
7380                 goto err;
7381         data->rssq_num = MLX5_RSSQ_DEFAULT_NUM;
7382         return data;
7383 err:
7384         free(data->rss_desc.queue);
7385         free(data);
7386         return NULL;
7387 }
7388
7389 /**
7390  * Get new thread specific flow workspace.
7391  *
7392  * If current workspace inuse, create new one and set as current.
7393  *
7394  * @return pointer to thread specific flow workspace data, NULL on error.
7395  */
7396 static struct mlx5_flow_workspace*
7397 mlx5_flow_push_thread_workspace(void)
7398 {
7399         struct mlx5_flow_workspace *curr;
7400         struct mlx5_flow_workspace *data;
7401
7402         curr = mlx5_flow_os_get_specific_workspace();
7403         if (!curr) {
7404                 data = flow_alloc_thread_workspace();
7405                 if (!data)
7406                         return NULL;
7407         } else if (!curr->inuse) {
7408                 data = curr;
7409         } else if (curr->next) {
7410                 data = curr->next;
7411         } else {
7412                 data = flow_alloc_thread_workspace();
7413                 if (!data)
7414                         return NULL;
7415                 curr->next = data;
7416                 data->prev = curr;
7417         }
7418         data->inuse = 1;
7419         data->flow_idx = 0;
7420         /* Set as current workspace */
7421         if (mlx5_flow_os_set_specific_workspace(data))
7422                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
7423         return data;
7424 }
7425
7426 /**
7427  * Close current thread specific flow workspace.
7428  *
7429  * If previous workspace available, set it as current.
7430  *
7431  * @return pointer to thread specific flow workspace data, NULL on error.
7432  */
7433 static void
7434 mlx5_flow_pop_thread_workspace(void)
7435 {
7436         struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace();
7437
7438         if (!data)
7439                 return;
7440         if (!data->inuse) {
7441                 DRV_LOG(ERR, "Failed to close unused flow workspace.");
7442                 return;
7443         }
7444         data->inuse = 0;
7445         if (!data->prev)
7446                 return;
7447         if (mlx5_flow_os_set_specific_workspace(data->prev))
7448                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
7449 }
7450
7451 /**
7452  * Verify the flow list is empty
7453  *
7454  * @param dev
7455  *  Pointer to Ethernet device.
7456  *
7457  * @return the number of flows not released.
7458  */
7459 int
7460 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused)
7461 {
7462         struct mlx5_priv *priv = dev->data->dev_private;
7463         struct rte_flow *flow;
7464         uint32_t idx = 0;
7465         int ret = 0, i;
7466
7467         for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
7468                 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) {
7469                         DRV_LOG(DEBUG, "port %u flow %p still referenced",
7470                                 dev->data->port_id, (void *)flow);
7471                         ret++;
7472                 }
7473         }
7474         return ret;
7475 }
7476
7477 /**
7478  * Enable default hairpin egress flow.
7479  *
7480  * @param dev
7481  *   Pointer to Ethernet device.
7482  * @param queue
7483  *   The queue index.
7484  *
7485  * @return
7486  *   0 on success, a negative errno value otherwise and rte_errno is set.
7487  */
7488 int
7489 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
7490                             uint32_t queue)
7491 {
7492         const struct rte_flow_attr attr = {
7493                 .egress = 1,
7494                 .priority = 0,
7495         };
7496         struct mlx5_rte_flow_item_tx_queue queue_spec = {
7497                 .queue = queue,
7498         };
7499         struct mlx5_rte_flow_item_tx_queue queue_mask = {
7500                 .queue = UINT32_MAX,
7501         };
7502         struct rte_flow_item items[] = {
7503                 {
7504                         .type = (enum rte_flow_item_type)
7505                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
7506                         .spec = &queue_spec,
7507                         .last = NULL,
7508                         .mask = &queue_mask,
7509                 },
7510                 {
7511                         .type = RTE_FLOW_ITEM_TYPE_END,
7512                 },
7513         };
7514         struct rte_flow_action_jump jump = {
7515                 .group = MLX5_HAIRPIN_TX_TABLE,
7516         };
7517         struct rte_flow_action actions[2];
7518         uint32_t flow_idx;
7519         struct rte_flow_error error;
7520
7521         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
7522         actions[0].conf = &jump;
7523         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
7524         flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7525                                     &attr, items, actions, false, &error);
7526         if (!flow_idx) {
7527                 DRV_LOG(DEBUG,
7528                         "Failed to create ctrl flow: rte_errno(%d),"
7529                         " type(%d), message(%s)",
7530                         rte_errno, error.type,
7531                         error.message ? error.message : " (no stated reason)");
7532                 return -rte_errno;
7533         }
7534         return 0;
7535 }
7536
7537 /**
7538  * Enable a control flow configured from the control plane.
7539  *
7540  * @param dev
7541  *   Pointer to Ethernet device.
7542  * @param eth_spec
7543  *   An Ethernet flow spec to apply.
7544  * @param eth_mask
7545  *   An Ethernet flow mask to apply.
7546  * @param vlan_spec
7547  *   A VLAN flow spec to apply.
7548  * @param vlan_mask
7549  *   A VLAN flow mask to apply.
7550  *
7551  * @return
7552  *   0 on success, a negative errno value otherwise and rte_errno is set.
7553  */
7554 int
7555 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
7556                     struct rte_flow_item_eth *eth_spec,
7557                     struct rte_flow_item_eth *eth_mask,
7558                     struct rte_flow_item_vlan *vlan_spec,
7559                     struct rte_flow_item_vlan *vlan_mask)
7560 {
7561         struct mlx5_priv *priv = dev->data->dev_private;
7562         const struct rte_flow_attr attr = {
7563                 .ingress = 1,
7564                 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
7565         };
7566         struct rte_flow_item items[] = {
7567                 {
7568                         .type = RTE_FLOW_ITEM_TYPE_ETH,
7569                         .spec = eth_spec,
7570                         .last = NULL,
7571                         .mask = eth_mask,
7572                 },
7573                 {
7574                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
7575                                               RTE_FLOW_ITEM_TYPE_END,
7576                         .spec = vlan_spec,
7577                         .last = NULL,
7578                         .mask = vlan_mask,
7579                 },
7580                 {
7581                         .type = RTE_FLOW_ITEM_TYPE_END,
7582                 },
7583         };
7584         uint16_t queue[priv->reta_idx_n];
7585         struct rte_flow_action_rss action_rss = {
7586                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
7587                 .level = 0,
7588                 .types = priv->rss_conf.rss_hf,
7589                 .key_len = priv->rss_conf.rss_key_len,
7590                 .queue_num = priv->reta_idx_n,
7591                 .key = priv->rss_conf.rss_key,
7592                 .queue = queue,
7593         };
7594         struct rte_flow_action actions[] = {
7595                 {
7596                         .type = RTE_FLOW_ACTION_TYPE_RSS,
7597                         .conf = &action_rss,
7598                 },
7599                 {
7600                         .type = RTE_FLOW_ACTION_TYPE_END,
7601                 },
7602         };
7603         uint32_t flow_idx;
7604         struct rte_flow_error error;
7605         unsigned int i;
7606
7607         if (!priv->reta_idx_n || !priv->rxqs_n) {
7608                 return 0;
7609         }
7610         if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG))
7611                 action_rss.types = 0;
7612         for (i = 0; i != priv->reta_idx_n; ++i)
7613                 queue[i] = (*priv->reta_idx)[i];
7614         flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7615                                     &attr, items, actions, false, &error);
7616         if (!flow_idx)
7617                 return -rte_errno;
7618         return 0;
7619 }
7620
7621 /**
7622  * Enable a flow control configured from the control plane.
7623  *
7624  * @param dev
7625  *   Pointer to Ethernet device.
7626  * @param eth_spec
7627  *   An Ethernet flow spec to apply.
7628  * @param eth_mask
7629  *   An Ethernet flow mask to apply.
7630  *
7631  * @return
7632  *   0 on success, a negative errno value otherwise and rte_errno is set.
7633  */
7634 int
7635 mlx5_ctrl_flow(struct rte_eth_dev *dev,
7636                struct rte_flow_item_eth *eth_spec,
7637                struct rte_flow_item_eth *eth_mask)
7638 {
7639         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
7640 }
7641
7642 /**
7643  * Create default miss flow rule matching lacp traffic
7644  *
7645  * @param dev
7646  *   Pointer to Ethernet device.
7647  * @param eth_spec
7648  *   An Ethernet flow spec to apply.
7649  *
7650  * @return
7651  *   0 on success, a negative errno value otherwise and rte_errno is set.
7652  */
7653 int
7654 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
7655 {
7656         /*
7657          * The LACP matching is done by only using ether type since using
7658          * a multicast dst mac causes kernel to give low priority to this flow.
7659          */
7660         static const struct rte_flow_item_eth lacp_spec = {
7661                 .type = RTE_BE16(0x8809),
7662         };
7663         static const struct rte_flow_item_eth lacp_mask = {
7664                 .type = 0xffff,
7665         };
7666         const struct rte_flow_attr attr = {
7667                 .ingress = 1,
7668         };
7669         struct rte_flow_item items[] = {
7670                 {
7671                         .type = RTE_FLOW_ITEM_TYPE_ETH,
7672                         .spec = &lacp_spec,
7673                         .mask = &lacp_mask,
7674                 },
7675                 {
7676                         .type = RTE_FLOW_ITEM_TYPE_END,
7677                 },
7678         };
7679         struct rte_flow_action actions[] = {
7680                 {
7681                         .type = (enum rte_flow_action_type)
7682                                 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
7683                 },
7684                 {
7685                         .type = RTE_FLOW_ACTION_TYPE_END,
7686                 },
7687         };
7688         struct rte_flow_error error;
7689         uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7690                                         &attr, items, actions,
7691                                         false, &error);
7692
7693         if (!flow_idx)
7694                 return -rte_errno;
7695         return 0;
7696 }
7697
7698 /**
7699  * Destroy a flow.
7700  *
7701  * @see rte_flow_destroy()
7702  * @see rte_flow_ops
7703  */
7704 int
7705 mlx5_flow_destroy(struct rte_eth_dev *dev,
7706                   struct rte_flow *flow,
7707                   struct rte_flow_error *error __rte_unused)
7708 {
7709         struct mlx5_priv *priv = dev->data->dev_private;
7710
7711         if (priv->sh->config.dv_flow_en == 2)
7712                 return rte_flow_error_set(error, ENOTSUP,
7713                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7714                           NULL,
7715                           "Flow non-Q destruction not supported");
7716         flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
7717                                 (uintptr_t)(void *)flow);
7718         return 0;
7719 }
7720
7721 /**
7722  * Destroy all flows.
7723  *
7724  * @see rte_flow_flush()
7725  * @see rte_flow_ops
7726  */
7727 int
7728 mlx5_flow_flush(struct rte_eth_dev *dev,
7729                 struct rte_flow_error *error __rte_unused)
7730 {
7731         mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false);
7732         return 0;
7733 }
7734
7735 /**
7736  * Isolated mode.
7737  *
7738  * @see rte_flow_isolate()
7739  * @see rte_flow_ops
7740  */
7741 int
7742 mlx5_flow_isolate(struct rte_eth_dev *dev,
7743                   int enable,
7744                   struct rte_flow_error *error)
7745 {
7746         struct mlx5_priv *priv = dev->data->dev_private;
7747
7748         if (dev->data->dev_started) {
7749                 rte_flow_error_set(error, EBUSY,
7750                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7751                                    NULL,
7752                                    "port must be stopped first");
7753                 return -rte_errno;
7754         }
7755         priv->isolated = !!enable;
7756         if (enable)
7757                 dev->dev_ops = &mlx5_dev_ops_isolate;
7758         else
7759                 dev->dev_ops = &mlx5_dev_ops;
7760
7761         dev->rx_descriptor_status = mlx5_rx_descriptor_status;
7762         dev->tx_descriptor_status = mlx5_tx_descriptor_status;
7763
7764         return 0;
7765 }
7766
7767 /**
7768  * Query a flow.
7769  *
7770  * @see rte_flow_query()
7771  * @see rte_flow_ops
7772  */
7773 static int
7774 flow_drv_query(struct rte_eth_dev *dev,
7775                uint32_t flow_idx,
7776                const struct rte_flow_action *actions,
7777                void *data,
7778                struct rte_flow_error *error)
7779 {
7780         struct mlx5_priv *priv = dev->data->dev_private;
7781         const struct mlx5_flow_driver_ops *fops;
7782         struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
7783                                                flow_idx);
7784         enum mlx5_flow_drv_type ftype;
7785
7786         if (!flow) {
7787                 return rte_flow_error_set(error, ENOENT,
7788                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7789                           NULL,
7790                           "invalid flow handle");
7791         }
7792         ftype = flow->drv_type;
7793         MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
7794         fops = flow_get_drv_ops(ftype);
7795
7796         return fops->query(dev, flow, actions, data, error);
7797 }
7798
7799 /**
7800  * Query a flow.
7801  *
7802  * @see rte_flow_query()
7803  * @see rte_flow_ops
7804  */
7805 int
7806 mlx5_flow_query(struct rte_eth_dev *dev,
7807                 struct rte_flow *flow,
7808                 const struct rte_flow_action *actions,
7809                 void *data,
7810                 struct rte_flow_error *error)
7811 {
7812         int ret;
7813         struct mlx5_priv *priv = dev->data->dev_private;
7814
7815         if (priv->sh->config.dv_flow_en == 2)
7816                 return rte_flow_error_set(error, ENOTSUP,
7817                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7818                           NULL,
7819                           "Flow non-Q query not supported");
7820         ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
7821                              error);
7822         if (ret < 0)
7823                 return ret;
7824         return 0;
7825 }
7826
7827 /**
7828  * Get rte_flow callbacks.
7829  *
7830  * @param dev
7831  *   Pointer to Ethernet device structure.
7832  * @param ops
7833  *   Pointer to operation-specific structure.
7834  *
7835  * @return 0
7836  */
7837 int
7838 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused,
7839                   const struct rte_flow_ops **ops)
7840 {
7841         *ops = &mlx5_flow_ops;
7842         return 0;
7843 }
7844
7845 /**
7846  * Validate meter policy actions.
7847  * Dispatcher for action type specific validation.
7848  *
7849  * @param[in] dev
7850  *   Pointer to the Ethernet device structure.
7851  * @param[in] action
7852  *   The meter policy action object to validate.
7853  * @param[in] attr
7854  *   Attributes of flow to determine steering domain.
7855  * @param[out] is_rss
7856  *   Is RSS or not.
7857  * @param[out] domain_bitmap
7858  *   Domain bitmap.
7859  * @param[out] is_def_policy
7860  *   Is default policy or not.
7861  * @param[out] error
7862  *   Perform verbose error reporting if not NULL. Initialized in case of
7863  *   error only.
7864  *
7865  * @return
7866  *   0 on success, otherwise negative errno value.
7867  */
7868 int
7869 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev,
7870                         const struct rte_flow_action *actions[RTE_COLORS],
7871                         struct rte_flow_attr *attr,
7872                         bool *is_rss,
7873                         uint8_t *domain_bitmap,
7874                         uint8_t *policy_mode,
7875                         struct rte_mtr_error *error)
7876 {
7877         const struct mlx5_flow_driver_ops *fops;
7878
7879         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7880         return fops->validate_mtr_acts(dev, actions, attr, is_rss,
7881                                        domain_bitmap, policy_mode, error);
7882 }
7883
7884 /**
7885  * Destroy the meter table set.
7886  *
7887  * @param[in] dev
7888  *   Pointer to Ethernet device.
7889  * @param[in] mtr_policy
7890  *   Meter policy struct.
7891  */
7892 void
7893 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev,
7894                       struct mlx5_flow_meter_policy *mtr_policy)
7895 {
7896         const struct mlx5_flow_driver_ops *fops;
7897
7898         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7899         fops->destroy_mtr_acts(dev, mtr_policy);
7900 }
7901
7902 /**
7903  * Create policy action, lock free,
7904  * (mutex should be acquired by caller).
7905  * Dispatcher for action type specific call.
7906  *
7907  * @param[in] dev
7908  *   Pointer to the Ethernet device structure.
7909  * @param[in] mtr_policy
7910  *   Meter policy struct.
7911  * @param[in] action
7912  *   Action specification used to create meter actions.
7913  * @param[out] error
7914  *   Perform verbose error reporting if not NULL. Initialized in case of
7915  *   error only.
7916  *
7917  * @return
7918  *   0 on success, otherwise negative errno value.
7919  */
7920 int
7921 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev,
7922                       struct mlx5_flow_meter_policy *mtr_policy,
7923                       const struct rte_flow_action *actions[RTE_COLORS],
7924                       struct rte_mtr_error *error)
7925 {
7926         const struct mlx5_flow_driver_ops *fops;
7927
7928         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7929         return fops->create_mtr_acts(dev, mtr_policy, actions, error);
7930 }
7931
7932 /**
7933  * Create policy rules, lock free,
7934  * (mutex should be acquired by caller).
7935  * Dispatcher for action type specific call.
7936  *
7937  * @param[in] dev
7938  *   Pointer to the Ethernet device structure.
7939  * @param[in] mtr_policy
7940  *   Meter policy struct.
7941  *
7942  * @return
7943  *   0 on success, -1 otherwise.
7944  */
7945 int
7946 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev,
7947                              struct mlx5_flow_meter_policy *mtr_policy)
7948 {
7949         const struct mlx5_flow_driver_ops *fops;
7950
7951         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7952         return fops->create_policy_rules(dev, mtr_policy);
7953 }
7954
7955 /**
7956  * Destroy policy rules, lock free,
7957  * (mutex should be acquired by caller).
7958  * Dispatcher for action type specific call.
7959  *
7960  * @param[in] dev
7961  *   Pointer to the Ethernet device structure.
7962  * @param[in] mtr_policy
7963  *   Meter policy struct.
7964  */
7965 void
7966 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev,
7967                              struct mlx5_flow_meter_policy *mtr_policy)
7968 {
7969         const struct mlx5_flow_driver_ops *fops;
7970
7971         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7972         fops->destroy_policy_rules(dev, mtr_policy);
7973 }
7974
7975 /**
7976  * Destroy the default policy table set.
7977  *
7978  * @param[in] dev
7979  *   Pointer to Ethernet device.
7980  */
7981 void
7982 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev)
7983 {
7984         const struct mlx5_flow_driver_ops *fops;
7985
7986         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7987         fops->destroy_def_policy(dev);
7988 }
7989
7990 /**
7991  * Destroy the default policy table set.
7992  *
7993  * @param[in] dev
7994  *   Pointer to Ethernet device.
7995  *
7996  * @return
7997  *   0 on success, -1 otherwise.
7998  */
7999 int
8000 mlx5_flow_create_def_policy(struct rte_eth_dev *dev)
8001 {
8002         const struct mlx5_flow_driver_ops *fops;
8003
8004         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8005         return fops->create_def_policy(dev);
8006 }
8007
8008 /**
8009  * Create the needed meter and suffix tables.
8010  *
8011  * @param[in] dev
8012  *   Pointer to Ethernet device.
8013  *
8014  * @return
8015  *   0 on success, -1 otherwise.
8016  */
8017 int
8018 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
8019                         struct mlx5_flow_meter_info *fm,
8020                         uint32_t mtr_idx,
8021                         uint8_t domain_bitmap)
8022 {
8023         const struct mlx5_flow_driver_ops *fops;
8024
8025         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8026         return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap);
8027 }
8028
8029 /**
8030  * Destroy the meter table set.
8031  *
8032  * @param[in] dev
8033  *   Pointer to Ethernet device.
8034  * @param[in] tbl
8035  *   Pointer to the meter table set.
8036  */
8037 void
8038 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
8039                            struct mlx5_flow_meter_info *fm)
8040 {
8041         const struct mlx5_flow_driver_ops *fops;
8042
8043         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8044         fops->destroy_mtr_tbls(dev, fm);
8045 }
8046
8047 /**
8048  * Destroy the global meter drop table.
8049  *
8050  * @param[in] dev
8051  *   Pointer to Ethernet device.
8052  */
8053 void
8054 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
8055 {
8056         const struct mlx5_flow_driver_ops *fops;
8057
8058         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8059         fops->destroy_mtr_drop_tbls(dev);
8060 }
8061
8062 /**
8063  * Destroy the sub policy table with RX queue.
8064  *
8065  * @param[in] dev
8066  *   Pointer to Ethernet device.
8067  * @param[in] mtr_policy
8068  *   Pointer to meter policy table.
8069  */
8070 void
8071 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
8072                 struct mlx5_flow_meter_policy *mtr_policy)
8073 {
8074         const struct mlx5_flow_driver_ops *fops;
8075
8076         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8077         fops->destroy_sub_policy_with_rxq(dev, mtr_policy);
8078 }
8079
8080 /**
8081  * Allocate the needed aso flow meter id.
8082  *
8083  * @param[in] dev
8084  *   Pointer to Ethernet device.
8085  *
8086  * @return
8087  *   Index to aso flow meter on success, NULL otherwise.
8088  */
8089 uint32_t
8090 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev)
8091 {
8092         const struct mlx5_flow_driver_ops *fops;
8093
8094         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8095         return fops->create_meter(dev);
8096 }
8097
8098 /**
8099  * Free the aso flow meter id.
8100  *
8101  * @param[in] dev
8102  *   Pointer to Ethernet device.
8103  * @param[in] mtr_idx
8104  *  Index to aso flow meter to be free.
8105  *
8106  * @return
8107  *   0 on success.
8108  */
8109 void
8110 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx)
8111 {
8112         const struct mlx5_flow_driver_ops *fops;
8113
8114         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8115         fops->free_meter(dev, mtr_idx);
8116 }
8117
8118 /**
8119  * Allocate a counter.
8120  *
8121  * @param[in] dev
8122  *   Pointer to Ethernet device structure.
8123  *
8124  * @return
8125  *   Index to allocated counter  on success, 0 otherwise.
8126  */
8127 uint32_t
8128 mlx5_counter_alloc(struct rte_eth_dev *dev)
8129 {
8130         const struct mlx5_flow_driver_ops *fops;
8131         struct rte_flow_attr attr = { .transfer = 0 };
8132
8133         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
8134                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8135                 return fops->counter_alloc(dev);
8136         }
8137         DRV_LOG(ERR,
8138                 "port %u counter allocate is not supported.",
8139                  dev->data->port_id);
8140         return 0;
8141 }
8142
8143 /**
8144  * Free a counter.
8145  *
8146  * @param[in] dev
8147  *   Pointer to Ethernet device structure.
8148  * @param[in] cnt
8149  *   Index to counter to be free.
8150  */
8151 void
8152 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
8153 {
8154         const struct mlx5_flow_driver_ops *fops;
8155         struct rte_flow_attr attr = { .transfer = 0 };
8156
8157         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
8158                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8159                 fops->counter_free(dev, cnt);
8160                 return;
8161         }
8162         DRV_LOG(ERR,
8163                 "port %u counter free is not supported.",
8164                  dev->data->port_id);
8165 }
8166
8167 /**
8168  * Query counter statistics.
8169  *
8170  * @param[in] dev
8171  *   Pointer to Ethernet device structure.
8172  * @param[in] cnt
8173  *   Index to counter to query.
8174  * @param[in] clear
8175  *   Set to clear counter statistics.
8176  * @param[out] pkts
8177  *   The counter hits packets number to save.
8178  * @param[out] bytes
8179  *   The counter hits bytes number to save.
8180  *
8181  * @return
8182  *   0 on success, a negative errno value otherwise.
8183  */
8184 int
8185 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
8186                    bool clear, uint64_t *pkts, uint64_t *bytes, void **action)
8187 {
8188         const struct mlx5_flow_driver_ops *fops;
8189         struct rte_flow_attr attr = { .transfer = 0 };
8190
8191         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
8192                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8193                 return fops->counter_query(dev, cnt, clear, pkts,
8194                                         bytes, action);
8195         }
8196         DRV_LOG(ERR,
8197                 "port %u counter query is not supported.",
8198                  dev->data->port_id);
8199         return -ENOTSUP;
8200 }
8201
8202 /**
8203  * Get information about HWS pre-configurable resources.
8204  *
8205  * @param[in] dev
8206  *   Pointer to the rte_eth_dev structure.
8207  * @param[out] port_info
8208  *   Pointer to port information.
8209  * @param[out] queue_info
8210  *   Pointer to queue information.
8211  * @param[out] error
8212  *   Pointer to error structure.
8213  *
8214  * @return
8215  *   0 on success, a negative errno value otherwise and rte_errno is set.
8216  */
8217 static int
8218 mlx5_flow_info_get(struct rte_eth_dev *dev,
8219                    struct rte_flow_port_info *port_info,
8220                    struct rte_flow_queue_info *queue_info,
8221                    struct rte_flow_error *error)
8222 {
8223         const struct mlx5_flow_driver_ops *fops;
8224
8225         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8226                 return rte_flow_error_set(error, ENOTSUP,
8227                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8228                                 NULL,
8229                                 "info get with incorrect steering mode");
8230         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8231         return fops->info_get(dev, port_info, queue_info, error);
8232 }
8233
8234 /**
8235  * Configure port HWS resources.
8236  *
8237  * @param[in] dev
8238  *   Pointer to the rte_eth_dev structure.
8239  * @param[in] port_attr
8240  *   Port configuration attributes.
8241  * @param[in] nb_queue
8242  *   Number of queue.
8243  * @param[in] queue_attr
8244  *   Array that holds attributes for each flow queue.
8245  * @param[out] error
8246  *   Pointer to error structure.
8247  *
8248  * @return
8249  *   0 on success, a negative errno value otherwise and rte_errno is set.
8250  */
8251 static int
8252 mlx5_flow_port_configure(struct rte_eth_dev *dev,
8253                          const struct rte_flow_port_attr *port_attr,
8254                          uint16_t nb_queue,
8255                          const struct rte_flow_queue_attr *queue_attr[],
8256                          struct rte_flow_error *error)
8257 {
8258         const struct mlx5_flow_driver_ops *fops;
8259
8260         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8261                 return rte_flow_error_set(error, ENOTSUP,
8262                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8263                                 NULL,
8264                                 "port configure with incorrect steering mode");
8265         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8266         return fops->configure(dev, port_attr, nb_queue, queue_attr, error);
8267 }
8268
8269 /**
8270  * Create flow item template.
8271  *
8272  * @param[in] dev
8273  *   Pointer to the rte_eth_dev structure.
8274  * @param[in] attr
8275  *   Pointer to the item template attributes.
8276  * @param[in] items
8277  *   The template item pattern.
8278  * @param[out] error
8279  *   Pointer to error structure.
8280  *
8281  * @return
8282  *   0 on success, a negative errno value otherwise and rte_errno is set.
8283  */
8284 static struct rte_flow_pattern_template *
8285 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev,
8286                 const struct rte_flow_pattern_template_attr *attr,
8287                 const struct rte_flow_item items[],
8288                 struct rte_flow_error *error)
8289 {
8290         const struct mlx5_flow_driver_ops *fops;
8291
8292         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) {
8293                 rte_flow_error_set(error, ENOTSUP,
8294                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8295                                 NULL,
8296                                 "pattern create with incorrect steering mode");
8297                 return NULL;
8298         }
8299         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8300         return fops->pattern_template_create(dev, attr, items, error);
8301 }
8302
8303 /**
8304  * Destroy flow item template.
8305  *
8306  * @param[in] dev
8307  *   Pointer to the rte_eth_dev structure.
8308  * @param[in] template
8309  *   Pointer to the item template to be destroyed.
8310  * @param[out] error
8311  *   Pointer to error structure.
8312  *
8313  * @return
8314  *   0 on success, a negative errno value otherwise and rte_errno is set.
8315  */
8316 static int
8317 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev,
8318                                    struct rte_flow_pattern_template *template,
8319                                    struct rte_flow_error *error)
8320 {
8321         const struct mlx5_flow_driver_ops *fops;
8322
8323         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8324                 return rte_flow_error_set(error, ENOTSUP,
8325                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8326                                 NULL,
8327                                 "pattern destroy with incorrect steering mode");
8328         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8329         return fops->pattern_template_destroy(dev, template, error);
8330 }
8331
8332 /**
8333  * Create flow item template.
8334  *
8335  * @param[in] dev
8336  *   Pointer to the rte_eth_dev structure.
8337  * @param[in] attr
8338  *   Pointer to the action template attributes.
8339  * @param[in] actions
8340  *   Associated actions (list terminated by the END action).
8341  * @param[in] masks
8342  *   List of actions that marks which of the action's member is constant.
8343  * @param[out] error
8344  *   Pointer to error structure.
8345  *
8346  * @return
8347  *   0 on success, a negative errno value otherwise and rte_errno is set.
8348  */
8349 static struct rte_flow_actions_template *
8350 mlx5_flow_actions_template_create(struct rte_eth_dev *dev,
8351                         const struct rte_flow_actions_template_attr *attr,
8352                         const struct rte_flow_action actions[],
8353                         const struct rte_flow_action masks[],
8354                         struct rte_flow_error *error)
8355 {
8356         const struct mlx5_flow_driver_ops *fops;
8357
8358         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) {
8359                 rte_flow_error_set(error, ENOTSUP,
8360                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8361                                 NULL,
8362                                 "action create with incorrect steering mode");
8363                 return NULL;
8364         }
8365         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8366         return fops->actions_template_create(dev, attr, actions, masks, error);
8367 }
8368
8369 /**
8370  * Destroy flow action template.
8371  *
8372  * @param[in] dev
8373  *   Pointer to the rte_eth_dev structure.
8374  * @param[in] template
8375  *   Pointer to the action template to be destroyed.
8376  * @param[out] error
8377  *   Pointer to error structure.
8378  *
8379  * @return
8380  *   0 on success, a negative errno value otherwise and rte_errno is set.
8381  */
8382 static int
8383 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev,
8384                                    struct rte_flow_actions_template *template,
8385                                    struct rte_flow_error *error)
8386 {
8387         const struct mlx5_flow_driver_ops *fops;
8388
8389         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8390                 return rte_flow_error_set(error, ENOTSUP,
8391                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8392                                 NULL,
8393                                 "action destroy with incorrect steering mode");
8394         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8395         return fops->actions_template_destroy(dev, template, error);
8396 }
8397
8398 /**
8399  * Create flow table.
8400  *
8401  * @param[in] dev
8402  *   Pointer to the rte_eth_dev structure.
8403  * @param[in] attr
8404  *   Pointer to the table attributes.
8405  * @param[in] item_templates
8406  *   Item template array to be binded to the table.
8407  * @param[in] nb_item_templates
8408  *   Number of item template.
8409  * @param[in] action_templates
8410  *   Action template array to be binded to the table.
8411  * @param[in] nb_action_templates
8412  *   Number of action template.
8413  * @param[out] error
8414  *   Pointer to error structure.
8415  *
8416  * @return
8417  *    Table on success, NULL otherwise and rte_errno is set.
8418  */
8419 static struct rte_flow_template_table *
8420 mlx5_flow_table_create(struct rte_eth_dev *dev,
8421                        const struct rte_flow_template_table_attr *attr,
8422                        struct rte_flow_pattern_template *item_templates[],
8423                        uint8_t nb_item_templates,
8424                        struct rte_flow_actions_template *action_templates[],
8425                        uint8_t nb_action_templates,
8426                        struct rte_flow_error *error)
8427 {
8428         const struct mlx5_flow_driver_ops *fops;
8429
8430         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) {
8431                 rte_flow_error_set(error, ENOTSUP,
8432                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8433                                 NULL,
8434                                 "table create with incorrect steering mode");
8435                 return NULL;
8436         }
8437         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8438         return fops->template_table_create(dev,
8439                                            attr,
8440                                            item_templates,
8441                                            nb_item_templates,
8442                                            action_templates,
8443                                            nb_action_templates,
8444                                            error);
8445 }
8446
8447 /**
8448  * PMD destroy flow table.
8449  *
8450  * @param[in] dev
8451  *   Pointer to the rte_eth_dev structure.
8452  * @param[in] table
8453  *   Pointer to the table to be destroyed.
8454  * @param[out] error
8455  *   Pointer to error structure.
8456  *
8457  * @return
8458  *   0 on success, a negative errno value otherwise and rte_errno is set.
8459  */
8460 static int
8461 mlx5_flow_table_destroy(struct rte_eth_dev *dev,
8462                         struct rte_flow_template_table *table,
8463                         struct rte_flow_error *error)
8464 {
8465         const struct mlx5_flow_driver_ops *fops;
8466
8467         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8468                 return rte_flow_error_set(error, ENOTSUP,
8469                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8470                                 NULL,
8471                                 "table destroy with incorrect steering mode");
8472         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8473         return fops->template_table_destroy(dev, table, error);
8474 }
8475
8476 /**
8477  * Enqueue flow creation.
8478  *
8479  * @param[in] dev
8480  *   Pointer to the rte_eth_dev structure.
8481  * @param[in] queue_id
8482  *   The queue to create the flow.
8483  * @param[in] attr
8484  *   Pointer to the flow operation attributes.
8485  * @param[in] items
8486  *   Items with flow spec value.
8487  * @param[in] pattern_template_index
8488  *   The item pattern flow follows from the table.
8489  * @param[in] actions
8490  *   Action with flow spec value.
8491  * @param[in] action_template_index
8492  *   The action pattern flow follows from the table.
8493  * @param[in] user_data
8494  *   Pointer to the user_data.
8495  * @param[out] error
8496  *   Pointer to error structure.
8497  *
8498  * @return
8499  *    Flow pointer on success, NULL otherwise and rte_errno is set.
8500  */
8501 static struct rte_flow *
8502 mlx5_flow_async_flow_create(struct rte_eth_dev *dev,
8503                             uint32_t queue_id,
8504                             const struct rte_flow_op_attr *attr,
8505                             struct rte_flow_template_table *table,
8506                             const struct rte_flow_item items[],
8507                             uint8_t pattern_template_index,
8508                             const struct rte_flow_action actions[],
8509                             uint8_t action_template_index,
8510                             void *user_data,
8511                             struct rte_flow_error *error)
8512 {
8513         const struct mlx5_flow_driver_ops *fops;
8514
8515         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) {
8516                 rte_flow_error_set(error, ENOTSUP,
8517                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8518                                 NULL,
8519                                 "flow_q create with incorrect steering mode");
8520                 return NULL;
8521         }
8522         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8523         return fops->async_flow_create(dev, queue_id, attr, table,
8524                                        items, pattern_template_index,
8525                                        actions, action_template_index,
8526                                        user_data, error);
8527 }
8528
8529 /**
8530  * Enqueue flow destruction.
8531  *
8532  * @param[in] dev
8533  *   Pointer to the rte_eth_dev structure.
8534  * @param[in] queue
8535  *   The queue to destroy the flow.
8536  * @param[in] attr
8537  *   Pointer to the flow operation attributes.
8538  * @param[in] flow
8539  *   Pointer to the flow to be destroyed.
8540  * @param[in] user_data
8541  *   Pointer to the user_data.
8542  * @param[out] error
8543  *   Pointer to error structure.
8544  *
8545  * @return
8546  *    0 on success, negative value otherwise and rte_errno is set.
8547  */
8548 static int
8549 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev,
8550                              uint32_t queue,
8551                              const struct rte_flow_op_attr *attr,
8552                              struct rte_flow *flow,
8553                              void *user_data,
8554                              struct rte_flow_error *error)
8555 {
8556         const struct mlx5_flow_driver_ops *fops;
8557
8558         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8559                 return rte_flow_error_set(error, ENOTSUP,
8560                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8561                                 NULL,
8562                                 "flow_q destroy with incorrect steering mode");
8563         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8564         return fops->async_flow_destroy(dev, queue, attr, flow,
8565                                         user_data, error);
8566 }
8567
8568 /**
8569  * Pull the enqueued flows.
8570  *
8571  * @param[in] dev
8572  *   Pointer to the rte_eth_dev structure.
8573  * @param[in] queue
8574  *   The queue to pull the result.
8575  * @param[in/out] res
8576  *   Array to save the results.
8577  * @param[in] n_res
8578  *   Available result with the array.
8579  * @param[out] error
8580  *   Pointer to error structure.
8581  *
8582  * @return
8583  *    Result number on success, negative value otherwise and rte_errno is set.
8584  */
8585 static int
8586 mlx5_flow_pull(struct rte_eth_dev *dev,
8587                uint32_t queue,
8588                struct rte_flow_op_result res[],
8589                uint16_t n_res,
8590                struct rte_flow_error *error)
8591 {
8592         const struct mlx5_flow_driver_ops *fops;
8593
8594         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8595                 return rte_flow_error_set(error, ENOTSUP,
8596                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8597                                 NULL,
8598                                 "flow_q pull with incorrect steering mode");
8599         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8600         return fops->pull(dev, queue, res, n_res, error);
8601 }
8602
8603 /**
8604  * Push the enqueued flows.
8605  *
8606  * @param[in] dev
8607  *   Pointer to the rte_eth_dev structure.
8608  * @param[in] queue
8609  *   The queue to push the flows.
8610  * @param[out] error
8611  *   Pointer to error structure.
8612  *
8613  * @return
8614  *    0 on success, negative value otherwise and rte_errno is set.
8615  */
8616 static int
8617 mlx5_flow_push(struct rte_eth_dev *dev,
8618                uint32_t queue,
8619                struct rte_flow_error *error)
8620 {
8621         const struct mlx5_flow_driver_ops *fops;
8622
8623         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8624                 return rte_flow_error_set(error, ENOTSUP,
8625                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8626                                 NULL,
8627                                 "flow_q push with incorrect steering mode");
8628         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8629         return fops->push(dev, queue, error);
8630 }
8631
8632 /**
8633  * Create shared action.
8634  *
8635  * @param[in] dev
8636  *   Pointer to the rte_eth_dev structure.
8637  * @param[in] queue
8638  *   Which queue to be used..
8639  * @param[in] attr
8640  *   Operation attribute.
8641  * @param[in] conf
8642  *   Indirect action configuration.
8643  * @param[in] action
8644  *   rte_flow action detail.
8645  * @param[in] user_data
8646  *   Pointer to the user_data.
8647  * @param[out] error
8648  *   Pointer to error structure.
8649  *
8650  * @return
8651  *   Action handle on success, NULL otherwise and rte_errno is set.
8652  */
8653 static struct rte_flow_action_handle *
8654 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue,
8655                                  const struct rte_flow_op_attr *attr,
8656                                  const struct rte_flow_indir_action_conf *conf,
8657                                  const struct rte_flow_action *action,
8658                                  void *user_data,
8659                                  struct rte_flow_error *error)
8660 {
8661         const struct mlx5_flow_driver_ops *fops =
8662                         flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8663
8664         return fops->async_action_create(dev, queue, attr, conf, action,
8665                                          user_data, error);
8666 }
8667
8668 /**
8669  * Update shared action.
8670  *
8671  * @param[in] dev
8672  *   Pointer to the rte_eth_dev structure.
8673  * @param[in] queue
8674  *   Which queue to be used..
8675  * @param[in] attr
8676  *   Operation attribute.
8677  * @param[in] handle
8678  *   Action handle to be updated.
8679  * @param[in] update
8680  *   Update value.
8681  * @param[in] user_data
8682  *   Pointer to the user_data.
8683  * @param[out] error
8684  *   Pointer to error structure.
8685  *
8686  * @return
8687  *   0 on success, negative value otherwise and rte_errno is set.
8688  */
8689 static int
8690 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue,
8691                                      const struct rte_flow_op_attr *attr,
8692                                      struct rte_flow_action_handle *handle,
8693                                      const void *update,
8694                                      void *user_data,
8695                                      struct rte_flow_error *error)
8696 {
8697         const struct mlx5_flow_driver_ops *fops =
8698                         flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8699
8700         return fops->async_action_update(dev, queue, attr, handle,
8701                                          update, user_data, error);
8702 }
8703
8704 /**
8705  * Destroy shared action.
8706  *
8707  * @param[in] dev
8708  *   Pointer to the rte_eth_dev structure.
8709  * @param[in] queue
8710  *   Which queue to be used..
8711  * @param[in] attr
8712  *   Operation attribute.
8713  * @param[in] handle
8714  *   Action handle to be destroyed.
8715  * @param[in] user_data
8716  *   Pointer to the user_data.
8717  * @param[out] error
8718  *   Pointer to error structure.
8719  *
8720  * @return
8721  *   0 on success, negative value otherwise and rte_errno is set.
8722  */
8723 static int
8724 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue,
8725                                       const struct rte_flow_op_attr *attr,
8726                                       struct rte_flow_action_handle *handle,
8727                                       void *user_data,
8728                                       struct rte_flow_error *error)
8729 {
8730         const struct mlx5_flow_driver_ops *fops =
8731                         flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8732
8733         return fops->async_action_destroy(dev, queue, attr, handle,
8734                                           user_data, error);
8735 }
8736
8737 /**
8738  * Allocate a new memory for the counter values wrapped by all the needed
8739  * management.
8740  *
8741  * @param[in] sh
8742  *   Pointer to mlx5_dev_ctx_shared object.
8743  *
8744  * @return
8745  *   0 on success, a negative errno value otherwise.
8746  */
8747 static int
8748 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
8749 {
8750         struct mlx5_counter_stats_mem_mng *mem_mng;
8751         volatile struct flow_counter_stats *raw_data;
8752         int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
8753         int size = (sizeof(struct flow_counter_stats) *
8754                         MLX5_COUNTERS_PER_POOL +
8755                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
8756                         sizeof(struct mlx5_counter_stats_mem_mng);
8757         size_t pgsize = rte_mem_page_size();
8758         uint8_t *mem;
8759         int ret;
8760         int i;
8761
8762         if (pgsize == (size_t)-1) {
8763                 DRV_LOG(ERR, "Failed to get mem page size");
8764                 rte_errno = ENOMEM;
8765                 return -ENOMEM;
8766         }
8767         mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
8768         if (!mem) {
8769                 rte_errno = ENOMEM;
8770                 return -ENOMEM;
8771         }
8772         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
8773         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
8774         ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd,
8775                                           sh->cdev->pdn, mem, size,
8776                                           &mem_mng->wm);
8777         if (ret) {
8778                 rte_errno = errno;
8779                 mlx5_free(mem);
8780                 return -rte_errno;
8781         }
8782         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
8783         raw_data = (volatile struct flow_counter_stats *)mem;
8784         for (i = 0; i < raws_n; ++i) {
8785                 mem_mng->raws[i].mem_mng = mem_mng;
8786                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
8787         }
8788         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
8789                 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
8790                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
8791                                  next);
8792         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
8793         sh->cmng.mem_mng = mem_mng;
8794         return 0;
8795 }
8796
8797 /**
8798  * Set the statistic memory to the new counter pool.
8799  *
8800  * @param[in] sh
8801  *   Pointer to mlx5_dev_ctx_shared object.
8802  * @param[in] pool
8803  *   Pointer to the pool to set the statistic memory.
8804  *
8805  * @return
8806  *   0 on success, a negative errno value otherwise.
8807  */
8808 static int
8809 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
8810                                struct mlx5_flow_counter_pool *pool)
8811 {
8812         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
8813         /* Resize statistic memory once used out. */
8814         if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
8815             mlx5_flow_create_counter_stat_mem_mng(sh)) {
8816                 DRV_LOG(ERR, "Cannot resize counter stat mem.");
8817                 return -1;
8818         }
8819         rte_spinlock_lock(&pool->sl);
8820         pool->raw = cmng->mem_mng->raws + pool->index %
8821                     MLX5_CNT_CONTAINER_RESIZE;
8822         rte_spinlock_unlock(&pool->sl);
8823         pool->raw_hw = NULL;
8824         return 0;
8825 }
8826
8827 #define MLX5_POOL_QUERY_FREQ_US 1000000
8828
8829 /**
8830  * Set the periodic procedure for triggering asynchronous batch queries for all
8831  * the counter pools.
8832  *
8833  * @param[in] sh
8834  *   Pointer to mlx5_dev_ctx_shared object.
8835  */
8836 void
8837 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
8838 {
8839         uint32_t pools_n, us;
8840
8841         pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED);
8842         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
8843         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
8844         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
8845                 sh->cmng.query_thread_on = 0;
8846                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
8847         } else {
8848                 sh->cmng.query_thread_on = 1;
8849         }
8850 }
8851
8852 /**
8853  * The periodic procedure for triggering asynchronous batch queries for all the
8854  * counter pools. This function is probably called by the host thread.
8855  *
8856  * @param[in] arg
8857  *   The parameter for the alarm process.
8858  */
8859 void
8860 mlx5_flow_query_alarm(void *arg)
8861 {
8862         struct mlx5_dev_ctx_shared *sh = arg;
8863         int ret;
8864         uint16_t pool_index = sh->cmng.pool_index;
8865         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
8866         struct mlx5_flow_counter_pool *pool;
8867         uint16_t n_valid;
8868
8869         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
8870                 goto set_alarm;
8871         rte_spinlock_lock(&cmng->pool_update_sl);
8872         pool = cmng->pools[pool_index];
8873         n_valid = cmng->n_valid;
8874         rte_spinlock_unlock(&cmng->pool_update_sl);
8875         /* Set the statistic memory to the new created pool. */
8876         if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
8877                 goto set_alarm;
8878         if (pool->raw_hw)
8879                 /* There is a pool query in progress. */
8880                 goto set_alarm;
8881         pool->raw_hw =
8882                 LIST_FIRST(&sh->cmng.free_stat_raws);
8883         if (!pool->raw_hw)
8884                 /* No free counter statistics raw memory. */
8885                 goto set_alarm;
8886         /*
8887          * Identify the counters released between query trigger and query
8888          * handle more efficiently. The counter released in this gap period
8889          * should wait for a new round of query as the new arrived packets
8890          * will not be taken into account.
8891          */
8892         pool->query_gen++;
8893         ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
8894                                                MLX5_COUNTERS_PER_POOL,
8895                                                NULL, NULL,
8896                                                pool->raw_hw->mem_mng->wm.lkey,
8897                                                (void *)(uintptr_t)
8898                                                pool->raw_hw->data,
8899                                                sh->devx_comp,
8900                                                (uint64_t)(uintptr_t)pool);
8901         if (ret) {
8902                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
8903                         " %d", pool->min_dcs->id);
8904                 pool->raw_hw = NULL;
8905                 goto set_alarm;
8906         }
8907         LIST_REMOVE(pool->raw_hw, next);
8908         sh->cmng.pending_queries++;
8909         pool_index++;
8910         if (pool_index >= n_valid)
8911                 pool_index = 0;
8912 set_alarm:
8913         sh->cmng.pool_index = pool_index;
8914         mlx5_set_query_alarm(sh);
8915 }
8916
8917 /**
8918  * Check and callback event for new aged flow in the counter pool
8919  *
8920  * @param[in] sh
8921  *   Pointer to mlx5_dev_ctx_shared object.
8922  * @param[in] pool
8923  *   Pointer to Current counter pool.
8924  */
8925 static void
8926 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
8927                    struct mlx5_flow_counter_pool *pool)
8928 {
8929         struct mlx5_priv *priv;
8930         struct mlx5_flow_counter *cnt;
8931         struct mlx5_age_info *age_info;
8932         struct mlx5_age_param *age_param;
8933         struct mlx5_counter_stats_raw *cur = pool->raw_hw;
8934         struct mlx5_counter_stats_raw *prev = pool->raw;
8935         const uint64_t curr_time = MLX5_CURR_TIME_SEC;
8936         const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
8937         uint16_t expected = AGE_CANDIDATE;
8938         uint32_t i;
8939
8940         pool->time_of_last_age_check = curr_time;
8941         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
8942                 cnt = MLX5_POOL_GET_CNT(pool, i);
8943                 age_param = MLX5_CNT_TO_AGE(cnt);
8944                 if (__atomic_load_n(&age_param->state,
8945                                     __ATOMIC_RELAXED) != AGE_CANDIDATE)
8946                         continue;
8947                 if (cur->data[i].hits != prev->data[i].hits) {
8948                         __atomic_store_n(&age_param->sec_since_last_hit, 0,
8949                                          __ATOMIC_RELAXED);
8950                         continue;
8951                 }
8952                 if (__atomic_add_fetch(&age_param->sec_since_last_hit,
8953                                        time_delta,
8954                                        __ATOMIC_RELAXED) <= age_param->timeout)
8955                         continue;
8956                 /**
8957                  * Hold the lock first, or if between the
8958                  * state AGE_TMOUT and tailq operation the
8959                  * release happened, the release procedure
8960                  * may delete a non-existent tailq node.
8961                  */
8962                 priv = rte_eth_devices[age_param->port_id].data->dev_private;
8963                 age_info = GET_PORT_AGE_INFO(priv);
8964                 rte_spinlock_lock(&age_info->aged_sl);
8965                 if (__atomic_compare_exchange_n(&age_param->state, &expected,
8966                                                 AGE_TMOUT, false,
8967                                                 __ATOMIC_RELAXED,
8968                                                 __ATOMIC_RELAXED)) {
8969                         TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
8970                         MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
8971                 }
8972                 rte_spinlock_unlock(&age_info->aged_sl);
8973         }
8974         mlx5_age_event_prepare(sh);
8975 }
8976
8977 /**
8978  * Handler for the HW respond about ready values from an asynchronous batch
8979  * query. This function is probably called by the host thread.
8980  *
8981  * @param[in] sh
8982  *   The pointer to the shared device context.
8983  * @param[in] async_id
8984  *   The Devx async ID.
8985  * @param[in] status
8986  *   The status of the completion.
8987  */
8988 void
8989 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
8990                                   uint64_t async_id, int status)
8991 {
8992         struct mlx5_flow_counter_pool *pool =
8993                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
8994         struct mlx5_counter_stats_raw *raw_to_free;
8995         uint8_t query_gen = pool->query_gen ^ 1;
8996         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
8997         enum mlx5_counter_type cnt_type =
8998                 pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
8999                                 MLX5_COUNTER_TYPE_ORIGIN;
9000
9001         if (unlikely(status)) {
9002                 raw_to_free = pool->raw_hw;
9003         } else {
9004                 raw_to_free = pool->raw;
9005                 if (pool->is_aged)
9006                         mlx5_flow_aging_check(sh, pool);
9007                 rte_spinlock_lock(&pool->sl);
9008                 pool->raw = pool->raw_hw;
9009                 rte_spinlock_unlock(&pool->sl);
9010                 /* Be sure the new raw counters data is updated in memory. */
9011                 rte_io_wmb();
9012                 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
9013                         rte_spinlock_lock(&cmng->csl[cnt_type]);
9014                         TAILQ_CONCAT(&cmng->counters[cnt_type],
9015                                      &pool->counters[query_gen], next);
9016                         rte_spinlock_unlock(&cmng->csl[cnt_type]);
9017                 }
9018         }
9019         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
9020         pool->raw_hw = NULL;
9021         sh->cmng.pending_queries--;
9022 }
9023
9024 static int
9025 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
9026                     const struct flow_grp_info *grp_info,
9027                     struct rte_flow_error *error)
9028 {
9029         if (grp_info->transfer && grp_info->external &&
9030             grp_info->fdb_def_rule) {
9031                 if (group == UINT32_MAX)
9032                         return rte_flow_error_set
9033                                                 (error, EINVAL,
9034                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
9035                                                  NULL,
9036                                                  "group index not supported");
9037                 *table = group + 1;
9038         } else {
9039                 *table = group;
9040         }
9041         DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
9042         return 0;
9043 }
9044
9045 /**
9046  * Translate the rte_flow group index to HW table value.
9047  *
9048  * If tunnel offload is disabled, all group ids converted to flow table
9049  * id using the standard method.
9050  * If tunnel offload is enabled, group id can be converted using the
9051  * standard or tunnel conversion method. Group conversion method
9052  * selection depends on flags in `grp_info` parameter:
9053  * - Internal (grp_info.external == 0) groups conversion uses the
9054  *   standard method.
9055  * - Group ids in JUMP action converted with the tunnel conversion.
9056  * - Group id in rule attribute conversion depends on a rule type and
9057  *   group id value:
9058  *   ** non zero group attributes converted with the tunnel method
9059  *   ** zero group attribute in non-tunnel rule is converted using the
9060  *      standard method - there's only one root table
9061  *   ** zero group attribute in steer tunnel rule is converted with the
9062  *      standard method - single root table
9063  *   ** zero group attribute in match tunnel rule is a special OvS
9064  *      case: that value is used for portability reasons. That group
9065  *      id is converted with the tunnel conversion method.
9066  *
9067  * @param[in] dev
9068  *   Port device
9069  * @param[in] tunnel
9070  *   PMD tunnel offload object
9071  * @param[in] group
9072  *   rte_flow group index value.
9073  * @param[out] table
9074  *   HW table value.
9075  * @param[in] grp_info
9076  *   flags used for conversion
9077  * @param[out] error
9078  *   Pointer to error structure.
9079  *
9080  * @return
9081  *   0 on success, a negative errno value otherwise and rte_errno is set.
9082  */
9083 int
9084 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
9085                          const struct mlx5_flow_tunnel *tunnel,
9086                          uint32_t group, uint32_t *table,
9087                          const struct flow_grp_info *grp_info,
9088                          struct rte_flow_error *error)
9089 {
9090         int ret;
9091         bool standard_translation;
9092
9093         if (!grp_info->skip_scale && grp_info->external &&
9094             group < MLX5_MAX_TABLES_EXTERNAL)
9095                 group *= MLX5_FLOW_TABLE_FACTOR;
9096         if (is_tunnel_offload_active(dev)) {
9097                 standard_translation = !grp_info->external ||
9098                                         grp_info->std_tbl_fix;
9099         } else {
9100                 standard_translation = true;
9101         }
9102         DRV_LOG(DEBUG,
9103                 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s",
9104                 dev->data->port_id, group, grp_info->transfer,
9105                 grp_info->external, grp_info->fdb_def_rule,
9106                 standard_translation ? "STANDARD" : "TUNNEL");
9107         if (standard_translation)
9108                 ret = flow_group_to_table(dev->data->port_id, group, table,
9109                                           grp_info, error);
9110         else
9111                 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
9112                                                       table, error);
9113
9114         return ret;
9115 }
9116
9117 /**
9118  * Discover availability of metadata reg_c's.
9119  *
9120  * Iteratively use test flows to check availability.
9121  *
9122  * @param[in] dev
9123  *   Pointer to the Ethernet device structure.
9124  *
9125  * @return
9126  *   0 on success, a negative errno value otherwise and rte_errno is set.
9127  */
9128 int
9129 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
9130 {
9131         struct mlx5_priv *priv = dev->data->dev_private;
9132         enum modify_reg idx;
9133         int n = 0;
9134
9135         /* reg_c[0] and reg_c[1] are reserved. */
9136         priv->sh->flow_mreg_c[n++] = REG_C_0;
9137         priv->sh->flow_mreg_c[n++] = REG_C_1;
9138         /* Discover availability of other reg_c's. */
9139         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
9140                 struct rte_flow_attr attr = {
9141                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
9142                         .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
9143                         .ingress = 1,
9144                 };
9145                 struct rte_flow_item items[] = {
9146                         [0] = {
9147                                 .type = RTE_FLOW_ITEM_TYPE_END,
9148                         },
9149                 };
9150                 struct rte_flow_action actions[] = {
9151                         [0] = {
9152                                 .type = (enum rte_flow_action_type)
9153                                         MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
9154                                 .conf = &(struct mlx5_flow_action_copy_mreg){
9155                                         .src = REG_C_1,
9156                                         .dst = idx,
9157                                 },
9158                         },
9159                         [1] = {
9160                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
9161                                 .conf = &(struct rte_flow_action_jump){
9162                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
9163                                 },
9164                         },
9165                         [2] = {
9166                                 .type = RTE_FLOW_ACTION_TYPE_END,
9167                         },
9168                 };
9169                 uint32_t flow_idx;
9170                 struct rte_flow *flow;
9171                 struct rte_flow_error error;
9172
9173                 if (!priv->sh->config.dv_flow_en)
9174                         break;
9175                 /* Create internal flow, validation skips copy action. */
9176                 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
9177                                         items, actions, false, &error);
9178                 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
9179                                       flow_idx);
9180                 if (!flow)
9181                         continue;
9182                 priv->sh->flow_mreg_c[n++] = idx;
9183                 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
9184         }
9185         for (; n < MLX5_MREG_C_NUM; ++n)
9186                 priv->sh->flow_mreg_c[n] = REG_NON;
9187         priv->sh->metadata_regc_check_flag = 1;
9188         return 0;
9189 }
9190
9191 int
9192 save_dump_file(const uint8_t *data, uint32_t size,
9193         uint32_t type, uint64_t id, void *arg, FILE *file)
9194 {
9195         char line[BUF_SIZE];
9196         uint32_t out = 0;
9197         uint32_t k;
9198         uint32_t actions_num;
9199         struct rte_flow_query_count *count;
9200
9201         memset(line, 0, BUF_SIZE);
9202         switch (type) {
9203         case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR:
9204                 actions_num = *(uint32_t *)(arg);
9205                 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,",
9206                                 type, id, actions_num);
9207                 break;
9208         case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT:
9209                 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",",
9210                                 type, id);
9211                 break;
9212         case DR_DUMP_REC_TYPE_PMD_COUNTER:
9213                 count = (struct rte_flow_query_count *)arg;
9214                 fprintf(file,
9215                         "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n",
9216                         type, id, count->hits, count->bytes);
9217                 return 0;
9218         default:
9219                 return -1;
9220         }
9221
9222         for (k = 0; k < size; k++) {
9223                 /* Make sure we do not overrun the line buffer length. */
9224                 if (out >= BUF_SIZE - 4) {
9225                         line[out] = '\0';
9226                         break;
9227                 }
9228                 out += snprintf(line + out, BUF_SIZE - out, "%02x",
9229                                 (data[k]) & 0xff);
9230         }
9231         fprintf(file, "%s\n", line);
9232         return 0;
9233 }
9234
9235 int
9236 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow,
9237         struct rte_flow_query_count *count, struct rte_flow_error *error)
9238 {
9239         struct rte_flow_action action[2];
9240         enum mlx5_flow_drv_type ftype;
9241         const struct mlx5_flow_driver_ops *fops;
9242
9243         if (!flow) {
9244                 return rte_flow_error_set(error, ENOENT,
9245                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9246                                 NULL,
9247                                 "invalid flow handle");
9248         }
9249         action[0].type = RTE_FLOW_ACTION_TYPE_COUNT;
9250         action[1].type = RTE_FLOW_ACTION_TYPE_END;
9251         if (flow->counter) {
9252                 memset(count, 0, sizeof(struct rte_flow_query_count));
9253                 ftype = (enum mlx5_flow_drv_type)(flow->drv_type);
9254                 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN &&
9255                                                 ftype < MLX5_FLOW_TYPE_MAX);
9256                 fops = flow_get_drv_ops(ftype);
9257                 return fops->query(dev, flow, action, count, error);
9258         }
9259         return -1;
9260 }
9261
9262 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
9263 /**
9264  * Dump flow ipool data to file
9265  *
9266  * @param[in] dev
9267  *   The pointer to Ethernet device.
9268  * @param[in] file
9269  *   A pointer to a file for output.
9270  * @param[out] error
9271  *   Perform verbose error reporting if not NULL. PMDs initialize this
9272  *   structure in case of error only.
9273  * @return
9274  *   0 on success, a negative value otherwise.
9275  */
9276 int
9277 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev,
9278         struct rte_flow *flow, FILE *file,
9279         struct rte_flow_error *error)
9280 {
9281         struct mlx5_priv *priv = dev->data->dev_private;
9282         struct mlx5_flow_dv_modify_hdr_resource  *modify_hdr;
9283         struct mlx5_flow_dv_encap_decap_resource *encap_decap;
9284         uint32_t handle_idx;
9285         struct mlx5_flow_handle *dh;
9286         struct rte_flow_query_count count;
9287         uint32_t actions_num;
9288         const uint8_t *data;
9289         size_t size;
9290         uint64_t id;
9291         uint32_t type;
9292         void *action = NULL;
9293
9294         if (!flow) {
9295                 return rte_flow_error_set(error, ENOENT,
9296                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9297                                 NULL,
9298                                 "invalid flow handle");
9299         }
9300         handle_idx = flow->dev_handles;
9301         /* query counter */
9302         if (flow->counter &&
9303         (!mlx5_counter_query(dev, flow->counter, false,
9304         &count.hits, &count.bytes, &action)) && action) {
9305                 id = (uint64_t)(uintptr_t)action;
9306                 type = DR_DUMP_REC_TYPE_PMD_COUNTER;
9307                 save_dump_file(NULL, 0, type,
9308                         id, (void *)&count, file);
9309         }
9310
9311         while (handle_idx) {
9312                 dh = mlx5_ipool_get(priv->sh->ipool
9313                                 [MLX5_IPOOL_MLX5_FLOW], handle_idx);
9314                 if (!dh)
9315                         continue;
9316                 handle_idx = dh->next.next;
9317
9318                 /* Get modify_hdr and encap_decap buf from ipools. */
9319                 encap_decap = NULL;
9320                 modify_hdr = dh->dvh.modify_hdr;
9321
9322                 if (dh->dvh.rix_encap_decap) {
9323                         encap_decap = mlx5_ipool_get(priv->sh->ipool
9324                                                 [MLX5_IPOOL_DECAP_ENCAP],
9325                                                 dh->dvh.rix_encap_decap);
9326                 }
9327                 if (modify_hdr) {
9328                         data = (const uint8_t *)modify_hdr->actions;
9329                         size = (size_t)(modify_hdr->actions_num) * 8;
9330                         id = (uint64_t)(uintptr_t)modify_hdr->action;
9331                         actions_num = modify_hdr->actions_num;
9332                         type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR;
9333                         save_dump_file(data, size, type, id,
9334                                                 (void *)(&actions_num), file);
9335                 }
9336                 if (encap_decap) {
9337                         data = encap_decap->buf;
9338                         size = encap_decap->size;
9339                         id = (uint64_t)(uintptr_t)encap_decap->action;
9340                         type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT;
9341                         save_dump_file(data, size, type,
9342                                                 id, NULL, file);
9343                 }
9344         }
9345         return 0;
9346 }
9347
9348 /**
9349  * Dump all flow's encap_decap/modify_hdr/counter data to file
9350  *
9351  * @param[in] dev
9352  *   The pointer to Ethernet device.
9353  * @param[in] file
9354  *   A pointer to a file for output.
9355  * @param[out] error
9356  *   Perform verbose error reporting if not NULL. PMDs initialize this
9357  *   structure in case of error only.
9358  * @return
9359  *   0 on success, a negative value otherwise.
9360  */
9361 static int
9362 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev,
9363         FILE *file, struct rte_flow_error *error __rte_unused)
9364 {
9365         struct mlx5_priv *priv = dev->data->dev_private;
9366         struct mlx5_dev_ctx_shared *sh = priv->sh;
9367         struct mlx5_hlist *h;
9368         struct mlx5_flow_dv_modify_hdr_resource  *modify_hdr;
9369         struct mlx5_flow_dv_encap_decap_resource *encap_decap;
9370         struct rte_flow_query_count count;
9371         uint32_t actions_num;
9372         const uint8_t *data;
9373         size_t size;
9374         uint64_t id;
9375         uint32_t type;
9376         uint32_t i;
9377         uint32_t j;
9378         struct mlx5_list_inconst *l_inconst;
9379         struct mlx5_list_entry *e;
9380         int lcore_index;
9381         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
9382         uint32_t max;
9383         void *action;
9384
9385         /* encap_decap hlist is lcore_share, get global core cache. */
9386         i = MLX5_LIST_GLOBAL;
9387         h = sh->encaps_decaps;
9388         if (h) {
9389                 for (j = 0; j <= h->mask; j++) {
9390                         l_inconst = &h->buckets[j].l;
9391                         if (!l_inconst || !l_inconst->cache[i])
9392                                 continue;
9393
9394                         e = LIST_FIRST(&l_inconst->cache[i]->h);
9395                         while (e) {
9396                                 encap_decap =
9397                                 (struct mlx5_flow_dv_encap_decap_resource *)e;
9398                                 data = encap_decap->buf;
9399                                 size = encap_decap->size;
9400                                 id = (uint64_t)(uintptr_t)encap_decap->action;
9401                                 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT;
9402                                 save_dump_file(data, size, type,
9403                                         id, NULL, file);
9404                                 e = LIST_NEXT(e, next);
9405                         }
9406                 }
9407         }
9408
9409         /* get modify_hdr */
9410         h = sh->modify_cmds;
9411         if (h) {
9412                 lcore_index = rte_lcore_index(rte_lcore_id());
9413                 if (unlikely(lcore_index == -1)) {
9414                         lcore_index = MLX5_LIST_NLCORE;
9415                         rte_spinlock_lock(&h->l_const.lcore_lock);
9416                 }
9417                 i = lcore_index;
9418
9419                 for (j = 0; j <= h->mask; j++) {
9420                         l_inconst = &h->buckets[j].l;
9421                         if (!l_inconst || !l_inconst->cache[i])
9422                                 continue;
9423
9424                         e = LIST_FIRST(&l_inconst->cache[i]->h);
9425                         while (e) {
9426                                 modify_hdr =
9427                                 (struct mlx5_flow_dv_modify_hdr_resource *)e;
9428                                 data = (const uint8_t *)modify_hdr->actions;
9429                                 size = (size_t)(modify_hdr->actions_num) * 8;
9430                                 actions_num = modify_hdr->actions_num;
9431                                 id = (uint64_t)(uintptr_t)modify_hdr->action;
9432                                 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR;
9433                                 save_dump_file(data, size, type, id,
9434                                                 (void *)(&actions_num), file);
9435                                 e = LIST_NEXT(e, next);
9436                         }
9437                 }
9438
9439                 if (unlikely(lcore_index == MLX5_LIST_NLCORE))
9440                         rte_spinlock_unlock(&h->l_const.lcore_lock);
9441         }
9442
9443         /* get counter */
9444         MLX5_ASSERT(cmng->n_valid <= cmng->n);
9445         max = MLX5_COUNTERS_PER_POOL * cmng->n_valid;
9446         for (j = 1; j <= max; j++) {
9447                 action = NULL;
9448                 if ((!mlx5_counter_query(dev, j, false, &count.hits,
9449                 &count.bytes, &action)) && action) {
9450                         id = (uint64_t)(uintptr_t)action;
9451                         type = DR_DUMP_REC_TYPE_PMD_COUNTER;
9452                         save_dump_file(NULL, 0, type,
9453                                         id, (void *)&count, file);
9454                 }
9455         }
9456         return 0;
9457 }
9458 #endif
9459
9460 /**
9461  * Dump flow raw hw data to file
9462  *
9463  * @param[in] dev
9464  *    The pointer to Ethernet device.
9465  * @param[in] file
9466  *   A pointer to a file for output.
9467  * @param[out] error
9468  *   Perform verbose error reporting if not NULL. PMDs initialize this
9469  *   structure in case of error only.
9470  * @return
9471  *   0 on success, a negative value otherwise.
9472  */
9473 int
9474 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
9475                    FILE *file,
9476                    struct rte_flow_error *error __rte_unused)
9477 {
9478         struct mlx5_priv *priv = dev->data->dev_private;
9479         struct mlx5_dev_ctx_shared *sh = priv->sh;
9480         uint32_t handle_idx;
9481         int ret;
9482         struct mlx5_flow_handle *dh;
9483         struct rte_flow *flow;
9484
9485         if (!sh->config.dv_flow_en) {
9486                 if (fputs("device dv flow disabled\n", file) <= 0)
9487                         return -errno;
9488                 return -ENOTSUP;
9489         }
9490
9491         /* dump all */
9492         if (!flow_idx) {
9493 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
9494                 if (mlx5_flow_dev_dump_sh_all(dev, file, error))
9495                         return -EINVAL;
9496 #endif
9497                 return mlx5_devx_cmd_flow_dump(sh->fdb_domain,
9498                                         sh->rx_domain,
9499                                         sh->tx_domain, file);
9500         }
9501         /* dump one */
9502         flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
9503                         (uintptr_t)(void *)flow_idx);
9504         if (!flow)
9505                 return -EINVAL;
9506
9507 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
9508         mlx5_flow_dev_dump_ipool(dev, flow, file, error);
9509 #endif
9510         handle_idx = flow->dev_handles;
9511         while (handle_idx) {
9512                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
9513                                 handle_idx);
9514                 if (!dh)
9515                         return -ENOENT;
9516                 if (dh->drv_flow) {
9517                         ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow,
9518                                         file);
9519                         if (ret)
9520                                 return -ENOENT;
9521                 }
9522                 handle_idx = dh->next.next;
9523         }
9524         return 0;
9525 }
9526
9527 /**
9528  * Get aged-out flows.
9529  *
9530  * @param[in] dev
9531  *   Pointer to the Ethernet device structure.
9532  * @param[in] context
9533  *   The address of an array of pointers to the aged-out flows contexts.
9534  * @param[in] nb_countexts
9535  *   The length of context array pointers.
9536  * @param[out] error
9537  *   Perform verbose error reporting if not NULL. Initialized in case of
9538  *   error only.
9539  *
9540  * @return
9541  *   how many contexts get in success, otherwise negative errno value.
9542  *   if nb_contexts is 0, return the amount of all aged contexts.
9543  *   if nb_contexts is not 0 , return the amount of aged flows reported
9544  *   in the context array.
9545  */
9546 int
9547 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
9548                         uint32_t nb_contexts, struct rte_flow_error *error)
9549 {
9550         const struct mlx5_flow_driver_ops *fops;
9551         struct rte_flow_attr attr = { .transfer = 0 };
9552
9553         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
9554                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
9555                 return fops->get_aged_flows(dev, contexts, nb_contexts,
9556                                                     error);
9557         }
9558         DRV_LOG(ERR,
9559                 "port %u get aged flows is not supported.",
9560                  dev->data->port_id);
9561         return -ENOTSUP;
9562 }
9563
9564 /* Wrapper for driver action_validate op callback */
9565 static int
9566 flow_drv_action_validate(struct rte_eth_dev *dev,
9567                          const struct rte_flow_indir_action_conf *conf,
9568                          const struct rte_flow_action *action,
9569                          const struct mlx5_flow_driver_ops *fops,
9570                          struct rte_flow_error *error)
9571 {
9572         static const char err_msg[] = "indirect action validation unsupported";
9573
9574         if (!fops->action_validate) {
9575                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
9576                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
9577                                    NULL, err_msg);
9578                 return -rte_errno;
9579         }
9580         return fops->action_validate(dev, conf, action, error);
9581 }
9582
9583 /**
9584  * Destroys the shared action by handle.
9585  *
9586  * @param dev
9587  *   Pointer to Ethernet device structure.
9588  * @param[in] handle
9589  *   Handle for the indirect action object to be destroyed.
9590  * @param[out] error
9591  *   Perform verbose error reporting if not NULL. PMDs initialize this
9592  *   structure in case of error only.
9593  *
9594  * @return
9595  *   0 on success, a negative errno value otherwise and rte_errno is set.
9596  *
9597  * @note: wrapper for driver action_create op callback.
9598  */
9599 static int
9600 mlx5_action_handle_destroy(struct rte_eth_dev *dev,
9601                            struct rte_flow_action_handle *handle,
9602                            struct rte_flow_error *error)
9603 {
9604         static const char err_msg[] = "indirect action destruction unsupported";
9605         struct rte_flow_attr attr = { .transfer = 0 };
9606         const struct mlx5_flow_driver_ops *fops =
9607                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
9608
9609         if (!fops->action_destroy) {
9610                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
9611                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
9612                                    NULL, err_msg);
9613                 return -rte_errno;
9614         }
9615         return fops->action_destroy(dev, handle, error);
9616 }
9617
9618 /* Wrapper for driver action_destroy op callback */
9619 static int
9620 flow_drv_action_update(struct rte_eth_dev *dev,
9621                        struct rte_flow_action_handle *handle,
9622                        const void *update,
9623                        const struct mlx5_flow_driver_ops *fops,
9624                        struct rte_flow_error *error)
9625 {
9626         static const char err_msg[] = "indirect action update unsupported";
9627
9628         if (!fops->action_update) {
9629                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
9630                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
9631                                    NULL, err_msg);
9632                 return -rte_errno;
9633         }
9634         return fops->action_update(dev, handle, update, error);
9635 }
9636
9637 /* Wrapper for driver action_destroy op callback */
9638 static int
9639 flow_drv_action_query(struct rte_eth_dev *dev,
9640                       const struct rte_flow_action_handle *handle,
9641                       void *data,
9642                       const struct mlx5_flow_driver_ops *fops,
9643                       struct rte_flow_error *error)
9644 {
9645         static const char err_msg[] = "indirect action query unsupported";
9646
9647         if (!fops->action_query) {
9648                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
9649                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
9650                                    NULL, err_msg);
9651                 return -rte_errno;
9652         }
9653         return fops->action_query(dev, handle, data, error);
9654 }
9655
9656 /**
9657  * Create indirect action for reuse in multiple flow rules.
9658  *
9659  * @param dev
9660  *   Pointer to Ethernet device structure.
9661  * @param conf
9662  *   Pointer to indirect action object configuration.
9663  * @param[in] action
9664  *   Action configuration for indirect action object creation.
9665  * @param[out] error
9666  *   Perform verbose error reporting if not NULL. PMDs initialize this
9667  *   structure in case of error only.
9668  * @return
9669  *   A valid handle in case of success, NULL otherwise and rte_errno is set.
9670  */
9671 static struct rte_flow_action_handle *
9672 mlx5_action_handle_create(struct rte_eth_dev *dev,
9673                           const struct rte_flow_indir_action_conf *conf,
9674                           const struct rte_flow_action *action,
9675                           struct rte_flow_error *error)
9676 {
9677         static const char err_msg[] = "indirect action creation unsupported";
9678         struct rte_flow_attr attr = { .transfer = 0 };
9679         const struct mlx5_flow_driver_ops *fops =
9680                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
9681
9682         if (flow_drv_action_validate(dev, conf, action, fops, error))
9683                 return NULL;
9684         if (!fops->action_create) {
9685                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
9686                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
9687                                    NULL, err_msg);
9688                 return NULL;
9689         }
9690         return fops->action_create(dev, conf, action, error);
9691 }
9692
9693 /**
9694  * Updates inplace the indirect action configuration pointed by *handle*
9695  * with the configuration provided as *update* argument.
9696  * The update of the indirect action configuration effects all flow rules
9697  * reusing the action via handle.
9698  *
9699  * @param dev
9700  *   Pointer to Ethernet device structure.
9701  * @param[in] handle
9702  *   Handle for the indirect action to be updated.
9703  * @param[in] update
9704  *   Action specification used to modify the action pointed by handle.
9705  *   *update* could be of same type with the action pointed by the *handle*
9706  *   handle argument, or some other structures like a wrapper, depending on
9707  *   the indirect action type.
9708  * @param[out] error
9709  *   Perform verbose error reporting if not NULL. PMDs initialize this
9710  *   structure in case of error only.
9711  *
9712  * @return
9713  *   0 on success, a negative errno value otherwise and rte_errno is set.
9714  */
9715 static int
9716 mlx5_action_handle_update(struct rte_eth_dev *dev,
9717                 struct rte_flow_action_handle *handle,
9718                 const void *update,
9719                 struct rte_flow_error *error)
9720 {
9721         struct rte_flow_attr attr = { .transfer = 0 };
9722         const struct mlx5_flow_driver_ops *fops =
9723                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
9724         int ret;
9725
9726         ret = flow_drv_action_validate(dev, NULL,
9727                         (const struct rte_flow_action *)update, fops, error);
9728         if (ret)
9729                 return ret;
9730         return flow_drv_action_update(dev, handle, update, fops,
9731                                       error);
9732 }
9733
9734 /**
9735  * Query the indirect action by handle.
9736  *
9737  * This function allows retrieving action-specific data such as counters.
9738  * Data is gathered by special action which may be present/referenced in
9739  * more than one flow rule definition.
9740  *
9741  * see @RTE_FLOW_ACTION_TYPE_COUNT
9742  *
9743  * @param dev
9744  *   Pointer to Ethernet device structure.
9745  * @param[in] handle
9746  *   Handle for the indirect action to query.
9747  * @param[in, out] data
9748  *   Pointer to storage for the associated query data type.
9749  * @param[out] error
9750  *   Perform verbose error reporting if not NULL. PMDs initialize this
9751  *   structure in case of error only.
9752  *
9753  * @return
9754  *   0 on success, a negative errno value otherwise and rte_errno is set.
9755  */
9756 static int
9757 mlx5_action_handle_query(struct rte_eth_dev *dev,
9758                          const struct rte_flow_action_handle *handle,
9759                          void *data,
9760                          struct rte_flow_error *error)
9761 {
9762         struct rte_flow_attr attr = { .transfer = 0 };
9763         const struct mlx5_flow_driver_ops *fops =
9764                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
9765
9766         return flow_drv_action_query(dev, handle, data, fops, error);
9767 }
9768
9769 /**
9770  * Destroy all indirect actions (shared RSS).
9771  *
9772  * @param dev
9773  *   Pointer to Ethernet device.
9774  *
9775  * @return
9776  *   0 on success, a negative errno value otherwise and rte_errno is set.
9777  */
9778 int
9779 mlx5_action_handle_flush(struct rte_eth_dev *dev)
9780 {
9781         struct rte_flow_error error;
9782         struct mlx5_priv *priv = dev->data->dev_private;
9783         struct mlx5_shared_action_rss *shared_rss;
9784         int ret = 0;
9785         uint32_t idx;
9786
9787         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
9788                       priv->rss_shared_actions, idx, shared_rss, next) {
9789                 ret |= mlx5_action_handle_destroy(dev,
9790                        (struct rte_flow_action_handle *)(uintptr_t)idx, &error);
9791         }
9792         return ret;
9793 }
9794
9795 /**
9796  * Validate existing indirect actions against current device configuration
9797  * and attach them to device resources.
9798  *
9799  * @param dev
9800  *   Pointer to Ethernet device.
9801  *
9802  * @return
9803  *   0 on success, a negative errno value otherwise and rte_errno is set.
9804  */
9805 int
9806 mlx5_action_handle_attach(struct rte_eth_dev *dev)
9807 {
9808         struct mlx5_priv *priv = dev->data->dev_private;
9809         int ret = 0;
9810         struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last;
9811
9812         LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
9813                 const char *message;
9814                 uint32_t queue_idx;
9815
9816                 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues,
9817                                                ind_tbl->queues_n,
9818                                                &message, &queue_idx);
9819                 if (ret != 0) {
9820                         DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s",
9821                                 dev->data->port_id, ind_tbl->queues[queue_idx],
9822                                 message);
9823                         break;
9824                 }
9825         }
9826         if (ret != 0)
9827                 return ret;
9828         LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
9829                 ret = mlx5_ind_table_obj_attach(dev, ind_tbl);
9830                 if (ret != 0) {
9831                         DRV_LOG(ERR, "Port %u could not attach "
9832                                 "indirection table obj %p",
9833                                 dev->data->port_id, (void *)ind_tbl);
9834                         goto error;
9835                 }
9836         }
9837
9838         return 0;
9839 error:
9840         ind_tbl_last = ind_tbl;
9841         LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
9842                 if (ind_tbl == ind_tbl_last)
9843                         break;
9844                 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0)
9845                         DRV_LOG(CRIT, "Port %u could not detach "
9846                                 "indirection table obj %p on rollback",
9847                                 dev->data->port_id, (void *)ind_tbl);
9848         }
9849         return ret;
9850 }
9851
9852 /**
9853  * Detach indirect actions of the device from its resources.
9854  *
9855  * @param dev
9856  *   Pointer to Ethernet device.
9857  *
9858  * @return
9859  *   0 on success, a negative errno value otherwise and rte_errno is set.
9860  */
9861 int
9862 mlx5_action_handle_detach(struct rte_eth_dev *dev)
9863 {
9864         struct mlx5_priv *priv = dev->data->dev_private;
9865         int ret = 0;
9866         struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last;
9867
9868         LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
9869                 ret = mlx5_ind_table_obj_detach(dev, ind_tbl);
9870                 if (ret != 0) {
9871                         DRV_LOG(ERR, "Port %u could not detach "
9872                                 "indirection table obj %p",
9873                                 dev->data->port_id, (void *)ind_tbl);
9874                         goto error;
9875                 }
9876         }
9877         return 0;
9878 error:
9879         ind_tbl_last = ind_tbl;
9880         LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
9881                 if (ind_tbl == ind_tbl_last)
9882                         break;
9883                 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0)
9884                         DRV_LOG(CRIT, "Port %u could not attach "
9885                                 "indirection table obj %p on rollback",
9886                                 dev->data->port_id, (void *)ind_tbl);
9887         }
9888         return ret;
9889 }
9890
9891 #ifndef HAVE_MLX5DV_DR
9892 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1))
9893 #else
9894 #define MLX5_DOMAIN_SYNC_FLOW \
9895         (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW)
9896 #endif
9897
9898 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
9899 {
9900         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
9901         const struct mlx5_flow_driver_ops *fops;
9902         int ret;
9903         struct rte_flow_attr attr = { .transfer = 0 };
9904
9905         fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr));
9906         ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW);
9907         if (ret > 0)
9908                 ret = -ret;
9909         return ret;
9910 }
9911
9912 const struct mlx5_flow_tunnel *
9913 mlx5_get_tof(const struct rte_flow_item *item,
9914              const struct rte_flow_action *action,
9915              enum mlx5_tof_rule_type *rule_type)
9916 {
9917         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
9918                 if (item->type == (typeof(item->type))
9919                                   MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) {
9920                         *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE;
9921                         return flow_items_to_tunnel(item);
9922                 }
9923         }
9924         for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) {
9925                 if (action->type == (typeof(action->type))
9926                                     MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) {
9927                         *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE;
9928                         return flow_actions_to_tunnel(action);
9929                 }
9930         }
9931         return NULL;
9932 }
9933
9934 /**
9935  * tunnel offload functionality is defined for DV environment only
9936  */
9937 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
9938 __extension__
9939 union tunnel_offload_mark {
9940         uint32_t val;
9941         struct {
9942                 uint32_t app_reserve:8;
9943                 uint32_t table_id:15;
9944                 uint32_t transfer:1;
9945                 uint32_t _unused_:8;
9946         };
9947 };
9948
9949 static bool
9950 mlx5_access_tunnel_offload_db
9951         (struct rte_eth_dev *dev,
9952          bool (*match)(struct rte_eth_dev *,
9953                        struct mlx5_flow_tunnel *, const void *),
9954          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
9955          void (*miss)(struct rte_eth_dev *, void *),
9956          void *ctx, bool lock_op);
9957
9958 static int
9959 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
9960                              struct rte_flow *flow,
9961                              const struct rte_flow_attr *attr,
9962                              const struct rte_flow_action *app_actions,
9963                              uint32_t flow_idx,
9964                              const struct mlx5_flow_tunnel *tunnel,
9965                              struct tunnel_default_miss_ctx *ctx,
9966                              struct rte_flow_error *error)
9967 {
9968         struct mlx5_priv *priv = dev->data->dev_private;
9969         struct mlx5_flow *dev_flow;
9970         struct rte_flow_attr miss_attr = *attr;
9971         const struct rte_flow_item miss_items[2] = {
9972                 {
9973                         .type = RTE_FLOW_ITEM_TYPE_ETH,
9974                         .spec = NULL,
9975                         .last = NULL,
9976                         .mask = NULL
9977                 },
9978                 {
9979                         .type = RTE_FLOW_ITEM_TYPE_END,
9980                         .spec = NULL,
9981                         .last = NULL,
9982                         .mask = NULL
9983                 }
9984         };
9985         union tunnel_offload_mark mark_id;
9986         struct rte_flow_action_mark miss_mark;
9987         struct rte_flow_action miss_actions[3] = {
9988                 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
9989                 [2] = { .type = RTE_FLOW_ACTION_TYPE_END,  .conf = NULL }
9990         };
9991         const struct rte_flow_action_jump *jump_data;
9992         uint32_t i, flow_table = 0; /* prevent compilation warning */
9993         struct flow_grp_info grp_info = {
9994                 .external = 1,
9995                 .transfer = attr->transfer,
9996                 .fdb_def_rule = !!priv->fdb_def_rule,
9997                 .std_tbl_fix = 0,
9998         };
9999         int ret;
10000
10001         if (!attr->transfer) {
10002                 uint32_t q_size;
10003
10004                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
10005                 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
10006                 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
10007                                          0, SOCKET_ID_ANY);
10008                 if (!ctx->queue)
10009                         return rte_flow_error_set
10010                                 (error, ENOMEM,
10011                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
10012                                 NULL, "invalid default miss RSS");
10013                 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
10014                 ctx->action_rss.level = 0,
10015                 ctx->action_rss.types = priv->rss_conf.rss_hf,
10016                 ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
10017                 ctx->action_rss.queue_num = priv->reta_idx_n,
10018                 ctx->action_rss.key = priv->rss_conf.rss_key,
10019                 ctx->action_rss.queue = ctx->queue;
10020                 if (!priv->reta_idx_n || !priv->rxqs_n)
10021                         return rte_flow_error_set
10022                                 (error, EINVAL,
10023                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
10024                                 NULL, "invalid port configuration");
10025                 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG))
10026                         ctx->action_rss.types = 0;
10027                 for (i = 0; i != priv->reta_idx_n; ++i)
10028                         ctx->queue[i] = (*priv->reta_idx)[i];
10029         } else {
10030                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
10031                 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
10032         }
10033         miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
10034         for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
10035         jump_data = app_actions->conf;
10036         miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
10037         miss_attr.group = jump_data->group;
10038         ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
10039                                        &flow_table, &grp_info, error);
10040         if (ret)
10041                 return rte_flow_error_set(error, EINVAL,
10042                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
10043                                           NULL, "invalid tunnel id");
10044         mark_id.app_reserve = 0;
10045         mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
10046         mark_id.transfer = !!attr->transfer;
10047         mark_id._unused_ = 0;
10048         miss_mark.id = mark_id.val;
10049         dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
10050                                     miss_items, miss_actions, flow_idx, error);
10051         if (!dev_flow)
10052                 return -rte_errno;
10053         dev_flow->flow = flow;
10054         dev_flow->external = true;
10055         dev_flow->tunnel = tunnel;
10056         dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE;
10057         /* Subflow object was created, we must include one in the list. */
10058         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
10059                       dev_flow->handle, next);
10060         DRV_LOG(DEBUG,
10061                 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
10062                 dev->data->port_id, tunnel->app_tunnel.type,
10063                 tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
10064         ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
10065                                   miss_actions, error);
10066         if (!ret)
10067                 ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
10068                                                   error);
10069
10070         return ret;
10071 }
10072
10073 static const struct mlx5_flow_tbl_data_entry  *
10074 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
10075 {
10076         struct mlx5_priv *priv = dev->data->dev_private;
10077         struct mlx5_dev_ctx_shared *sh = priv->sh;
10078         struct mlx5_list_entry *he;
10079         union tunnel_offload_mark mbits = { .val = mark };
10080         union mlx5_flow_tbl_key table_key = {
10081                 {
10082                         .level = tunnel_id_to_flow_tbl(mbits.table_id),
10083                         .id = 0,
10084                         .reserved = 0,
10085                         .dummy = 0,
10086                         .is_fdb = !!mbits.transfer,
10087                         .is_egress = 0,
10088                 }
10089         };
10090         struct mlx5_flow_cb_ctx ctx = {
10091                 .data = &table_key.v64,
10092         };
10093
10094         he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx);
10095         return he ?
10096                container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
10097 }
10098
10099 static void
10100 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx,
10101                                    struct mlx5_list_entry *entry)
10102 {
10103         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10104         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
10105
10106         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
10107                         tunnel_flow_tbl_to_id(tte->flow_table));
10108         mlx5_free(tte);
10109 }
10110
10111 static int
10112 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused,
10113                                   struct mlx5_list_entry *entry, void *cb_ctx)
10114 {
10115         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10116         union tunnel_tbl_key tbl = {
10117                 .val = *(uint64_t *)(ctx->data),
10118         };
10119         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
10120
10121         return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
10122 }
10123
10124 static struct mlx5_list_entry *
10125 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx)
10126 {
10127         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10128         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10129         struct tunnel_tbl_entry *tte;
10130         union tunnel_tbl_key tbl = {
10131                 .val = *(uint64_t *)(ctx->data),
10132         };
10133
10134         tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
10135                           sizeof(*tte), 0,
10136                           SOCKET_ID_ANY);
10137         if (!tte)
10138                 goto err;
10139         mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
10140                           &tte->flow_table);
10141         if (tte->flow_table >= MLX5_MAX_TABLES) {
10142                 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
10143                         tte->flow_table);
10144                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
10145                                 tte->flow_table);
10146                 goto err;
10147         } else if (!tte->flow_table) {
10148                 goto err;
10149         }
10150         tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
10151         tte->tunnel_id = tbl.tunnel_id;
10152         tte->group = tbl.group;
10153         return &tte->hash;
10154 err:
10155         if (tte)
10156                 mlx5_free(tte);
10157         return NULL;
10158 }
10159
10160 static struct mlx5_list_entry *
10161 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused,
10162                                   struct mlx5_list_entry *oentry,
10163                                   void *cb_ctx __rte_unused)
10164 {
10165         struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte),
10166                                                    0, SOCKET_ID_ANY);
10167
10168         if (!tte)
10169                 return NULL;
10170         memcpy(tte, oentry, sizeof(*tte));
10171         return &tte->hash;
10172 }
10173
10174 static void
10175 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused,
10176                                        struct mlx5_list_entry *entry)
10177 {
10178         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
10179
10180         mlx5_free(tte);
10181 }
10182
10183 static uint32_t
10184 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
10185                                 const struct mlx5_flow_tunnel *tunnel,
10186                                 uint32_t group, uint32_t *table,
10187                                 struct rte_flow_error *error)
10188 {
10189         struct mlx5_list_entry *he;
10190         struct tunnel_tbl_entry *tte;
10191         union tunnel_tbl_key key = {
10192                 .tunnel_id = tunnel ? tunnel->tunnel_id : 0,
10193                 .group = group
10194         };
10195         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
10196         struct mlx5_hlist *group_hash;
10197         struct mlx5_flow_cb_ctx ctx = {
10198                 .data = &key.val,
10199         };
10200
10201         group_hash = tunnel ? tunnel->groups : thub->groups;
10202         he = mlx5_hlist_register(group_hash, key.val, &ctx);
10203         if (!he)
10204                 return rte_flow_error_set(error, EINVAL,
10205                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
10206                                           NULL,
10207                                           "tunnel group index not supported");
10208         tte = container_of(he, typeof(*tte), hash);
10209         *table = tte->flow_table;
10210         DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
10211                 dev->data->port_id, key.tunnel_id, group, *table);
10212         return 0;
10213 }
10214
10215 static void
10216 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
10217                       struct mlx5_flow_tunnel *tunnel)
10218 {
10219         struct mlx5_priv *priv = dev->data->dev_private;
10220         struct mlx5_indexed_pool *ipool;
10221
10222         DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
10223                 dev->data->port_id, tunnel->tunnel_id);
10224         LIST_REMOVE(tunnel, chain);
10225         mlx5_hlist_destroy(tunnel->groups);
10226         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
10227         mlx5_ipool_free(ipool, tunnel->tunnel_id);
10228 }
10229
10230 static bool
10231 mlx5_access_tunnel_offload_db
10232         (struct rte_eth_dev *dev,
10233          bool (*match)(struct rte_eth_dev *,
10234                        struct mlx5_flow_tunnel *, const void *),
10235          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
10236          void (*miss)(struct rte_eth_dev *, void *),
10237          void *ctx, bool lock_op)
10238 {
10239         bool verdict = false;
10240         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
10241         struct mlx5_flow_tunnel *tunnel;
10242
10243         rte_spinlock_lock(&thub->sl);
10244         LIST_FOREACH(tunnel, &thub->tunnels, chain) {
10245                 verdict = match(dev, tunnel, (const void *)ctx);
10246                 if (verdict)
10247                         break;
10248         }
10249         if (!lock_op)
10250                 rte_spinlock_unlock(&thub->sl);
10251         if (verdict && hit)
10252                 hit(dev, tunnel, ctx);
10253         if (!verdict && miss)
10254                 miss(dev, ctx);
10255         if (lock_op)
10256                 rte_spinlock_unlock(&thub->sl);
10257
10258         return verdict;
10259 }
10260
10261 struct tunnel_db_find_tunnel_id_ctx {
10262         uint32_t tunnel_id;
10263         struct mlx5_flow_tunnel *tunnel;
10264 };
10265
10266 static bool
10267 find_tunnel_id_match(struct rte_eth_dev *dev,
10268                      struct mlx5_flow_tunnel *tunnel, const void *x)
10269 {
10270         const struct tunnel_db_find_tunnel_id_ctx *ctx = x;
10271
10272         RTE_SET_USED(dev);
10273         return tunnel->tunnel_id == ctx->tunnel_id;
10274 }
10275
10276 static void
10277 find_tunnel_id_hit(struct rte_eth_dev *dev,
10278                    struct mlx5_flow_tunnel *tunnel, void *x)
10279 {
10280         struct tunnel_db_find_tunnel_id_ctx *ctx = x;
10281         RTE_SET_USED(dev);
10282         ctx->tunnel = tunnel;
10283 }
10284
10285 static struct mlx5_flow_tunnel *
10286 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
10287 {
10288         struct tunnel_db_find_tunnel_id_ctx ctx = {
10289                 .tunnel_id = id,
10290         };
10291
10292         mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match,
10293                                       find_tunnel_id_hit, NULL, &ctx, true);
10294
10295         return ctx.tunnel;
10296 }
10297
10298 static struct mlx5_flow_tunnel *
10299 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
10300                           const struct rte_flow_tunnel *app_tunnel)
10301 {
10302         struct mlx5_priv *priv = dev->data->dev_private;
10303         struct mlx5_indexed_pool *ipool;
10304         struct mlx5_flow_tunnel *tunnel;
10305         uint32_t id;
10306
10307         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
10308         tunnel = mlx5_ipool_zmalloc(ipool, &id);
10309         if (!tunnel)
10310                 return NULL;
10311         if (id >= MLX5_MAX_TUNNELS) {
10312                 mlx5_ipool_free(ipool, id);
10313                 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
10314                 return NULL;
10315         }
10316         tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true,
10317                                            priv->sh,
10318                                            mlx5_flow_tunnel_grp2tbl_create_cb,
10319                                            mlx5_flow_tunnel_grp2tbl_match_cb,
10320                                            mlx5_flow_tunnel_grp2tbl_remove_cb,
10321                                            mlx5_flow_tunnel_grp2tbl_clone_cb,
10322                                         mlx5_flow_tunnel_grp2tbl_clone_free_cb);
10323         if (!tunnel->groups) {
10324                 mlx5_ipool_free(ipool, id);
10325                 return NULL;
10326         }
10327         /* initiate new PMD tunnel */
10328         memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
10329         tunnel->tunnel_id = id;
10330         tunnel->action.type = (typeof(tunnel->action.type))
10331                               MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
10332         tunnel->action.conf = tunnel;
10333         tunnel->item.type = (typeof(tunnel->item.type))
10334                             MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
10335         tunnel->item.spec = tunnel;
10336         tunnel->item.last = NULL;
10337         tunnel->item.mask = NULL;
10338
10339         DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
10340                 dev->data->port_id, tunnel->tunnel_id);
10341
10342         return tunnel;
10343 }
10344
10345 struct tunnel_db_get_tunnel_ctx {
10346         const struct rte_flow_tunnel *app_tunnel;
10347         struct mlx5_flow_tunnel *tunnel;
10348 };
10349
10350 static bool get_tunnel_match(struct rte_eth_dev *dev,
10351                              struct mlx5_flow_tunnel *tunnel, const void *x)
10352 {
10353         const struct tunnel_db_get_tunnel_ctx *ctx = x;
10354
10355         RTE_SET_USED(dev);
10356         return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel,
10357                        sizeof(*ctx->app_tunnel));
10358 }
10359
10360 static void get_tunnel_hit(struct rte_eth_dev *dev,
10361                            struct mlx5_flow_tunnel *tunnel, void *x)
10362 {
10363         /* called under tunnel spinlock protection */
10364         struct tunnel_db_get_tunnel_ctx *ctx = x;
10365
10366         RTE_SET_USED(dev);
10367         tunnel->refctn++;
10368         ctx->tunnel = tunnel;
10369 }
10370
10371 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x)
10372 {
10373         /* called under tunnel spinlock protection */
10374         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
10375         struct tunnel_db_get_tunnel_ctx *ctx = x;
10376
10377         rte_spinlock_unlock(&thub->sl);
10378         ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel);
10379         rte_spinlock_lock(&thub->sl);
10380         if (ctx->tunnel) {
10381                 ctx->tunnel->refctn = 1;
10382                 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain);
10383         }
10384 }
10385
10386
10387 static int
10388 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
10389                      const struct rte_flow_tunnel *app_tunnel,
10390                      struct mlx5_flow_tunnel **tunnel)
10391 {
10392         struct tunnel_db_get_tunnel_ctx ctx = {
10393                 .app_tunnel = app_tunnel,
10394         };
10395
10396         mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit,
10397                                       get_tunnel_miss, &ctx, true);
10398         *tunnel = ctx.tunnel;
10399         return ctx.tunnel ? 0 : -ENOMEM;
10400 }
10401
10402 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
10403 {
10404         struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
10405
10406         if (!thub)
10407                 return;
10408         if (!LIST_EMPTY(&thub->tunnels))
10409                 DRV_LOG(WARNING, "port %u tunnels present", port_id);
10410         mlx5_hlist_destroy(thub->groups);
10411         mlx5_free(thub);
10412 }
10413
10414 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
10415 {
10416         int err;
10417         struct mlx5_flow_tunnel_hub *thub;
10418
10419         thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
10420                            0, SOCKET_ID_ANY);
10421         if (!thub)
10422                 return -ENOMEM;
10423         LIST_INIT(&thub->tunnels);
10424         rte_spinlock_init(&thub->sl);
10425         thub->groups = mlx5_hlist_create("flow groups", 64,
10426                                          false, true, sh,
10427                                          mlx5_flow_tunnel_grp2tbl_create_cb,
10428                                          mlx5_flow_tunnel_grp2tbl_match_cb,
10429                                          mlx5_flow_tunnel_grp2tbl_remove_cb,
10430                                          mlx5_flow_tunnel_grp2tbl_clone_cb,
10431                                         mlx5_flow_tunnel_grp2tbl_clone_free_cb);
10432         if (!thub->groups) {
10433                 err = -rte_errno;
10434                 goto err;
10435         }
10436         sh->tunnel_hub = thub;
10437
10438         return 0;
10439
10440 err:
10441         if (thub->groups)
10442                 mlx5_hlist_destroy(thub->groups);
10443         if (thub)
10444                 mlx5_free(thub);
10445         return err;
10446 }
10447
10448 static inline int
10449 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
10450                           struct rte_flow_tunnel *tunnel,
10451                           struct rte_flow_error *error)
10452 {
10453         struct mlx5_priv *priv = dev->data->dev_private;
10454
10455         if (!priv->sh->config.dv_flow_en)
10456                 return rte_flow_error_set(error, ENOTSUP,
10457                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
10458                                           "flow DV interface is off");
10459         if (!is_tunnel_offload_active(dev))
10460                 return rte_flow_error_set(error, ENOTSUP,
10461                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
10462                                           "tunnel offload was not activated");
10463         if (!tunnel)
10464                 return rte_flow_error_set(error, EINVAL,
10465                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
10466                                           "no application tunnel");
10467         switch (tunnel->type) {
10468         default:
10469                 return rte_flow_error_set(error, EINVAL,
10470                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
10471                                           "unsupported tunnel type");
10472         case RTE_FLOW_ITEM_TYPE_VXLAN:
10473         case RTE_FLOW_ITEM_TYPE_GRE:
10474         case RTE_FLOW_ITEM_TYPE_NVGRE:
10475         case RTE_FLOW_ITEM_TYPE_GENEVE:
10476                 break;
10477         }
10478         return 0;
10479 }
10480
10481 static int
10482 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
10483                     struct rte_flow_tunnel *app_tunnel,
10484                     struct rte_flow_action **actions,
10485                     uint32_t *num_of_actions,
10486                     struct rte_flow_error *error)
10487 {
10488         struct mlx5_flow_tunnel *tunnel;
10489         int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error);
10490
10491         if (ret)
10492                 return ret;
10493         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
10494         if (ret < 0) {
10495                 return rte_flow_error_set(error, ret,
10496                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
10497                                           "failed to initialize pmd tunnel");
10498         }
10499         *actions = &tunnel->action;
10500         *num_of_actions = 1;
10501         return 0;
10502 }
10503
10504 static int
10505 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
10506                        struct rte_flow_tunnel *app_tunnel,
10507                        struct rte_flow_item **items,
10508                        uint32_t *num_of_items,
10509                        struct rte_flow_error *error)
10510 {
10511         struct mlx5_flow_tunnel *tunnel;
10512         int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error);
10513
10514         if (ret)
10515                 return ret;
10516         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
10517         if (ret < 0) {
10518                 return rte_flow_error_set(error, ret,
10519                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
10520                                           "failed to initialize pmd tunnel");
10521         }
10522         *items = &tunnel->item;
10523         *num_of_items = 1;
10524         return 0;
10525 }
10526
10527 struct tunnel_db_element_release_ctx {
10528         struct rte_flow_item *items;
10529         struct rte_flow_action *actions;
10530         uint32_t num_elements;
10531         struct rte_flow_error *error;
10532         int ret;
10533 };
10534
10535 static bool
10536 tunnel_element_release_match(struct rte_eth_dev *dev,
10537                              struct mlx5_flow_tunnel *tunnel, const void *x)
10538 {
10539         const struct tunnel_db_element_release_ctx *ctx = x;
10540
10541         RTE_SET_USED(dev);
10542         if (ctx->num_elements != 1)
10543                 return false;
10544         else if (ctx->items)
10545                 return ctx->items == &tunnel->item;
10546         else if (ctx->actions)
10547                 return ctx->actions == &tunnel->action;
10548
10549         return false;
10550 }
10551
10552 static void
10553 tunnel_element_release_hit(struct rte_eth_dev *dev,
10554                            struct mlx5_flow_tunnel *tunnel, void *x)
10555 {
10556         struct tunnel_db_element_release_ctx *ctx = x;
10557         ctx->ret = 0;
10558         if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
10559                 mlx5_flow_tunnel_free(dev, tunnel);
10560 }
10561
10562 static void
10563 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x)
10564 {
10565         struct tunnel_db_element_release_ctx *ctx = x;
10566         RTE_SET_USED(dev);
10567         ctx->ret = rte_flow_error_set(ctx->error, EINVAL,
10568                                       RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
10569                                       "invalid argument");
10570 }
10571
10572 static int
10573 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
10574                        struct rte_flow_item *pmd_items,
10575                        uint32_t num_items, struct rte_flow_error *err)
10576 {
10577         struct tunnel_db_element_release_ctx ctx = {
10578                 .items = pmd_items,
10579                 .actions = NULL,
10580                 .num_elements = num_items,
10581                 .error = err,
10582         };
10583
10584         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
10585                                       tunnel_element_release_hit,
10586                                       tunnel_element_release_miss, &ctx, false);
10587
10588         return ctx.ret;
10589 }
10590
10591 static int
10592 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
10593                          struct rte_flow_action *pmd_actions,
10594                          uint32_t num_actions, struct rte_flow_error *err)
10595 {
10596         struct tunnel_db_element_release_ctx ctx = {
10597                 .items = NULL,
10598                 .actions = pmd_actions,
10599                 .num_elements = num_actions,
10600                 .error = err,
10601         };
10602
10603         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
10604                                       tunnel_element_release_hit,
10605                                       tunnel_element_release_miss, &ctx, false);
10606
10607         return ctx.ret;
10608 }
10609
10610 static int
10611 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
10612                                   struct rte_mbuf *m,
10613                                   struct rte_flow_restore_info *info,
10614                                   struct rte_flow_error *err)
10615 {
10616         uint64_t ol_flags = m->ol_flags;
10617         const struct mlx5_flow_tbl_data_entry *tble;
10618         const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID;
10619
10620         if (!is_tunnel_offload_active(dev)) {
10621                 info->flags = 0;
10622                 return 0;
10623         }
10624
10625         if ((ol_flags & mask) != mask)
10626                 goto err;
10627         tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
10628         if (!tble) {
10629                 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
10630                         dev->data->port_id, m->hash.fdir.hi);
10631                 goto err;
10632         }
10633         MLX5_ASSERT(tble->tunnel);
10634         memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
10635         info->group_id = tble->group_id;
10636         info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
10637                       RTE_FLOW_RESTORE_INFO_GROUP_ID |
10638                       RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
10639
10640         return 0;
10641
10642 err:
10643         return rte_flow_error_set(err, EINVAL,
10644                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10645                                   "failed to get restore info");
10646 }
10647
10648 #else /* HAVE_IBV_FLOW_DV_SUPPORT */
10649 static int
10650 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev,
10651                            __rte_unused struct rte_flow_tunnel *app_tunnel,
10652                            __rte_unused struct rte_flow_action **actions,
10653                            __rte_unused uint32_t *num_of_actions,
10654                            __rte_unused struct rte_flow_error *error)
10655 {
10656         return -ENOTSUP;
10657 }
10658
10659 static int
10660 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev,
10661                        __rte_unused struct rte_flow_tunnel *app_tunnel,
10662                        __rte_unused struct rte_flow_item **items,
10663                        __rte_unused uint32_t *num_of_items,
10664                        __rte_unused struct rte_flow_error *error)
10665 {
10666         return -ENOTSUP;
10667 }
10668
10669 static int
10670 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev,
10671                               __rte_unused struct rte_flow_item *pmd_items,
10672                               __rte_unused uint32_t num_items,
10673                               __rte_unused struct rte_flow_error *err)
10674 {
10675         return -ENOTSUP;
10676 }
10677
10678 static int
10679 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev,
10680                                 __rte_unused struct rte_flow_action *pmd_action,
10681                                 __rte_unused uint32_t num_actions,
10682                                 __rte_unused struct rte_flow_error *err)
10683 {
10684         return -ENOTSUP;
10685 }
10686
10687 static int
10688 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev,
10689                                   __rte_unused struct rte_mbuf *m,
10690                                   __rte_unused struct rte_flow_restore_info *i,
10691                                   __rte_unused struct rte_flow_error *err)
10692 {
10693         return -ENOTSUP;
10694 }
10695
10696 static int
10697 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev,
10698                              __rte_unused struct rte_flow *flow,
10699                              __rte_unused const struct rte_flow_attr *attr,
10700                              __rte_unused const struct rte_flow_action *actions,
10701                              __rte_unused uint32_t flow_idx,
10702                              __rte_unused const struct mlx5_flow_tunnel *tunnel,
10703                              __rte_unused struct tunnel_default_miss_ctx *ctx,
10704                              __rte_unused struct rte_flow_error *error)
10705 {
10706         return -ENOTSUP;
10707 }
10708
10709 static struct mlx5_flow_tunnel *
10710 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev,
10711                     __rte_unused uint32_t id)
10712 {
10713         return NULL;
10714 }
10715
10716 static void
10717 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev,
10718                       __rte_unused struct mlx5_flow_tunnel *tunnel)
10719 {
10720 }
10721
10722 static uint32_t
10723 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev,
10724                                 __rte_unused const struct mlx5_flow_tunnel *t,
10725                                 __rte_unused uint32_t group,
10726                                 __rte_unused uint32_t *table,
10727                                 struct rte_flow_error *error)
10728 {
10729         return rte_flow_error_set(error, ENOTSUP,
10730                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10731                                   "tunnel offload requires DV support");
10732 }
10733
10734 void
10735 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh,
10736                         __rte_unused  uint16_t port_id)
10737 {
10738 }
10739 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
10740
10741 /* Flex flow item API */
10742 static struct rte_flow_item_flex_handle *
10743 mlx5_flow_flex_item_create(struct rte_eth_dev *dev,
10744                            const struct rte_flow_item_flex_conf *conf,
10745                            struct rte_flow_error *error)
10746 {
10747         static const char err_msg[] = "flex item creation unsupported";
10748         struct mlx5_priv *priv = dev->data->dev_private;
10749         struct rte_flow_attr attr = { .transfer = 0 };
10750         const struct mlx5_flow_driver_ops *fops =
10751                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
10752
10753         if (!priv->pci_dev) {
10754                 rte_flow_error_set(error, ENOTSUP,
10755                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10756                                    "create flex item on PF only");
10757                 return NULL;
10758         }
10759         switch (priv->pci_dev->id.device_id) {
10760         case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF:
10761         case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF:
10762                 break;
10763         default:
10764                 rte_flow_error_set(error, ENOTSUP,
10765                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10766                                    "flex item available on BlueField ports only");
10767                 return NULL;
10768         }
10769         if (!fops->item_create) {
10770                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
10771                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
10772                                    NULL, err_msg);
10773                 return NULL;
10774         }
10775         return fops->item_create(dev, conf, error);
10776 }
10777
10778 static int
10779 mlx5_flow_flex_item_release(struct rte_eth_dev *dev,
10780                             const struct rte_flow_item_flex_handle *handle,
10781                             struct rte_flow_error *error)
10782 {
10783         static const char err_msg[] = "flex item release unsupported";
10784         struct rte_flow_attr attr = { .transfer = 0 };
10785         const struct mlx5_flow_driver_ops *fops =
10786                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
10787
10788         if (!fops->item_release) {
10789                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
10790                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
10791                                    NULL, err_msg);
10792                 return -rte_errno;
10793         }
10794         return fops->item_release(dev, handle, error);
10795 }
10796
10797 static void
10798 mlx5_dbg__print_pattern(const struct rte_flow_item *item)
10799 {
10800         int ret;
10801         struct rte_flow_error error;
10802
10803         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
10804                 char *item_name;
10805                 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name,
10806                                     sizeof(item_name),
10807                                     (void *)(uintptr_t)item->type, &error);
10808                 if (ret > 0)
10809                         printf("%s ", item_name);
10810                 else
10811                         printf("%d\n", (int)item->type);
10812         }
10813         printf("END\n");
10814 }
10815
10816 static int
10817 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item)
10818 {
10819         const struct rte_flow_item_udp *spec = udp_item->spec;
10820         const struct rte_flow_item_udp *mask = udp_item->mask;
10821         uint16_t udp_dport = 0;
10822
10823         if (spec != NULL) {
10824                 if (!mask)
10825                         mask = &rte_flow_item_udp_mask;
10826                 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port &
10827                                 mask->hdr.dst_port);
10828         }
10829         return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN);
10830 }
10831
10832 static const struct mlx5_flow_expand_node *
10833 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern,
10834                 unsigned int item_idx,
10835                 const struct mlx5_flow_expand_node graph[],
10836                 const struct mlx5_flow_expand_node *node)
10837 {
10838         const struct rte_flow_item *item = pattern + item_idx, *prev_item;
10839
10840         if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN &&
10841                         node != NULL &&
10842                         node->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
10843                 /*
10844                  * The expansion node is VXLAN and it is also the last
10845                  * expandable item in the pattern, so need to continue
10846                  * expansion of the inner tunnel.
10847                  */
10848                 MLX5_ASSERT(item_idx > 0);
10849                 prev_item = pattern + item_idx - 1;
10850                 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP);
10851                 if (mlx5_flow_is_std_vxlan_port(prev_item))
10852                         return &graph[MLX5_EXPANSION_STD_VXLAN];
10853                 return &graph[MLX5_EXPANSION_L3_VXLAN];
10854         }
10855         return node;
10856 }
10857
10858 /* Map of Verbs to Flow priority with 8 Verbs priorities. */
10859 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
10860         { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
10861 };
10862
10863 /* Map of Verbs to Flow priority with 16 Verbs priorities. */
10864 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
10865         { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
10866         { 9, 10, 11 }, { 12, 13, 14 },
10867 };
10868
10869 /**
10870  * Discover the number of available flow priorities.
10871  *
10872  * @param dev
10873  *   Ethernet device.
10874  *
10875  * @return
10876  *   On success, number of available flow priorities.
10877  *   On failure, a negative errno-style code and rte_errno is set.
10878  */
10879 int
10880 mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
10881 {
10882         static const uint16_t vprio[] = {8, 16};
10883         const struct mlx5_priv *priv = dev->data->dev_private;
10884         const struct mlx5_flow_driver_ops *fops;
10885         enum mlx5_flow_drv_type type;
10886         int ret;
10887
10888         type = mlx5_flow_os_get_type();
10889         if (type == MLX5_FLOW_TYPE_MAX) {
10890                 type = MLX5_FLOW_TYPE_VERBS;
10891                 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en)
10892                         type = MLX5_FLOW_TYPE_DV;
10893         }
10894         fops = flow_get_drv_ops(type);
10895         if (fops->discover_priorities == NULL) {
10896                 DRV_LOG(ERR, "Priority discovery not supported");
10897                 rte_errno = ENOTSUP;
10898                 return -rte_errno;
10899         }
10900         ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio));
10901         if (ret < 0)
10902                 return ret;
10903         switch (ret) {
10904         case 8:
10905                 ret = RTE_DIM(priority_map_3);
10906                 break;
10907         case 16:
10908                 ret = RTE_DIM(priority_map_5);
10909                 break;
10910         default:
10911                 rte_errno = ENOTSUP;
10912                 DRV_LOG(ERR,
10913                         "port %u maximum priority: %d expected 8/16",
10914                         dev->data->port_id, ret);
10915                 return -rte_errno;
10916         }
10917         DRV_LOG(INFO, "port %u supported flow priorities:"
10918                 " 0-%d for ingress or egress root table,"
10919                 " 0-%d for non-root table or transfer root table.",
10920                 dev->data->port_id, ret - 2,
10921                 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1);
10922         return ret;
10923 }
10924
10925 /**
10926  * Adjust flow priority based on the highest layer and the request priority.
10927  *
10928  * @param[in] dev
10929  *   Pointer to the Ethernet device structure.
10930  * @param[in] priority
10931  *   The rule base priority.
10932  * @param[in] subpriority
10933  *   The priority based on the items.
10934  *
10935  * @return
10936  *   The new priority.
10937  */
10938 uint32_t
10939 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
10940                           uint32_t subpriority)
10941 {
10942         uint32_t res = 0;
10943         struct mlx5_priv *priv = dev->data->dev_private;
10944
10945         switch (priv->sh->flow_max_priority) {
10946         case RTE_DIM(priority_map_3):
10947                 res = priority_map_3[priority][subpriority];
10948                 break;
10949         case RTE_DIM(priority_map_5):
10950                 res = priority_map_5[priority][subpriority];
10951                 break;
10952         }
10953         return  res;
10954 }