drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <stdbool.h>
  10 #include <sys/queue.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_ether.h>
  14 #include <ethdev_driver.h>
  15 #include <rte_eal_paging.h>
  16 #include <rte_flow.h>
  17 #include <rte_cycles.h>
  18 #include <rte_flow_driver.h>
  19 #include <rte_malloc.h>
  20 #include <rte_ip.h>
  21
  22 #include <mlx5_glue.h>
  23 #include <mlx5_devx_cmds.h>
  24 #include <mlx5_prm.h>
  25 #include <mlx5_malloc.h>
  26
  27 #include "mlx5_defs.h"
  28 #include "mlx5.h"
  29 #include "mlx5_flow.h"
  30 #include "mlx5_flow_os.h"
  31 #include "mlx5_rx.h"
  32 #include "mlx5_tx.h"
  33 #include "mlx5_common_os.h"
  34 #include "rte_pmd_mlx5.h"
  35
  36 struct tunnel_default_miss_ctx {
  37         uint16_t *queue;
  38         __extension__
  39         union {
  40                 struct rte_flow_action_rss action_rss;
  41                 struct rte_flow_action_queue miss_queue;
  42                 struct rte_flow_action_jump miss_jump;
  43                 uint8_t raw[0];
  44         };
  45 };
  46
  47 static int
  48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
  49                              struct rte_flow *flow,
  50                              const struct rte_flow_attr *attr,
  51                              const struct rte_flow_action *app_actions,
  52                              uint32_t flow_idx,
  53                              const struct mlx5_flow_tunnel *tunnel,
  54                              struct tunnel_default_miss_ctx *ctx,
  55                              struct rte_flow_error *error);
  56 static struct mlx5_flow_tunnel *
  57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
  58 static void
  59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
  60 static uint32_t
  61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
  62                                 const struct mlx5_flow_tunnel *tunnel,
  63                                 uint32_t group, uint32_t *table,
  64                                 struct rte_flow_error *error);
  65
  66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void);
  67 static void mlx5_flow_pop_thread_workspace(void);
  68
  69
  70 /** Device flow drivers. */
  71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  72
  73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  74
  75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  76         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  78         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  79         [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops,
  80 #endif
  81         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  82         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  83 };
  84
  85 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
  86 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
  87         (const int []){ \
  88                 __VA_ARGS__, 0, \
  89         }
  90
  91 /** Node object of input graph for mlx5_flow_expand_rss(). */
  92 struct mlx5_flow_expand_node {
  93         const int *const next;
  94         /**<
  95          * List of next node indexes. Index 0 is interpreted as a terminator.
  96          */
  97         const enum rte_flow_item_type type;
  98         /**< Pattern item type of current node. */
  99         uint64_t rss_types;
 100         /**<
 101          * RSS types bit-field associated with this node
 102          * (see RTE_ETH_RSS_* definitions).
 103          */
 104         uint64_t node_flags;
 105         /**<
 106          *  Bit-fields that define how the node is used in the expansion.
 107          * (see MLX5_EXPANSION_NODE_* definitions).
 108          */
 109 };
 110
 111 /* Optional expand field. The expansion alg will not go deeper. */
 112 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0)
 113
 114 /* The node is not added implicitly as expansion to the flow pattern.
 115  * If the node type does not match the flow pattern item type, the
 116  * expansion alg will go deeper to its next items.
 117  * In the current implementation, the list of next nodes indexes can
 118  * have up to one node with this flag set and it has to be the last
 119  * node index (before the list terminator).
 120  */
 121 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1)
 122
 123 /** Object returned by mlx5_flow_expand_rss(). */
 124 struct mlx5_flow_expand_rss {
 125         uint32_t entries;
 126         /**< Number of entries @p patterns and @p priorities. */
 127         struct {
 128                 struct rte_flow_item *pattern; /**< Expanded pattern array. */
 129                 uint32_t priority; /**< Priority offset for each expansion. */
 130         } entry[];
 131 };
 132
 133 static void
 134 mlx5_dbg__print_pattern(const struct rte_flow_item *item);
 135
 136 static const struct mlx5_flow_expand_node *
 137 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern,
 138                 unsigned int item_idx,
 139                 const struct mlx5_flow_expand_node graph[],
 140                 const struct mlx5_flow_expand_node *node);
 141
 142 static bool
 143 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item)
 144 {
 145         switch (item->type) {
 146         case RTE_FLOW_ITEM_TYPE_ETH:
 147         case RTE_FLOW_ITEM_TYPE_VLAN:
 148         case RTE_FLOW_ITEM_TYPE_IPV4:
 149         case RTE_FLOW_ITEM_TYPE_IPV6:
 150         case RTE_FLOW_ITEM_TYPE_UDP:
 151         case RTE_FLOW_ITEM_TYPE_TCP:
 152         case RTE_FLOW_ITEM_TYPE_ESP:
 153         case RTE_FLOW_ITEM_TYPE_VXLAN:
 154         case RTE_FLOW_ITEM_TYPE_NVGRE:
 155         case RTE_FLOW_ITEM_TYPE_GRE:
 156         case RTE_FLOW_ITEM_TYPE_GENEVE:
 157         case RTE_FLOW_ITEM_TYPE_MPLS:
 158         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 159         case RTE_FLOW_ITEM_TYPE_GRE_KEY:
 160         case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
 161         case RTE_FLOW_ITEM_TYPE_GTP:
 162                 return true;
 163         default:
 164                 break;
 165         }
 166         return false;
 167 }
 168
 169 /**
 170  * Network Service Header (NSH) and its next protocol values
 171  * are described in RFC-8393.
 172  */
 173 static enum rte_flow_item_type
 174 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask)
 175 {
 176         enum rte_flow_item_type type;
 177
 178         switch (proto_mask & proto_spec) {
 179         case 0:
 180                 type = RTE_FLOW_ITEM_TYPE_VOID;
 181                 break;
 182         case RTE_VXLAN_GPE_TYPE_IPV4:
 183                 type = RTE_FLOW_ITEM_TYPE_IPV4;
 184                 break;
 185         case RTE_VXLAN_GPE_TYPE_IPV6:
 186                 type = RTE_VXLAN_GPE_TYPE_IPV6;
 187                 break;
 188         case RTE_VXLAN_GPE_TYPE_ETH:
 189                 type = RTE_FLOW_ITEM_TYPE_ETH;
 190                 break;
 191         default:
 192                 type = RTE_FLOW_ITEM_TYPE_END;
 193         }
 194         return type;
 195 }
 196
 197 static enum rte_flow_item_type
 198 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask)
 199 {
 200         enum rte_flow_item_type type;
 201
 202         switch (proto_mask & proto_spec) {
 203         case 0:
 204                 type = RTE_FLOW_ITEM_TYPE_VOID;
 205                 break;
 206         case IPPROTO_UDP:
 207                 type = RTE_FLOW_ITEM_TYPE_UDP;
 208                 break;
 209         case IPPROTO_TCP:
 210                 type = RTE_FLOW_ITEM_TYPE_TCP;
 211                 break;
 212         case IPPROTO_IPIP:
 213                 type = RTE_FLOW_ITEM_TYPE_IPV4;
 214                 break;
 215         case IPPROTO_IPV6:
 216                 type = RTE_FLOW_ITEM_TYPE_IPV6;
 217                 break;
 218         case IPPROTO_ESP:
 219                 type = RTE_FLOW_ITEM_TYPE_ESP;
 220                 break;
 221         default:
 222                 type = RTE_FLOW_ITEM_TYPE_END;
 223         }
 224         return type;
 225 }
 226
 227 static enum rte_flow_item_type
 228 mlx5_ethertype_to_item_type(rte_be16_t type_spec,
 229                             rte_be16_t type_mask, bool is_tunnel)
 230 {
 231         enum rte_flow_item_type type;
 232
 233         switch (rte_be_to_cpu_16(type_spec & type_mask)) {
 234         case 0:
 235                 type = RTE_FLOW_ITEM_TYPE_VOID;
 236                 break;
 237         case RTE_ETHER_TYPE_TEB:
 238                 type = is_tunnel ?
 239                        RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END;
 240                 break;
 241         case RTE_ETHER_TYPE_VLAN:
 242                 type = !is_tunnel ?
 243                        RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END;
 244                 break;
 245         case RTE_ETHER_TYPE_IPV4:
 246                 type = RTE_FLOW_ITEM_TYPE_IPV4;
 247                 break;
 248         case RTE_ETHER_TYPE_IPV6:
 249                 type = RTE_FLOW_ITEM_TYPE_IPV6;
 250                 break;
 251         default:
 252                 type = RTE_FLOW_ITEM_TYPE_END;
 253         }
 254         return type;
 255 }
 256
 257 static enum rte_flow_item_type
 258 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
 259 {
 260 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld)                              \
 261         do {                                                             \
 262                 const void *m = item->mask;                              \
 263                 const void *s = item->spec;                              \
 264                 mask = m ?                                               \
 265                         ((const struct rte_flow_item_##type *)m)->fld :  \
 266                         rte_flow_item_##type##_mask.fld;                 \
 267                 spec = ((const struct rte_flow_item_##type *)s)->fld;    \
 268         } while (0)
 269
 270         enum rte_flow_item_type ret;
 271         uint16_t spec, mask;
 272
 273         if (item == NULL || item->spec == NULL)
 274                 return RTE_FLOW_ITEM_TYPE_VOID;
 275         switch (item->type) {
 276         case RTE_FLOW_ITEM_TYPE_ETH:
 277                 MLX5_XSET_ITEM_MASK_SPEC(eth, type);
 278                 if (!mask)
 279                         return RTE_FLOW_ITEM_TYPE_VOID;
 280                 ret = mlx5_ethertype_to_item_type(spec, mask, false);
 281                 break;
 282         case RTE_FLOW_ITEM_TYPE_VLAN:
 283                 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type);
 284                 if (!mask)
 285                         return RTE_FLOW_ITEM_TYPE_VOID;
 286                 ret = mlx5_ethertype_to_item_type(spec, mask, false);
 287                 break;
 288         case RTE_FLOW_ITEM_TYPE_IPV4:
 289                 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id);
 290                 if (!mask)
 291                         return RTE_FLOW_ITEM_TYPE_VOID;
 292                 ret = mlx5_inet_proto_to_item_type(spec, mask);
 293                 break;
 294         case RTE_FLOW_ITEM_TYPE_IPV6:
 295                 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto);
 296                 if (!mask)
 297                         return RTE_FLOW_ITEM_TYPE_VOID;
 298                 ret = mlx5_inet_proto_to_item_type(spec, mask);
 299                 break;
 300         case RTE_FLOW_ITEM_TYPE_GENEVE:
 301                 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol);
 302                 ret = mlx5_ethertype_to_item_type(spec, mask, true);
 303                 break;
 304         case RTE_FLOW_ITEM_TYPE_GRE:
 305                 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol);
 306                 ret = mlx5_ethertype_to_item_type(spec, mask, true);
 307                 break;
 308         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 309                 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol);
 310                 ret = mlx5_nsh_proto_to_item_type(spec, mask);
 311                 break;
 312         default:
 313                 ret = RTE_FLOW_ITEM_TYPE_VOID;
 314                 break;
 315         }
 316         return ret;
 317 #undef MLX5_XSET_ITEM_MASK_SPEC
 318 }
 319
 320 static const int *
 321 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[],
 322                 const int *next_node)
 323 {
 324         const struct mlx5_flow_expand_node *node = NULL;
 325         const int *next = next_node;
 326
 327         while (next && *next) {
 328                 /*
 329                  * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT
 330                  * flag set, because they were not found in the flow pattern.
 331                  */
 332                 node = &graph[*next];
 333                 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT))
 334                         break;
 335                 next = node->next;
 336         }
 337         return next;
 338 }
 339
 340 #define MLX5_RSS_EXP_ELT_N 16
 341
 342 /**
 343  * Expand RSS flows into several possible flows according to the RSS hash
 344  * fields requested and the driver capabilities.
 345  *
 346  * @param[out] buf
 347  *   Buffer to store the result expansion.
 348  * @param[in] size
 349  *   Buffer size in bytes. If 0, @p buf can be NULL.
 350  * @param[in] pattern
 351  *   User flow pattern.
 352  * @param[in] types
 353  *   RSS types to expand (see RTE_ETH_RSS_* definitions).
 354  * @param[in] graph
 355  *   Input graph to expand @p pattern according to @p types.
 356  * @param[in] graph_root_index
 357  *   Index of root node in @p graph, typically 0.
 358  *
 359  * @return
 360  *   A positive value representing the size of @p buf in bytes regardless of
 361  *   @p size on success, a negative errno value otherwise and rte_errno is
 362  *   set, the following errors are defined:
 363  *
 364  *   -E2BIG: graph-depth @p graph is too deep.
 365  *   -EINVAL: @p size has not enough space for expanded pattern.
 366  */
 367 static int
 368 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 369                      const struct rte_flow_item *pattern, uint64_t types,
 370                      const struct mlx5_flow_expand_node graph[],
 371                      int graph_root_index)
 372 {
 373         const struct rte_flow_item *item;
 374         const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
 375         const int *next_node;
 376         const int *stack[MLX5_RSS_EXP_ELT_N];
 377         int stack_pos = 0;
 378         struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N];
 379         unsigned int i, item_idx, last_expand_item_idx = 0;
 380         size_t lsize;
 381         size_t user_pattern_size = 0;
 382         void *addr = NULL;
 383         const struct mlx5_flow_expand_node *next = NULL;
 384         struct rte_flow_item missed_item;
 385         int missed = 0;
 386         int elt = 0;
 387         const struct rte_flow_item *last_expand_item = NULL;
 388
 389         memset(&missed_item, 0, sizeof(missed_item));
 390         lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
 391                 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]);
 392         if (lsize > size)
 393                 return -EINVAL;
 394         buf->entry[0].priority = 0;
 395         buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N];
 396         buf->entries = 0;
 397         addr = buf->entry[0].pattern;
 398         for (item = pattern, item_idx = 0;
 399                         item->type != RTE_FLOW_ITEM_TYPE_END;
 400                         item++, item_idx++) {
 401                 if (!mlx5_flow_is_rss_expandable_item(item)) {
 402                         user_pattern_size += sizeof(*item);
 403                         continue;
 404                 }
 405                 last_expand_item = item;
 406                 last_expand_item_idx = item_idx;
 407                 i = 0;
 408                 while (node->next && node->next[i]) {
 409                         next = &graph[node->next[i]];
 410                         if (next->type == item->type)
 411                                 break;
 412                         if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) {
 413                                 node = next;
 414                                 i = 0;
 415                         } else {
 416                                 ++i;
 417                         }
 418                 }
 419                 if (next)
 420                         node = next;
 421                 user_pattern_size += sizeof(*item);
 422         }
 423         user_pattern_size += sizeof(*item); /* Handle END item. */
 424         lsize += user_pattern_size;
 425         if (lsize > size)
 426                 return -EINVAL;
 427         /* Copy the user pattern in the first entry of the buffer. */
 428         rte_memcpy(addr, pattern, user_pattern_size);
 429         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 430         buf->entries = 1;
 431         /* Start expanding. */
 432         memset(flow_items, 0, sizeof(flow_items));
 433         user_pattern_size -= sizeof(*item);
 434         /*
 435          * Check if the last valid item has spec set, need complete pattern,
 436          * and the pattern can be used for expansion.
 437          */
 438         missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item);
 439         if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 440                 /* Item type END indicates expansion is not required. */
 441                 return lsize;
 442         }
 443         if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 444                 next = NULL;
 445                 missed = 1;
 446                 i = 0;
 447                 while (node->next && node->next[i]) {
 448                         next = &graph[node->next[i]];
 449                         if (next->type == missed_item.type) {
 450                                 flow_items[0].type = missed_item.type;
 451                                 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 452                                 break;
 453                         }
 454                         if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) {
 455                                 node = next;
 456                                 i = 0;
 457                         } else {
 458                                 ++i;
 459                         }
 460                         next = NULL;
 461                 }
 462         }
 463         if (next && missed) {
 464                 elt = 2; /* missed item + item end. */
 465                 node = next;
 466                 lsize += elt * sizeof(*item) + user_pattern_size;
 467                 if (lsize > size)
 468                         return -EINVAL;
 469                 if (node->rss_types & types) {
 470                         buf->entry[buf->entries].priority = 1;
 471                         buf->entry[buf->entries].pattern = addr;
 472                         buf->entries++;
 473                         rte_memcpy(addr, buf->entry[0].pattern,
 474                                    user_pattern_size);
 475                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 476                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 477                         addr = (void *)(((uintptr_t)addr) +
 478                                         elt * sizeof(*item));
 479                 }
 480         } else if (last_expand_item != NULL) {
 481                 node = mlx5_flow_expand_rss_adjust_node(pattern,
 482                                 last_expand_item_idx, graph, node);
 483         }
 484         memset(flow_items, 0, sizeof(flow_items));
 485         next_node = mlx5_flow_expand_rss_skip_explicit(graph,
 486                         node->next);
 487         stack[stack_pos] = next_node;
 488         node = next_node ? &graph[*next_node] : NULL;
 489         while (node) {
 490                 flow_items[stack_pos].type = node->type;
 491                 if (node->rss_types & types) {
 492                         size_t n;
 493                         /*
 494                          * compute the number of items to copy from the
 495                          * expansion and copy it.
 496                          * When the stack_pos is 0, there are 1 element in it,
 497                          * plus the addition END item.
 498                          */
 499                         elt = stack_pos + 2;
 500                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 501                         lsize += elt * sizeof(*item) + user_pattern_size;
 502                         if (lsize > size)
 503                                 return -EINVAL;
 504                         n = elt * sizeof(*item);
 505                         buf->entry[buf->entries].priority =
 506                                 stack_pos + 1 + missed;
 507                         buf->entry[buf->entries].pattern = addr;
 508                         buf->entries++;
 509                         rte_memcpy(addr, buf->entry[0].pattern,
 510                                    user_pattern_size);
 511                         addr = (void *)(((uintptr_t)addr) +
 512                                         user_pattern_size);
 513                         rte_memcpy(addr, &missed_item,
 514                                    missed * sizeof(*item));
 515                         addr = (void *)(((uintptr_t)addr) +
 516                                 missed * sizeof(*item));
 517                         rte_memcpy(addr, flow_items, n);
 518                         addr = (void *)(((uintptr_t)addr) + n);
 519                 }
 520                 /* Go deeper. */
 521                 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) &&
 522                                 node->next) {
 523                         next_node = mlx5_flow_expand_rss_skip_explicit(graph,
 524                                         node->next);
 525                         if (stack_pos++ == MLX5_RSS_EXP_ELT_N) {
 526                                 rte_errno = E2BIG;
 527                                 return -rte_errno;
 528                         }
 529                         stack[stack_pos] = next_node;
 530                 } else if (*(next_node + 1)) {
 531                         /* Follow up with the next possibility. */
 532                         next_node = mlx5_flow_expand_rss_skip_explicit(graph,
 533                                         ++next_node);
 534                 } else if (!stack_pos) {
 535                         /*
 536                          * Completing the traverse over the different paths.
 537                          * The next_node is advanced to the terminator.
 538                          */
 539                         ++next_node;
 540                 } else {
 541                         /* Move to the next path. */
 542                         while (stack_pos) {
 543                                 next_node = stack[--stack_pos];
 544                                 next_node++;
 545                                 if (*next_node)
 546                                         break;
 547                         }
 548                         next_node = mlx5_flow_expand_rss_skip_explicit(graph,
 549                                         next_node);
 550                         stack[stack_pos] = next_node;
 551                 }
 552                 node = next_node && *next_node ? &graph[*next_node] : NULL;
 553         };
 554         return lsize;
 555 }
 556
 557 enum mlx5_expansion {
 558         MLX5_EXPANSION_ROOT,
 559         MLX5_EXPANSION_ROOT_OUTER,
 560         MLX5_EXPANSION_OUTER_ETH,
 561         MLX5_EXPANSION_OUTER_VLAN,
 562         MLX5_EXPANSION_OUTER_IPV4,
 563         MLX5_EXPANSION_OUTER_IPV4_UDP,
 564         MLX5_EXPANSION_OUTER_IPV4_TCP,
 565         MLX5_EXPANSION_OUTER_IPV4_ESP,
 566         MLX5_EXPANSION_OUTER_IPV6,
 567         MLX5_EXPANSION_OUTER_IPV6_UDP,
 568         MLX5_EXPANSION_OUTER_IPV6_TCP,
 569         MLX5_EXPANSION_OUTER_IPV6_ESP,
 570         MLX5_EXPANSION_VXLAN,
 571         MLX5_EXPANSION_STD_VXLAN,
 572         MLX5_EXPANSION_L3_VXLAN,
 573         MLX5_EXPANSION_VXLAN_GPE,
 574         MLX5_EXPANSION_GRE,
 575         MLX5_EXPANSION_NVGRE,
 576         MLX5_EXPANSION_GRE_KEY,
 577         MLX5_EXPANSION_MPLS,
 578         MLX5_EXPANSION_ETH,
 579         MLX5_EXPANSION_VLAN,
 580         MLX5_EXPANSION_IPV4,
 581         MLX5_EXPANSION_IPV4_UDP,
 582         MLX5_EXPANSION_IPV4_TCP,
 583         MLX5_EXPANSION_IPV4_ESP,
 584         MLX5_EXPANSION_IPV6,
 585         MLX5_EXPANSION_IPV6_UDP,
 586         MLX5_EXPANSION_IPV6_TCP,
 587         MLX5_EXPANSION_IPV6_ESP,
 588         MLX5_EXPANSION_IPV6_FRAG_EXT,
 589         MLX5_EXPANSION_GTP,
 590         MLX5_EXPANSION_GENEVE,
 591 };
 592
 593 /** Supported expansion of items. */
 594 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 595         [MLX5_EXPANSION_ROOT] = {
 596                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 597                                                   MLX5_EXPANSION_IPV4,
 598                                                   MLX5_EXPANSION_IPV6),
 599                 .type = RTE_FLOW_ITEM_TYPE_END,
 600         },
 601         [MLX5_EXPANSION_ROOT_OUTER] = {
 602                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 603                                                   MLX5_EXPANSION_OUTER_IPV4,
 604                                                   MLX5_EXPANSION_OUTER_IPV6),
 605                 .type = RTE_FLOW_ITEM_TYPE_END,
 606         },
 607         [MLX5_EXPANSION_OUTER_ETH] = {
 608                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 609                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 610                 .rss_types = 0,
 611         },
 612         [MLX5_EXPANSION_OUTER_VLAN] = {
 613                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 614                                                   MLX5_EXPANSION_OUTER_IPV6),
 615                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 616                 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
 617         },
 618         [MLX5_EXPANSION_OUTER_IPV4] = {
 619                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 620                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 621                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 622                          MLX5_EXPANSION_OUTER_IPV4_ESP,
 623                          MLX5_EXPANSION_GRE,
 624                          MLX5_EXPANSION_NVGRE,
 625                          MLX5_EXPANSION_IPV4,
 626                          MLX5_EXPANSION_IPV6),
 627                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 628                 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 |
 629                         RTE_ETH_RSS_NONFRAG_IPV4_OTHER,
 630         },
 631         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 632                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 633                                                   MLX5_EXPANSION_VXLAN_GPE,
 634                                                   MLX5_EXPANSION_MPLS,
 635                                                   MLX5_EXPANSION_GENEVE,
 636                                                   MLX5_EXPANSION_GTP),
 637                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 638                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP,
 639         },
 640         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 641                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 642                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP,
 643         },
 644         [MLX5_EXPANSION_OUTER_IPV4_ESP] = {
 645                 .type = RTE_FLOW_ITEM_TYPE_ESP,
 646                 .rss_types = RTE_ETH_RSS_ESP,
 647         },
 648         [MLX5_EXPANSION_OUTER_IPV6] = {
 649                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 650                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 651                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 652                          MLX5_EXPANSION_OUTER_IPV6_ESP,
 653                          MLX5_EXPANSION_IPV4,
 654                          MLX5_EXPANSION_IPV6,
 655                          MLX5_EXPANSION_GRE,
 656                          MLX5_EXPANSION_NVGRE),
 657                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 658                 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 |
 659                         RTE_ETH_RSS_NONFRAG_IPV6_OTHER,
 660         },
 661         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 662                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 663                                                   MLX5_EXPANSION_VXLAN_GPE,
 664                                                   MLX5_EXPANSION_MPLS,
 665                                                   MLX5_EXPANSION_GENEVE,
 666                                                   MLX5_EXPANSION_GTP),
 667                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 668                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP,
 669         },
 670         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 671                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 672                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP,
 673         },
 674         [MLX5_EXPANSION_OUTER_IPV6_ESP] = {
 675                 .type = RTE_FLOW_ITEM_TYPE_ESP,
 676                 .rss_types = RTE_ETH_RSS_ESP,
 677         },
 678         [MLX5_EXPANSION_VXLAN] = {
 679                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 680                                                   MLX5_EXPANSION_IPV4,
 681                                                   MLX5_EXPANSION_IPV6),
 682                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 683         },
 684         [MLX5_EXPANSION_STD_VXLAN] = {
 685                         .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
 686                                         .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 687         },
 688         [MLX5_EXPANSION_L3_VXLAN] = {
 689                         .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 690                                         MLX5_EXPANSION_IPV6),
 691                                         .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 692         },
 693         [MLX5_EXPANSION_VXLAN_GPE] = {
 694                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 695                                                   MLX5_EXPANSION_IPV4,
 696                                                   MLX5_EXPANSION_IPV6),
 697                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 698         },
 699         [MLX5_EXPANSION_GRE] = {
 700                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 701                                                   MLX5_EXPANSION_IPV4,
 702                                                   MLX5_EXPANSION_IPV6,
 703                                                   MLX5_EXPANSION_GRE_KEY,
 704                                                   MLX5_EXPANSION_MPLS),
 705                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 706         },
 707         [MLX5_EXPANSION_GRE_KEY] = {
 708                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 709                                                   MLX5_EXPANSION_IPV6,
 710                                                   MLX5_EXPANSION_MPLS),
 711                 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY,
 712                 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
 713         },
 714         [MLX5_EXPANSION_NVGRE] = {
 715                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
 716                 .type = RTE_FLOW_ITEM_TYPE_NVGRE,
 717         },
 718         [MLX5_EXPANSION_MPLS] = {
 719                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 720                                                   MLX5_EXPANSION_IPV6,
 721                                                   MLX5_EXPANSION_ETH),
 722                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 723                 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
 724         },
 725         [MLX5_EXPANSION_ETH] = {
 726                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 727                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 728         },
 729         [MLX5_EXPANSION_VLAN] = {
 730                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 731                                                   MLX5_EXPANSION_IPV6),
 732                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 733                 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
 734         },
 735         [MLX5_EXPANSION_IPV4] = {
 736                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 737                                                   MLX5_EXPANSION_IPV4_TCP,
 738                                                   MLX5_EXPANSION_IPV4_ESP),
 739                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 740                 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 |
 741                         RTE_ETH_RSS_NONFRAG_IPV4_OTHER,
 742         },
 743         [MLX5_EXPANSION_IPV4_UDP] = {
 744                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 745                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP,
 746         },
 747         [MLX5_EXPANSION_IPV4_TCP] = {
 748                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 749                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP,
 750         },
 751         [MLX5_EXPANSION_IPV4_ESP] = {
 752                 .type = RTE_FLOW_ITEM_TYPE_ESP,
 753                 .rss_types = RTE_ETH_RSS_ESP,
 754         },
 755         [MLX5_EXPANSION_IPV6] = {
 756                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 757                                                   MLX5_EXPANSION_IPV6_TCP,
 758                                                   MLX5_EXPANSION_IPV6_ESP,
 759                                                   MLX5_EXPANSION_IPV6_FRAG_EXT),
 760                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 761                 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 |
 762                         RTE_ETH_RSS_NONFRAG_IPV6_OTHER,
 763         },
 764         [MLX5_EXPANSION_IPV6_UDP] = {
 765                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 766                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP,
 767         },
 768         [MLX5_EXPANSION_IPV6_TCP] = {
 769                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 770                 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP,
 771         },
 772         [MLX5_EXPANSION_IPV6_ESP] = {
 773                 .type = RTE_FLOW_ITEM_TYPE_ESP,
 774                 .rss_types = RTE_ETH_RSS_ESP,
 775         },
 776         [MLX5_EXPANSION_IPV6_FRAG_EXT] = {
 777                 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT,
 778         },
 779         [MLX5_EXPANSION_GTP] = {
 780                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 781                                                   MLX5_EXPANSION_IPV6),
 782                 .type = RTE_FLOW_ITEM_TYPE_GTP,
 783         },
 784         [MLX5_EXPANSION_GENEVE] = {
 785                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 786                                                   MLX5_EXPANSION_IPV4,
 787                                                   MLX5_EXPANSION_IPV6),
 788                 .type = RTE_FLOW_ITEM_TYPE_GENEVE,
 789         },
 790 };
 791
 792 static struct rte_flow_action_handle *
 793 mlx5_action_handle_create(struct rte_eth_dev *dev,
 794                           const struct rte_flow_indir_action_conf *conf,
 795                           const struct rte_flow_action *action,
 796                           struct rte_flow_error *error);
 797 static int mlx5_action_handle_destroy
 798                                 (struct rte_eth_dev *dev,
 799                                  struct rte_flow_action_handle *handle,
 800                                  struct rte_flow_error *error);
 801 static int mlx5_action_handle_update
 802                                 (struct rte_eth_dev *dev,
 803                                  struct rte_flow_action_handle *handle,
 804                                  const void *update,
 805                                  struct rte_flow_error *error);
 806 static int mlx5_action_handle_query
 807                                 (struct rte_eth_dev *dev,
 808                                  const struct rte_flow_action_handle *handle,
 809                                  void *data,
 810                                  struct rte_flow_error *error);
 811 static int
 812 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
 813                     struct rte_flow_tunnel *app_tunnel,
 814                     struct rte_flow_action **actions,
 815                     uint32_t *num_of_actions,
 816                     struct rte_flow_error *error);
 817 static int
 818 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
 819                        struct rte_flow_tunnel *app_tunnel,
 820                        struct rte_flow_item **items,
 821                        uint32_t *num_of_items,
 822                        struct rte_flow_error *error);
 823 static int
 824 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
 825                               struct rte_flow_item *pmd_items,
 826                               uint32_t num_items, struct rte_flow_error *err);
 827 static int
 828 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
 829                                 struct rte_flow_action *pmd_actions,
 830                                 uint32_t num_actions,
 831                                 struct rte_flow_error *err);
 832 static int
 833 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
 834                                   struct rte_mbuf *m,
 835                                   struct rte_flow_restore_info *info,
 836                                   struct rte_flow_error *err);
 837 static struct rte_flow_item_flex_handle *
 838 mlx5_flow_flex_item_create(struct rte_eth_dev *dev,
 839                            const struct rte_flow_item_flex_conf *conf,
 840                            struct rte_flow_error *error);
 841 static int
 842 mlx5_flow_flex_item_release(struct rte_eth_dev *dev,
 843                             const struct rte_flow_item_flex_handle *handle,
 844                             struct rte_flow_error *error);
 845 static int
 846 mlx5_flow_info_get(struct rte_eth_dev *dev,
 847                    struct rte_flow_port_info *port_info,
 848                    struct rte_flow_queue_info *queue_info,
 849                    struct rte_flow_error *error);
 850 static int
 851 mlx5_flow_port_configure(struct rte_eth_dev *dev,
 852                          const struct rte_flow_port_attr *port_attr,
 853                          uint16_t nb_queue,
 854                          const struct rte_flow_queue_attr *queue_attr[],
 855                          struct rte_flow_error *err);
 856
 857 static struct rte_flow_pattern_template *
 858 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev,
 859                 const struct rte_flow_pattern_template_attr *attr,
 860                 const struct rte_flow_item items[],
 861                 struct rte_flow_error *error);
 862
 863 static int
 864 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev,
 865                                    struct rte_flow_pattern_template *template,
 866                                    struct rte_flow_error *error);
 867 static struct rte_flow_actions_template *
 868 mlx5_flow_actions_template_create(struct rte_eth_dev *dev,
 869                         const struct rte_flow_actions_template_attr *attr,
 870                         const struct rte_flow_action actions[],
 871                         const struct rte_flow_action masks[],
 872                         struct rte_flow_error *error);
 873 static int
 874 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev,
 875                                    struct rte_flow_actions_template *template,
 876                                    struct rte_flow_error *error);
 877
 878 static struct rte_flow_template_table *
 879 mlx5_flow_table_create(struct rte_eth_dev *dev,
 880                        const struct rte_flow_template_table_attr *attr,
 881                        struct rte_flow_pattern_template *item_templates[],
 882                        uint8_t nb_item_templates,
 883                        struct rte_flow_actions_template *action_templates[],
 884                        uint8_t nb_action_templates,
 885                        struct rte_flow_error *error);
 886 static int
 887 mlx5_flow_table_destroy(struct rte_eth_dev *dev,
 888                         struct rte_flow_template_table *table,
 889                         struct rte_flow_error *error);
 890 static struct rte_flow *
 891 mlx5_flow_async_flow_create(struct rte_eth_dev *dev,
 892                             uint32_t queue,
 893                             const struct rte_flow_op_attr *attr,
 894                             struct rte_flow_template_table *table,
 895                             const struct rte_flow_item items[],
 896                             uint8_t pattern_template_index,
 897                             const struct rte_flow_action actions[],
 898                             uint8_t action_template_index,
 899                             void *user_data,
 900                             struct rte_flow_error *error);
 901 static int
 902 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev,
 903                              uint32_t queue,
 904                              const struct rte_flow_op_attr *attr,
 905                              struct rte_flow *flow,
 906                              void *user_data,
 907                              struct rte_flow_error *error);
 908 static int
 909 mlx5_flow_pull(struct rte_eth_dev *dev,
 910                uint32_t queue,
 911                struct rte_flow_op_result res[],
 912                uint16_t n_res,
 913                struct rte_flow_error *error);
 914 static int
 915 mlx5_flow_push(struct rte_eth_dev *dev,
 916                uint32_t queue,
 917                struct rte_flow_error *error);
 918
 919 static struct rte_flow_action_handle *
 920 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue,
 921                                  const struct rte_flow_op_attr *attr,
 922                                  const struct rte_flow_indir_action_conf *conf,
 923                                  const struct rte_flow_action *action,
 924                                  void *user_data,
 925                                  struct rte_flow_error *error);
 926
 927 static int
 928 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue,
 929                                  const struct rte_flow_op_attr *attr,
 930                                  struct rte_flow_action_handle *handle,
 931                                  const void *update,
 932                                  void *user_data,
 933                                  struct rte_flow_error *error);
 934
 935 static int
 936 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue,
 937                                   const struct rte_flow_op_attr *attr,
 938                                   struct rte_flow_action_handle *handle,
 939                                   void *user_data,
 940                                   struct rte_flow_error *error);
 941
 942 static const struct rte_flow_ops mlx5_flow_ops = {
 943         .validate = mlx5_flow_validate,
 944         .create = mlx5_flow_create,
 945         .destroy = mlx5_flow_destroy,
 946         .flush = mlx5_flow_flush,
 947         .isolate = mlx5_flow_isolate,
 948         .query = mlx5_flow_query,
 949         .dev_dump = mlx5_flow_dev_dump,
 950         .get_aged_flows = mlx5_flow_get_aged_flows,
 951         .action_handle_create = mlx5_action_handle_create,
 952         .action_handle_destroy = mlx5_action_handle_destroy,
 953         .action_handle_update = mlx5_action_handle_update,
 954         .action_handle_query = mlx5_action_handle_query,
 955         .tunnel_decap_set = mlx5_flow_tunnel_decap_set,
 956         .tunnel_match = mlx5_flow_tunnel_match,
 957         .tunnel_action_decap_release = mlx5_flow_tunnel_action_release,
 958         .tunnel_item_release = mlx5_flow_tunnel_item_release,
 959         .get_restore_info = mlx5_flow_tunnel_get_restore_info,
 960         .flex_item_create = mlx5_flow_flex_item_create,
 961         .flex_item_release = mlx5_flow_flex_item_release,
 962         .info_get = mlx5_flow_info_get,
 963         .configure = mlx5_flow_port_configure,
 964         .pattern_template_create = mlx5_flow_pattern_template_create,
 965         .pattern_template_destroy = mlx5_flow_pattern_template_destroy,
 966         .actions_template_create = mlx5_flow_actions_template_create,
 967         .actions_template_destroy = mlx5_flow_actions_template_destroy,
 968         .template_table_create = mlx5_flow_table_create,
 969         .template_table_destroy = mlx5_flow_table_destroy,
 970         .async_create = mlx5_flow_async_flow_create,
 971         .async_destroy = mlx5_flow_async_flow_destroy,
 972         .pull = mlx5_flow_pull,
 973         .push = mlx5_flow_push,
 974         .async_action_handle_create = mlx5_flow_async_action_handle_create,
 975         .async_action_handle_update = mlx5_flow_async_action_handle_update,
 976         .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy,
 977 };
 978
 979 /* Tunnel information. */
 980 struct mlx5_flow_tunnel_info {
 981         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 982         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 983 };
 984
 985 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 986         {
 987                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 988                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 989         },
 990         {
 991                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 992                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 993         },
 994         {
 995                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 996                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 997         },
 998         {
 999                 .tunnel = MLX5_FLOW_LAYER_GRE,
1000                 .ptype = RTE_PTYPE_TUNNEL_GRE,
1001         },
1002         {
1003                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
1004                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
1005         },
1006         {
1007                 .tunnel = MLX5_FLOW_LAYER_MPLS,
1008                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
1009         },
1010         {
1011                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
1012                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
1013         },
1014         {
1015                 .tunnel = MLX5_FLOW_LAYER_IPIP,
1016                 .ptype = RTE_PTYPE_TUNNEL_IP,
1017         },
1018         {
1019                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
1020                 .ptype = RTE_PTYPE_TUNNEL_IP,
1021         },
1022         {
1023                 .tunnel = MLX5_FLOW_LAYER_GTP,
1024                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
1025         },
1026 };
1027
1028
1029
1030 /**
1031  * Translate tag ID to register.
1032  *
1033  * @param[in] dev
1034  *   Pointer to the Ethernet device structure.
1035  * @param[in] feature
1036  *   The feature that request the register.
1037  * @param[in] id
1038  *   The request register ID.
1039  * @param[out] error
1040  *   Error description in case of any.
1041  *
1042  * @return
1043  *   The request register on success, a negative errno
1044  *   value otherwise and rte_errno is set.
1045  */
1046 int
1047 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
1048                      enum mlx5_feature_name feature,
1049                      uint32_t id,
1050                      struct rte_flow_error *error)
1051 {
1052         struct mlx5_priv *priv = dev->data->dev_private;
1053         struct mlx5_sh_config *config = &priv->sh->config;
1054         enum modify_reg start_reg;
1055         bool skip_mtr_reg = false;
1056
1057         switch (feature) {
1058         case MLX5_HAIRPIN_RX:
1059                 return REG_B;
1060         case MLX5_HAIRPIN_TX:
1061                 return REG_A;
1062         case MLX5_METADATA_RX:
1063                 switch (config->dv_xmeta_en) {
1064                 case MLX5_XMETA_MODE_LEGACY:
1065                         return REG_B;
1066                 case MLX5_XMETA_MODE_META16:
1067                         return REG_C_0;
1068                 case MLX5_XMETA_MODE_META32:
1069                         return REG_C_1;
1070                 }
1071                 break;
1072         case MLX5_METADATA_TX:
1073                 return REG_A;
1074         case MLX5_METADATA_FDB:
1075                 switch (config->dv_xmeta_en) {
1076                 case MLX5_XMETA_MODE_LEGACY:
1077                         return REG_NON;
1078                 case MLX5_XMETA_MODE_META16:
1079                         return REG_C_0;
1080                 case MLX5_XMETA_MODE_META32:
1081                         return REG_C_1;
1082                 }
1083                 break;
1084         case MLX5_FLOW_MARK:
1085                 switch (config->dv_xmeta_en) {
1086                 case MLX5_XMETA_MODE_LEGACY:
1087                         return REG_NON;
1088                 case MLX5_XMETA_MODE_META16:
1089                         return REG_C_1;
1090                 case MLX5_XMETA_MODE_META32:
1091                         return REG_C_0;
1092                 }
1093                 break;
1094         case MLX5_MTR_ID:
1095                 /*
1096                  * If meter color and meter id share one register, flow match
1097                  * should use the meter color register for match.
1098                  */
1099                 if (priv->mtr_reg_share)
1100                         return priv->mtr_color_reg;
1101                 else
1102                         return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
1103                                REG_C_3;
1104         case MLX5_MTR_COLOR:
1105         case MLX5_ASO_FLOW_HIT:
1106         case MLX5_ASO_CONNTRACK:
1107         case MLX5_SAMPLE_ID:
1108                 /* All features use the same REG_C. */
1109                 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
1110                 return priv->mtr_color_reg;
1111         case MLX5_COPY_MARK:
1112                 /*
1113                  * Metadata COPY_MARK register using is in meter suffix sub
1114                  * flow while with meter. It's safe to share the same register.
1115                  */
1116                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
1117         case MLX5_APP_TAG:
1118                 /*
1119                  * If meter is enable, it will engage the register for color
1120                  * match and flow match. If meter color match is not using the
1121                  * REG_C_2, need to skip the REG_C_x be used by meter color
1122                  * match.
1123                  * If meter is disable, free to use all available registers.
1124                  */
1125                 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
1126                             (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
1127                 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
1128                 if (id > (uint32_t)(REG_C_7 - start_reg))
1129                         return rte_flow_error_set(error, EINVAL,
1130                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1131                                                   NULL, "invalid tag id");
1132                 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
1133                         return rte_flow_error_set(error, ENOTSUP,
1134                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1135                                                   NULL, "unsupported tag id");
1136                 /*
1137                  * This case means meter is using the REG_C_x great than 2.
1138                  * Take care not to conflict with meter color REG_C_x.
1139                  * If the available index REG_C_y >= REG_C_x, skip the
1140                  * color register.
1141                  */
1142                 if (skip_mtr_reg && priv->sh->flow_mreg_c
1143                     [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
1144                         if (id >= (uint32_t)(REG_C_7 - start_reg))
1145                                 return rte_flow_error_set(error, EINVAL,
1146                                                        RTE_FLOW_ERROR_TYPE_ITEM,
1147                                                         NULL, "invalid tag id");
1148                         if (priv->sh->flow_mreg_c
1149                             [id + 1 + start_reg - REG_C_0] != REG_NON)
1150                                 return priv->sh->flow_mreg_c
1151                                                [id + 1 + start_reg - REG_C_0];
1152                         return rte_flow_error_set(error, ENOTSUP,
1153                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1154                                                   NULL, "unsupported tag id");
1155                 }
1156                 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0];
1157         }
1158         MLX5_ASSERT(false);
1159         return rte_flow_error_set(error, EINVAL,
1160                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1161                                   NULL, "invalid feature name");
1162 }
1163
1164 /**
1165  * Check extensive flow metadata register support.
1166  *
1167  * @param dev
1168  *   Pointer to rte_eth_dev structure.
1169  *
1170  * @return
1171  *   True if device supports extensive flow metadata register, otherwise false.
1172  */
1173 bool
1174 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
1175 {
1176         struct mlx5_priv *priv = dev->data->dev_private;
1177
1178         /*
1179          * Having available reg_c can be regarded inclusively as supporting
1180          * extensive flow metadata register, which could mean,
1181          * - metadata register copy action by modify header.
1182          * - 16 modify header actions is supported.
1183          * - reg_c's are preserved across different domain (FDB and NIC) on
1184          *   packet loopback by flow lookup miss.
1185          */
1186         return priv->sh->flow_mreg_c[2] != REG_NON;
1187 }
1188
1189 /**
1190  * Get the lowest priority.
1191  *
1192  * @param[in] dev
1193  *   Pointer to the Ethernet device structure.
1194  * @param[in] attributes
1195  *   Pointer to device flow rule attributes.
1196  *
1197  * @return
1198  *   The value of lowest priority of flow.
1199  */
1200 uint32_t
1201 mlx5_get_lowest_priority(struct rte_eth_dev *dev,
1202                           const struct rte_flow_attr *attr)
1203 {
1204         struct mlx5_priv *priv = dev->data->dev_private;
1205
1206         if (!attr->group && !attr->transfer)
1207                 return priv->sh->flow_max_priority - 2;
1208         return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1;
1209 }
1210
1211 /**
1212  * Calculate matcher priority of the flow.
1213  *
1214  * @param[in] dev
1215  *   Pointer to the Ethernet device structure.
1216  * @param[in] attr
1217  *   Pointer to device flow rule attributes.
1218  * @param[in] subpriority
1219  *   The priority based on the items.
1220  * @param[in] external
1221  *   Flow is user flow.
1222  * @return
1223  *   The matcher priority of the flow.
1224  */
1225 uint16_t
1226 mlx5_get_matcher_priority(struct rte_eth_dev *dev,
1227                           const struct rte_flow_attr *attr,
1228                           uint32_t subpriority, bool external)
1229 {
1230         uint16_t priority = (uint16_t)attr->priority;
1231         struct mlx5_priv *priv = dev->data->dev_private;
1232
1233         if (!attr->group && !attr->transfer) {
1234                 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
1235                         priority = priv->sh->flow_max_priority - 1;
1236                 return mlx5_os_flow_adjust_priority(dev, priority, subpriority);
1237         } else if (!external && attr->transfer && attr->group == 0 &&
1238                    attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) {
1239                 return (priv->sh->flow_max_priority - 1) * 3;
1240         }
1241         if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
1242                 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO;
1243         return priority * 3 + subpriority;
1244 }
1245
1246 /**
1247  * Verify the @p item specifications (spec, last, mask) are compatible with the
1248  * NIC capabilities.
1249  *
1250  * @param[in] item
1251  *   Item specification.
1252  * @param[in] mask
1253  *   @p item->mask or flow default bit-masks.
1254  * @param[in] nic_mask
1255  *   Bit-masks covering supported fields by the NIC to compare with user mask.
1256  * @param[in] size
1257  *   Bit-masks size in bytes.
1258  * @param[in] range_accepted
1259  *   True if range of values is accepted for specific fields, false otherwise.
1260  * @param[out] error
1261  *   Pointer to error structure.
1262  *
1263  * @return
1264  *   0 on success, a negative errno value otherwise and rte_errno is set.
1265  */
1266 int
1267 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
1268                           const uint8_t *mask,
1269                           const uint8_t *nic_mask,
1270                           unsigned int size,
1271                           bool range_accepted,
1272                           struct rte_flow_error *error)
1273 {
1274         unsigned int i;
1275
1276         MLX5_ASSERT(nic_mask);
1277         for (i = 0; i < size; ++i)
1278                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
1279                         return rte_flow_error_set(error, ENOTSUP,
1280                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1281                                                   item,
1282                                                   "mask enables non supported"
1283                                                   " bits");
1284         if (!item->spec && (item->mask || item->last))
1285                 return rte_flow_error_set(error, EINVAL,
1286                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1287                                           "mask/last without a spec is not"
1288                                           " supported");
1289         if (item->spec && item->last && !range_accepted) {
1290                 uint8_t spec[size];
1291                 uint8_t last[size];
1292                 unsigned int i;
1293                 int ret;
1294
1295                 for (i = 0; i < size; ++i) {
1296                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
1297                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
1298                 }
1299                 ret = memcmp(spec, last, size);
1300                 if (ret != 0)
1301                         return rte_flow_error_set(error, EINVAL,
1302                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1303                                                   item,
1304                                                   "range is not valid");
1305         }
1306         return 0;
1307 }
1308
1309 /**
1310  * Adjust the hash fields according to the @p flow information.
1311  *
1312  * @param[in] dev_flow.
1313  *   Pointer to the mlx5_flow.
1314  * @param[in] tunnel
1315  *   1 when the hash field is for a tunnel item.
1316  * @param[in] layer_types
1317  *   RTE_ETH_RSS_* types.
1318  * @param[in] hash_fields
1319  *   Item hash fields.
1320  *
1321  * @return
1322  *   The hash fields that should be used.
1323  */
1324 uint64_t
1325 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
1326                             int tunnel __rte_unused, uint64_t layer_types,
1327                             uint64_t hash_fields)
1328 {
1329 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1330         int rss_request_inner = rss_desc->level >= 2;
1331
1332         /* Check RSS hash level for tunnel. */
1333         if (tunnel && rss_request_inner)
1334                 hash_fields |= IBV_RX_HASH_INNER;
1335         else if (tunnel || rss_request_inner)
1336                 return 0;
1337 #endif
1338         /* Check if requested layer matches RSS hash fields. */
1339         if (!(rss_desc->types & layer_types))
1340                 return 0;
1341         return hash_fields;
1342 }
1343
1344 /**
1345  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
1346  * if several tunnel rules are used on this queue, the tunnel ptype will be
1347  * cleared.
1348  *
1349  * @param rxq_ctrl
1350  *   Rx queue to update.
1351  */
1352 static void
1353 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
1354 {
1355         unsigned int i;
1356         uint32_t tunnel_ptype = 0;
1357
1358         /* Look up for the ptype to use. */
1359         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
1360                 if (!rxq_ctrl->flow_tunnels_n[i])
1361                         continue;
1362                 if (!tunnel_ptype) {
1363                         tunnel_ptype = tunnels_info[i].ptype;
1364                 } else {
1365                         tunnel_ptype = 0;
1366                         break;
1367                 }
1368         }
1369         rxq_ctrl->rxq.tunnel = tunnel_ptype;
1370 }
1371
1372 /**
1373  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device
1374  * flow.
1375  *
1376  * @param[in] dev
1377  *   Pointer to the Ethernet device structure.
1378  * @param[in] dev_handle
1379  *   Pointer to device flow handle structure.
1380  */
1381 void
1382 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
1383                        struct mlx5_flow_handle *dev_handle)
1384 {
1385         struct mlx5_priv *priv = dev->data->dev_private;
1386         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1387         struct mlx5_ind_table_obj *ind_tbl = NULL;
1388         unsigned int i;
1389
1390         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1391                 struct mlx5_hrxq *hrxq;
1392
1393                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1394                               dev_handle->rix_hrxq);
1395                 if (hrxq)
1396                         ind_tbl = hrxq->ind_table;
1397         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1398                 struct mlx5_shared_action_rss *shared_rss;
1399
1400                 shared_rss = mlx5_ipool_get
1401                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1402                          dev_handle->rix_srss);
1403                 if (shared_rss)
1404                         ind_tbl = shared_rss->ind_tbl;
1405         }
1406         if (!ind_tbl)
1407                 return;
1408         for (i = 0; i != ind_tbl->queues_n; ++i) {
1409                 int idx = ind_tbl->queues[i];
1410                 struct mlx5_rxq_ctrl *rxq_ctrl;
1411
1412                 if (mlx5_is_external_rxq(dev, idx))
1413                         continue;
1414                 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx);
1415                 MLX5_ASSERT(rxq_ctrl != NULL);
1416                 if (rxq_ctrl == NULL)
1417                         continue;
1418                 /*
1419                  * To support metadata register copy on Tx loopback,
1420                  * this must be always enabled (metadata may arive
1421                  * from other port - not from local flows only.
1422                  */
1423                 if (tunnel) {
1424                         unsigned int j;
1425
1426                         /* Increase the counter matching the flow. */
1427                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1428                                 if ((tunnels_info[j].tunnel &
1429                                      dev_handle->layers) ==
1430                                     tunnels_info[j].tunnel) {
1431                                         rxq_ctrl->flow_tunnels_n[j]++;
1432                                         break;
1433                                 }
1434                         }
1435                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1436                 }
1437         }
1438 }
1439
1440 static void
1441 flow_rxq_mark_flag_set(struct rte_eth_dev *dev)
1442 {
1443         struct mlx5_priv *priv = dev->data->dev_private;
1444         struct mlx5_rxq_ctrl *rxq_ctrl;
1445
1446         if (priv->mark_enabled)
1447                 return;
1448         LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) {
1449                 rxq_ctrl->rxq.mark = 1;
1450         }
1451         priv->mark_enabled = 1;
1452 }
1453
1454 /**
1455  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1456  *
1457  * @param[in] dev
1458  *   Pointer to the Ethernet device structure.
1459  * @param[in] flow
1460  *   Pointer to flow structure.
1461  */
1462 static void
1463 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1464 {
1465         struct mlx5_priv *priv = dev->data->dev_private;
1466         uint32_t handle_idx;
1467         struct mlx5_flow_handle *dev_handle;
1468         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
1469
1470         MLX5_ASSERT(wks);
1471         if (wks->mark)
1472                 flow_rxq_mark_flag_set(dev);
1473         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1474                        handle_idx, dev_handle, next)
1475                 flow_drv_rxq_flags_set(dev, dev_handle);
1476 }
1477
1478 /**
1479  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1480  * device flow if no other flow uses it with the same kind of request.
1481  *
1482  * @param dev
1483  *   Pointer to Ethernet device.
1484  * @param[in] dev_handle
1485  *   Pointer to the device flow handle structure.
1486  */
1487 static void
1488 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1489                         struct mlx5_flow_handle *dev_handle)
1490 {
1491         struct mlx5_priv *priv = dev->data->dev_private;
1492         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1493         struct mlx5_ind_table_obj *ind_tbl = NULL;
1494         unsigned int i;
1495
1496         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1497                 struct mlx5_hrxq *hrxq;
1498
1499                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1500                               dev_handle->rix_hrxq);
1501                 if (hrxq)
1502                         ind_tbl = hrxq->ind_table;
1503         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1504                 struct mlx5_shared_action_rss *shared_rss;
1505
1506                 shared_rss = mlx5_ipool_get
1507                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1508                          dev_handle->rix_srss);
1509                 if (shared_rss)
1510                         ind_tbl = shared_rss->ind_tbl;
1511         }
1512         if (!ind_tbl)
1513                 return;
1514         MLX5_ASSERT(dev->data->dev_started);
1515         for (i = 0; i != ind_tbl->queues_n; ++i) {
1516                 int idx = ind_tbl->queues[i];
1517                 struct mlx5_rxq_ctrl *rxq_ctrl;
1518
1519                 if (mlx5_is_external_rxq(dev, idx))
1520                         continue;
1521                 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx);
1522                 MLX5_ASSERT(rxq_ctrl != NULL);
1523                 if (rxq_ctrl == NULL)
1524                         continue;
1525                 if (tunnel) {
1526                         unsigned int j;
1527
1528                         /* Decrease the counter matching the flow. */
1529                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1530                                 if ((tunnels_info[j].tunnel &
1531                                      dev_handle->layers) ==
1532                                     tunnels_info[j].tunnel) {
1533                                         rxq_ctrl->flow_tunnels_n[j]--;
1534                                         break;
1535                                 }
1536                         }
1537                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1538                 }
1539         }
1540 }
1541
1542 /**
1543  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1544  * @p flow if no other flow uses it with the same kind of request.
1545  *
1546  * @param dev
1547  *   Pointer to Ethernet device.
1548  * @param[in] flow
1549  *   Pointer to the flow.
1550  */
1551 static void
1552 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1553 {
1554         struct mlx5_priv *priv = dev->data->dev_private;
1555         uint32_t handle_idx;
1556         struct mlx5_flow_handle *dev_handle;
1557
1558         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1559                        handle_idx, dev_handle, next)
1560                 flow_drv_rxq_flags_trim(dev, dev_handle);
1561 }
1562
1563 /**
1564  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1565  *
1566  * @param dev
1567  *   Pointer to Ethernet device.
1568  */
1569 static void
1570 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1571 {
1572         struct mlx5_priv *priv = dev->data->dev_private;
1573         unsigned int i;
1574
1575         for (i = 0; i != priv->rxqs_n; ++i) {
1576                 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i);
1577                 unsigned int j;
1578
1579                 if (rxq == NULL || rxq->ctrl == NULL)
1580                         continue;
1581                 rxq->ctrl->rxq.mark = 0;
1582                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1583                         rxq->ctrl->flow_tunnels_n[j] = 0;
1584                 rxq->ctrl->rxq.tunnel = 0;
1585         }
1586         priv->mark_enabled = 0;
1587 }
1588
1589 /**
1590  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1591  *
1592  * @param[in] dev
1593  *   Pointer to the Ethernet device structure.
1594  */
1595 void
1596 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1597 {
1598         struct mlx5_priv *priv = dev->data->dev_private;
1599         unsigned int i;
1600
1601         for (i = 0; i != priv->rxqs_n; ++i) {
1602                 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i);
1603                 struct mlx5_rxq_data *data;
1604
1605                 if (rxq == NULL || rxq->ctrl == NULL)
1606                         continue;
1607                 data = &rxq->ctrl->rxq;
1608                 if (!rte_flow_dynf_metadata_avail()) {
1609                         data->dynf_meta = 0;
1610                         data->flow_meta_mask = 0;
1611                         data->flow_meta_offset = -1;
1612                         data->flow_meta_port_mask = 0;
1613                 } else {
1614                         data->dynf_meta = 1;
1615                         data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1616                         data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1617                         data->flow_meta_port_mask = priv->sh->dv_meta_mask;
1618                 }
1619         }
1620 }
1621
1622 /*
1623  * return a pointer to the desired action in the list of actions.
1624  *
1625  * @param[in] actions
1626  *   The list of actions to search the action in.
1627  * @param[in] action
1628  *   The action to find.
1629  *
1630  * @return
1631  *   Pointer to the action in the list, if found. NULL otherwise.
1632  */
1633 const struct rte_flow_action *
1634 mlx5_flow_find_action(const struct rte_flow_action *actions,
1635                       enum rte_flow_action_type action)
1636 {
1637         if (actions == NULL)
1638                 return NULL;
1639         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1640                 if (actions->type == action)
1641                         return actions;
1642         return NULL;
1643 }
1644
1645 /*
1646  * Validate the flag action.
1647  *
1648  * @param[in] action_flags
1649  *   Bit-fields that holds the actions detected until now.
1650  * @param[in] attr
1651  *   Attributes of flow that includes this action.
1652  * @param[out] error
1653  *   Pointer to error structure.
1654  *
1655  * @return
1656  *   0 on success, a negative errno value otherwise and rte_errno is set.
1657  */
1658 int
1659 mlx5_flow_validate_action_flag(uint64_t action_flags,
1660                                const struct rte_flow_attr *attr,
1661                                struct rte_flow_error *error)
1662 {
1663         if (action_flags & MLX5_FLOW_ACTION_MARK)
1664                 return rte_flow_error_set(error, EINVAL,
1665                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1666                                           "can't mark and flag in same flow");
1667         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1668                 return rte_flow_error_set(error, EINVAL,
1669                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1670                                           "can't have 2 flag"
1671                                           " actions in same flow");
1672         if (attr->egress)
1673                 return rte_flow_error_set(error, ENOTSUP,
1674                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1675                                           "flag action not supported for "
1676                                           "egress");
1677         return 0;
1678 }
1679
1680 /*
1681  * Validate the mark action.
1682  *
1683  * @param[in] action
1684  *   Pointer to the queue action.
1685  * @param[in] action_flags
1686  *   Bit-fields that holds the actions detected until now.
1687  * @param[in] attr
1688  *   Attributes of flow that includes this action.
1689  * @param[out] error
1690  *   Pointer to error structure.
1691  *
1692  * @return
1693  *   0 on success, a negative errno value otherwise and rte_errno is set.
1694  */
1695 int
1696 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1697                                uint64_t action_flags,
1698                                const struct rte_flow_attr *attr,
1699                                struct rte_flow_error *error)
1700 {
1701         const struct rte_flow_action_mark *mark = action->conf;
1702
1703         if (!mark)
1704                 return rte_flow_error_set(error, EINVAL,
1705                                           RTE_FLOW_ERROR_TYPE_ACTION,
1706                                           action,
1707                                           "configuration cannot be null");
1708         if (mark->id >= MLX5_FLOW_MARK_MAX)
1709                 return rte_flow_error_set(error, EINVAL,
1710                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1711                                           &mark->id,
1712                                           "mark id must in 0 <= id < "
1713                                           RTE_STR(MLX5_FLOW_MARK_MAX));
1714         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1715                 return rte_flow_error_set(error, EINVAL,
1716                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1717                                           "can't flag and mark in same flow");
1718         if (action_flags & MLX5_FLOW_ACTION_MARK)
1719                 return rte_flow_error_set(error, EINVAL,
1720                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1721                                           "can't have 2 mark actions in same"
1722                                           " flow");
1723         if (attr->egress)
1724                 return rte_flow_error_set(error, ENOTSUP,
1725                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1726                                           "mark action not supported for "
1727                                           "egress");
1728         return 0;
1729 }
1730
1731 /*
1732  * Validate the drop action.
1733  *
1734  * @param[in] action_flags
1735  *   Bit-fields that holds the actions detected until now.
1736  * @param[in] attr
1737  *   Attributes of flow that includes this action.
1738  * @param[out] error
1739  *   Pointer to error structure.
1740  *
1741  * @return
1742  *   0 on success, a negative errno value otherwise and rte_errno is set.
1743  */
1744 int
1745 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1746                                const struct rte_flow_attr *attr,
1747                                struct rte_flow_error *error)
1748 {
1749         if (attr->egress)
1750                 return rte_flow_error_set(error, ENOTSUP,
1751                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1752                                           "drop action not supported for "
1753                                           "egress");
1754         return 0;
1755 }
1756
1757 /*
1758  * Validate the queue action.
1759  *
1760  * @param[in] action
1761  *   Pointer to the queue action.
1762  * @param[in] action_flags
1763  *   Bit-fields that holds the actions detected until now.
1764  * @param[in] dev
1765  *   Pointer to the Ethernet device structure.
1766  * @param[in] attr
1767  *   Attributes of flow that includes this action.
1768  * @param[out] error
1769  *   Pointer to error structure.
1770  *
1771  * @return
1772  *   0 on success, a negative errno value otherwise and rte_errno is set.
1773  */
1774 int
1775 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1776                                 uint64_t action_flags,
1777                                 struct rte_eth_dev *dev,
1778                                 const struct rte_flow_attr *attr,
1779                                 struct rte_flow_error *error)
1780 {
1781         struct mlx5_priv *priv = dev->data->dev_private;
1782         const struct rte_flow_action_queue *queue = action->conf;
1783
1784         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1785                 return rte_flow_error_set(error, EINVAL,
1786                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1787                                           "can't have 2 fate actions in"
1788                                           " same flow");
1789         if (attr->egress)
1790                 return rte_flow_error_set(error, ENOTSUP,
1791                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1792                                           "queue action not supported for egress.");
1793         if (mlx5_is_external_rxq(dev, queue->index))
1794                 return 0;
1795         if (!priv->rxqs_n)
1796                 return rte_flow_error_set(error, EINVAL,
1797                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1798                                           NULL, "No Rx queues configured");
1799         if (queue->index >= priv->rxqs_n)
1800                 return rte_flow_error_set(error, EINVAL,
1801                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1802                                           &queue->index,
1803                                           "queue index out of range");
1804         if (mlx5_rxq_get(dev, queue->index) == NULL)
1805                 return rte_flow_error_set(error, EINVAL,
1806                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1807                                           &queue->index,
1808                                           "queue is not configured");
1809         return 0;
1810 }
1811
1812 /**
1813  * Validate queue numbers for device RSS.
1814  *
1815  * @param[in] dev
1816  *   Configured device.
1817  * @param[in] queues
1818  *   Array of queue numbers.
1819  * @param[in] queues_n
1820  *   Size of the @p queues array.
1821  * @param[out] error
1822  *   On error, filled with a textual error description.
1823  * @param[out] queue_idx
1824  *   On error, filled with an offending queue index in @p queues array.
1825  *
1826  * @return
1827  *   0 on success, a negative errno code on error.
1828  */
1829 static int
1830 mlx5_validate_rss_queues(struct rte_eth_dev *dev,
1831                          const uint16_t *queues, uint32_t queues_n,
1832                          const char **error, uint32_t *queue_idx)
1833 {
1834         const struct mlx5_priv *priv = dev->data->dev_private;
1835         bool is_hairpin = false;
1836         bool is_ext_rss = false;
1837         uint32_t i;
1838
1839         for (i = 0; i != queues_n; ++i) {
1840                 struct mlx5_rxq_ctrl *rxq_ctrl;
1841
1842                 if (mlx5_is_external_rxq(dev, queues[0])) {
1843                         is_ext_rss = true;
1844                         continue;
1845                 }
1846                 if (is_ext_rss) {
1847                         *error = "Combining external and regular RSS queues is not supported";
1848                         *queue_idx = i;
1849                         return -ENOTSUP;
1850                 }
1851                 if (queues[i] >= priv->rxqs_n) {
1852                         *error = "queue index out of range";
1853                         *queue_idx = i;
1854                         return -EINVAL;
1855                 }
1856                 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]);
1857                 if (rxq_ctrl == NULL) {
1858                         *error =  "queue is not configured";
1859                         *queue_idx = i;
1860                         return -EINVAL;
1861                 }
1862                 if (i == 0 && rxq_ctrl->is_hairpin)
1863                         is_hairpin = true;
1864                 if (is_hairpin != rxq_ctrl->is_hairpin) {
1865                         *error = "combining hairpin and regular RSS queues is not supported";
1866                         *queue_idx = i;
1867                         return -ENOTSUP;
1868                 }
1869         }
1870         return 0;
1871 }
1872
1873 /*
1874  * Validate the rss action.
1875  *
1876  * @param[in] dev
1877  *   Pointer to the Ethernet device structure.
1878  * @param[in] action
1879  *   Pointer to the queue action.
1880  * @param[out] error
1881  *   Pointer to error structure.
1882  *
1883  * @return
1884  *   0 on success, a negative errno value otherwise and rte_errno is set.
1885  */
1886 int
1887 mlx5_validate_action_rss(struct rte_eth_dev *dev,
1888                          const struct rte_flow_action *action,
1889                          struct rte_flow_error *error)
1890 {
1891         struct mlx5_priv *priv = dev->data->dev_private;
1892         const struct rte_flow_action_rss *rss = action->conf;
1893         int ret;
1894         const char *message;
1895         uint32_t queue_idx;
1896
1897         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1898             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1899                 return rte_flow_error_set(error, ENOTSUP,
1900                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1901                                           &rss->func,
1902                                           "RSS hash function not supported");
1903 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1904         if (rss->level > 2)
1905 #else
1906         if (rss->level > 1)
1907 #endif
1908                 return rte_flow_error_set(error, ENOTSUP,
1909                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1910                                           &rss->level,
1911                                           "tunnel RSS is not supported");
1912         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1913         if (rss->key_len == 0 && rss->key != NULL)
1914                 return rte_flow_error_set(error, ENOTSUP,
1915                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1916                                           &rss->key_len,
1917                                           "RSS hash key length 0");
1918         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1919                 return rte_flow_error_set(error, ENOTSUP,
1920                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1921                                           &rss->key_len,
1922                                           "RSS hash key too small");
1923         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1924                 return rte_flow_error_set(error, ENOTSUP,
1925                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1926                                           &rss->key_len,
1927                                           "RSS hash key too large");
1928         if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size)
1929                 return rte_flow_error_set(error, ENOTSUP,
1930                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1931                                           &rss->queue_num,
1932                                           "number of queues too large");
1933         if (rss->types & MLX5_RSS_HF_MASK)
1934                 return rte_flow_error_set(error, ENOTSUP,
1935                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1936                                           &rss->types,
1937                                           "some RSS protocols are not"
1938                                           " supported");
1939         if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) &&
1940             !(rss->types & RTE_ETH_RSS_IP))
1941                 return rte_flow_error_set(error, EINVAL,
1942                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1943                                           "L3 partial RSS requested but L3 RSS"
1944                                           " type not specified");
1945         if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) &&
1946             !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP)))
1947                 return rte_flow_error_set(error, EINVAL,
1948                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1949                                           "L4 partial RSS requested but L4 RSS"
1950                                           " type not specified");
1951         if (!priv->rxqs_n && priv->ext_rxqs == NULL)
1952                 return rte_flow_error_set(error, EINVAL,
1953                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1954                                           NULL, "No Rx queues configured");
1955         if (!rss->queue_num)
1956                 return rte_flow_error_set(error, EINVAL,
1957                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1958                                           NULL, "No queues configured");
1959         ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num,
1960                                        &message, &queue_idx);
1961         if (ret != 0) {
1962                 return rte_flow_error_set(error, -ret,
1963                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1964                                           &rss->queue[queue_idx], message);
1965         }
1966         return 0;
1967 }
1968
1969 /*
1970  * Validate the rss action.
1971  *
1972  * @param[in] action
1973  *   Pointer to the queue action.
1974  * @param[in] action_flags
1975  *   Bit-fields that holds the actions detected until now.
1976  * @param[in] dev
1977  *   Pointer to the Ethernet device structure.
1978  * @param[in] attr
1979  *   Attributes of flow that includes this action.
1980  * @param[in] item_flags
1981  *   Items that were detected.
1982  * @param[out] error
1983  *   Pointer to error structure.
1984  *
1985  * @return
1986  *   0 on success, a negative errno value otherwise and rte_errno is set.
1987  */
1988 int
1989 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1990                               uint64_t action_flags,
1991                               struct rte_eth_dev *dev,
1992                               const struct rte_flow_attr *attr,
1993                               uint64_t item_flags,
1994                               struct rte_flow_error *error)
1995 {
1996         const struct rte_flow_action_rss *rss = action->conf;
1997         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1998         int ret;
1999
2000         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
2001                 return rte_flow_error_set(error, EINVAL,
2002                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2003                                           "can't have 2 fate actions"
2004                                           " in same flow");
2005         ret = mlx5_validate_action_rss(dev, action, error);
2006         if (ret)
2007                 return ret;
2008         if (attr->egress)
2009                 return rte_flow_error_set(error, ENOTSUP,
2010                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
2011                                           "rss action not supported for "
2012                                           "egress");
2013         if (rss->level > 1 && !tunnel)
2014                 return rte_flow_error_set(error, EINVAL,
2015                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
2016                                           "inner RSS is not supported for "
2017                                           "non-tunnel flows");
2018         if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
2019             !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
2020                 return rte_flow_error_set(error, EINVAL,
2021                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
2022                                           "RSS on eCPRI is not supported now");
2023         }
2024         if ((item_flags & MLX5_FLOW_LAYER_MPLS) &&
2025             !(item_flags &
2026               (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) &&
2027             rss->level > 1)
2028                 return rte_flow_error_set(error, EINVAL,
2029                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2030                                           "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern");
2031         return 0;
2032 }
2033
2034 /*
2035  * Validate the default miss action.
2036  *
2037  * @param[in] action_flags
2038  *   Bit-fields that holds the actions detected until now.
2039  * @param[out] error
2040  *   Pointer to error structure.
2041  *
2042  * @return
2043  *   0 on success, a negative errno value otherwise and rte_errno is set.
2044  */
2045 int
2046 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
2047                                 const struct rte_flow_attr *attr,
2048                                 struct rte_flow_error *error)
2049 {
2050         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
2051                 return rte_flow_error_set(error, EINVAL,
2052                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2053                                           "can't have 2 fate actions in"
2054                                           " same flow");
2055         if (attr->egress)
2056                 return rte_flow_error_set(error, ENOTSUP,
2057                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
2058                                           "default miss action not supported "
2059                                           "for egress");
2060         if (attr->group)
2061                 return rte_flow_error_set(error, ENOTSUP,
2062                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
2063                                           "only group 0 is supported");
2064         if (attr->transfer)
2065                 return rte_flow_error_set(error, ENOTSUP,
2066                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
2067                                           NULL, "transfer is not supported");
2068         return 0;
2069 }
2070
2071 /*
2072  * Validate the count action.
2073  *
2074  * @param[in] dev
2075  *   Pointer to the Ethernet device structure.
2076  * @param[in] attr
2077  *   Attributes of flow that includes this action.
2078  * @param[out] error
2079  *   Pointer to error structure.
2080  *
2081  * @return
2082  *   0 on success, a negative errno value otherwise and rte_errno is set.
2083  */
2084 int
2085 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
2086                                 const struct rte_flow_attr *attr,
2087                                 struct rte_flow_error *error)
2088 {
2089         if (attr->egress)
2090                 return rte_flow_error_set(error, ENOTSUP,
2091                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
2092                                           "count action not supported for "
2093                                           "egress");
2094         return 0;
2095 }
2096
2097 /*
2098  * Validate the ASO CT action.
2099  *
2100  * @param[in] dev
2101  *   Pointer to the Ethernet device structure.
2102  * @param[in] conntrack
2103  *   Pointer to the CT action profile.
2104  * @param[out] error
2105  *   Pointer to error structure.
2106  *
2107  * @return
2108  *   0 on success, a negative errno value otherwise and rte_errno is set.
2109  */
2110 int
2111 mlx5_validate_action_ct(struct rte_eth_dev *dev,
2112                         const struct rte_flow_action_conntrack *conntrack,
2113                         struct rte_flow_error *error)
2114 {
2115         RTE_SET_USED(dev);
2116
2117         if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT)
2118                 return rte_flow_error_set(error, EINVAL,
2119                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2120                                           "Invalid CT state");
2121         if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST)
2122                 return rte_flow_error_set(error, EINVAL,
2123                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2124                                           "Invalid last TCP packet flag");
2125         return 0;
2126 }
2127
2128 /**
2129  * Verify the @p attributes will be correctly understood by the NIC and store
2130  * them in the @p flow if everything is correct.
2131  *
2132  * @param[in] dev
2133  *   Pointer to the Ethernet device structure.
2134  * @param[in] attributes
2135  *   Pointer to flow attributes
2136  * @param[out] error
2137  *   Pointer to error structure.
2138  *
2139  * @return
2140  *   0 on success, a negative errno value otherwise and rte_errno is set.
2141  */
2142 int
2143 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
2144                               const struct rte_flow_attr *attributes,
2145                               struct rte_flow_error *error)
2146 {
2147         struct mlx5_priv *priv = dev->data->dev_private;
2148         uint32_t priority_max = priv->sh->flow_max_priority - 1;
2149
2150         if (attributes->group)
2151                 return rte_flow_error_set(error, ENOTSUP,
2152                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
2153                                           NULL, "groups is not supported");
2154         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
2155             attributes->priority >= priority_max)
2156                 return rte_flow_error_set(error, ENOTSUP,
2157                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
2158                                           NULL, "priority out of range");
2159         if (attributes->egress)
2160                 return rte_flow_error_set(error, ENOTSUP,
2161                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
2162                                           "egress is not supported");
2163         if (attributes->transfer && !priv->sh->config.dv_esw_en)
2164                 return rte_flow_error_set(error, ENOTSUP,
2165                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
2166                                           NULL, "transfer is not supported");
2167         if (!attributes->ingress)
2168                 return rte_flow_error_set(error, EINVAL,
2169                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
2170                                           NULL,
2171                                           "ingress attribute is mandatory");
2172         return 0;
2173 }
2174
2175 /**
2176  * Validate ICMP6 item.
2177  *
2178  * @param[in] item
2179  *   Item specification.
2180  * @param[in] item_flags
2181  *   Bit-fields that holds the items detected until now.
2182  * @param[in] ext_vlan_sup
2183  *   Whether extended VLAN features are supported or not.
2184  * @param[out] error
2185  *   Pointer to error structure.
2186  *
2187  * @return
2188  *   0 on success, a negative errno value otherwise and rte_errno is set.
2189  */
2190 int
2191 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
2192                                uint64_t item_flags,
2193                                uint8_t target_protocol,
2194                                struct rte_flow_error *error)
2195 {
2196         const struct rte_flow_item_icmp6 *mask = item->mask;
2197         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2198         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
2199                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
2200         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2201                                       MLX5_FLOW_LAYER_OUTER_L4;
2202         int ret;
2203
2204         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
2205                 return rte_flow_error_set(error, EINVAL,
2206                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2207                                           "protocol filtering not compatible"
2208                                           " with ICMP6 layer");
2209         if (!(item_flags & l3m))
2210                 return rte_flow_error_set(error, EINVAL,
2211                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2212                                           "IPv6 is mandatory to filter on"
2213                                           " ICMP6");
2214         if (item_flags & l4m)
2215                 return rte_flow_error_set(error, EINVAL,
2216                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2217                                           "multiple L4 layers not supported");
2218         if (!mask)
2219                 mask = &rte_flow_item_icmp6_mask;
2220         ret = mlx5_flow_item_acceptable
2221                 (item, (const uint8_t *)mask,
2222                  (const uint8_t *)&rte_flow_item_icmp6_mask,
2223                  sizeof(struct rte_flow_item_icmp6),
2224                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2225         if (ret < 0)
2226                 return ret;
2227         return 0;
2228 }
2229
2230 /**
2231  * Validate ICMP item.
2232  *
2233  * @param[in] item
2234  *   Item specification.
2235  * @param[in] item_flags
2236  *   Bit-fields that holds the items detected until now.
2237  * @param[out] error
2238  *   Pointer to error structure.
2239  *
2240  * @return
2241  *   0 on success, a negative errno value otherwise and rte_errno is set.
2242  */
2243 int
2244 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
2245                              uint64_t item_flags,
2246                              uint8_t target_protocol,
2247                              struct rte_flow_error *error)
2248 {
2249         const struct rte_flow_item_icmp *mask = item->mask;
2250         const struct rte_flow_item_icmp nic_mask = {
2251                 .hdr.icmp_type = 0xff,
2252                 .hdr.icmp_code = 0xff,
2253                 .hdr.icmp_ident = RTE_BE16(0xffff),
2254                 .hdr.icmp_seq_nb = RTE_BE16(0xffff),
2255         };
2256         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2257         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
2258                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
2259         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2260                                       MLX5_FLOW_LAYER_OUTER_L4;
2261         int ret;
2262
2263         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
2264                 return rte_flow_error_set(error, EINVAL,
2265                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2266                                           "protocol filtering not compatible"
2267                                           " with ICMP layer");
2268         if (!(item_flags & l3m))
2269                 return rte_flow_error_set(error, EINVAL,
2270                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2271                                           "IPv4 is mandatory to filter"
2272                                           " on ICMP");
2273         if (item_flags & l4m)
2274                 return rte_flow_error_set(error, EINVAL,
2275                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2276                                           "multiple L4 layers not supported");
2277         if (!mask)
2278                 mask = &nic_mask;
2279         ret = mlx5_flow_item_acceptable
2280                 (item, (const uint8_t *)mask,
2281                  (const uint8_t *)&nic_mask,
2282                  sizeof(struct rte_flow_item_icmp),
2283                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2284         if (ret < 0)
2285                 return ret;
2286         return 0;
2287 }
2288
2289 /**
2290  * Validate Ethernet item.
2291  *
2292  * @param[in] item
2293  *   Item specification.
2294  * @param[in] item_flags
2295  *   Bit-fields that holds the items detected until now.
2296  * @param[out] error
2297  *   Pointer to error structure.
2298  *
2299  * @return
2300  *   0 on success, a negative errno value otherwise and rte_errno is set.
2301  */
2302 int
2303 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
2304                             uint64_t item_flags, bool ext_vlan_sup,
2305                             struct rte_flow_error *error)
2306 {
2307         const struct rte_flow_item_eth *mask = item->mask;
2308         const struct rte_flow_item_eth nic_mask = {
2309                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2310                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2311                 .type = RTE_BE16(0xffff),
2312                 .has_vlan = ext_vlan_sup ? 1 : 0,
2313         };
2314         int ret;
2315         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2316         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
2317                                        MLX5_FLOW_LAYER_OUTER_L2;
2318
2319         if (item_flags & ethm)
2320                 return rte_flow_error_set(error, ENOTSUP,
2321                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2322                                           "multiple L2 layers not supported");
2323         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
2324             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
2325                 return rte_flow_error_set(error, EINVAL,
2326                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2327                                           "L2 layer should not follow "
2328                                           "L3 layers");
2329         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
2330             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
2331                 return rte_flow_error_set(error, EINVAL,
2332                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2333                                           "L2 layer should not follow VLAN");
2334         if (item_flags & MLX5_FLOW_LAYER_GTP)
2335                 return rte_flow_error_set(error, EINVAL,
2336                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2337                                           "L2 layer should not follow GTP");
2338         if (!mask)
2339                 mask = &rte_flow_item_eth_mask;
2340         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2341                                         (const uint8_t *)&nic_mask,
2342                                         sizeof(struct rte_flow_item_eth),
2343                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2344         return ret;
2345 }
2346
2347 /**
2348  * Validate VLAN item.
2349  *
2350  * @param[in] item
2351  *   Item specification.
2352  * @param[in] item_flags
2353  *   Bit-fields that holds the items detected until now.
2354  * @param[in] dev
2355  *   Ethernet device flow is being created on.
2356  * @param[out] error
2357  *   Pointer to error structure.
2358  *
2359  * @return
2360  *   0 on success, a negative errno value otherwise and rte_errno is set.
2361  */
2362 int
2363 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
2364                              uint64_t item_flags,
2365                              struct rte_eth_dev *dev,
2366                              struct rte_flow_error *error)
2367 {
2368         const struct rte_flow_item_vlan *spec = item->spec;
2369         const struct rte_flow_item_vlan *mask = item->mask;
2370         const struct rte_flow_item_vlan nic_mask = {
2371                 .tci = RTE_BE16(UINT16_MAX),
2372                 .inner_type = RTE_BE16(UINT16_MAX),
2373         };
2374         uint16_t vlan_tag = 0;
2375         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2376         int ret;
2377         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2378                                         MLX5_FLOW_LAYER_INNER_L4) :
2379                                        (MLX5_FLOW_LAYER_OUTER_L3 |
2380                                         MLX5_FLOW_LAYER_OUTER_L4);
2381         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2382                                         MLX5_FLOW_LAYER_OUTER_VLAN;
2383
2384         if (item_flags & vlanm)
2385                 return rte_flow_error_set(error, EINVAL,
2386                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2387                                           "multiple VLAN layers not supported");
2388         else if ((item_flags & l34m) != 0)
2389                 return rte_flow_error_set(error, EINVAL,
2390                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2391                                           "VLAN cannot follow L3/L4 layer");
2392         if (!mask)
2393                 mask = &rte_flow_item_vlan_mask;
2394         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2395                                         (const uint8_t *)&nic_mask,
2396                                         sizeof(struct rte_flow_item_vlan),
2397                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2398         if (ret)
2399                 return ret;
2400         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2401                 struct mlx5_priv *priv = dev->data->dev_private;
2402
2403                 if (priv->vmwa_context) {
2404                         /*
2405                          * Non-NULL context means we have a virtual machine
2406                          * and SR-IOV enabled, we have to create VLAN interface
2407                          * to make hypervisor to setup E-Switch vport
2408                          * context correctly. We avoid creating the multiple
2409                          * VLAN interfaces, so we cannot support VLAN tag mask.
2410                          */
2411                         return rte_flow_error_set(error, EINVAL,
2412                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2413                                                   item,
2414                                                   "VLAN tag mask is not"
2415                                                   " supported in virtual"
2416                                                   " environment");
2417                 }
2418         }
2419         if (spec) {
2420                 vlan_tag = spec->tci;
2421                 vlan_tag &= mask->tci;
2422         }
2423         /*
2424          * From verbs perspective an empty VLAN is equivalent
2425          * to a packet without VLAN layer.
2426          */
2427         if (!vlan_tag)
2428                 return rte_flow_error_set(error, EINVAL,
2429                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2430                                           item->spec,
2431                                           "VLAN cannot be empty");
2432         return 0;
2433 }
2434
2435 /**
2436  * Validate IPV4 item.
2437  *
2438  * @param[in] item
2439  *   Item specification.
2440  * @param[in] item_flags
2441  *   Bit-fields that holds the items detected until now.
2442  * @param[in] last_item
2443  *   Previous validated item in the pattern items.
2444  * @param[in] ether_type
2445  *   Type in the ethernet layer header (including dot1q).
2446  * @param[in] acc_mask
2447  *   Acceptable mask, if NULL default internal default mask
2448  *   will be used to check whether item fields are supported.
2449  * @param[in] range_accepted
2450  *   True if range of values is accepted for specific fields, false otherwise.
2451  * @param[out] error
2452  *   Pointer to error structure.
2453  *
2454  * @return
2455  *   0 on success, a negative errno value otherwise and rte_errno is set.
2456  */
2457 int
2458 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
2459                              uint64_t item_flags,
2460                              uint64_t last_item,
2461                              uint16_t ether_type,
2462                              const struct rte_flow_item_ipv4 *acc_mask,
2463                              bool range_accepted,
2464                              struct rte_flow_error *error)
2465 {
2466         const struct rte_flow_item_ipv4 *mask = item->mask;
2467         const struct rte_flow_item_ipv4 *spec = item->spec;
2468         const struct rte_flow_item_ipv4 nic_mask = {
2469                 .hdr = {
2470                         .src_addr = RTE_BE32(0xffffffff),
2471                         .dst_addr = RTE_BE32(0xffffffff),
2472                         .type_of_service = 0xff,
2473                         .next_proto_id = 0xff,
2474                 },
2475         };
2476         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2477         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2478                                       MLX5_FLOW_LAYER_OUTER_L3;
2479         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2480                                       MLX5_FLOW_LAYER_OUTER_L4;
2481         int ret;
2482         uint8_t next_proto = 0xFF;
2483         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2484                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2485                                   MLX5_FLOW_LAYER_INNER_VLAN);
2486
2487         if ((last_item & l2_vlan) && ether_type &&
2488             ether_type != RTE_ETHER_TYPE_IPV4)
2489                 return rte_flow_error_set(error, EINVAL,
2490                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2491                                           "IPv4 cannot follow L2/VLAN layer "
2492                                           "which ether type is not IPv4");
2493         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2494                 if (mask && spec)
2495                         next_proto = mask->hdr.next_proto_id &
2496                                      spec->hdr.next_proto_id;
2497                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2498                         return rte_flow_error_set(error, EINVAL,
2499                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2500                                                   item,
2501                                                   "multiple tunnel "
2502                                                   "not supported");
2503         }
2504         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
2505                 return rte_flow_error_set(error, EINVAL,
2506                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2507                                           "wrong tunnel type - IPv6 specified "
2508                                           "but IPv4 item provided");
2509         if (item_flags & l3m)
2510                 return rte_flow_error_set(error, ENOTSUP,
2511                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2512                                           "multiple L3 layers not supported");
2513         else if (item_flags & l4m)
2514                 return rte_flow_error_set(error, EINVAL,
2515                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2516                                           "L3 cannot follow an L4 layer.");
2517         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2518                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2519                 return rte_flow_error_set(error, EINVAL,
2520                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2521                                           "L3 cannot follow an NVGRE layer.");
2522         if (!mask)
2523                 mask = &rte_flow_item_ipv4_mask;
2524         else if (mask->hdr.next_proto_id != 0 &&
2525                  mask->hdr.next_proto_id != 0xff)
2526                 return rte_flow_error_set(error, EINVAL,
2527                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2528                                           "partial mask is not supported"
2529                                           " for protocol");
2530         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2531                                         acc_mask ? (const uint8_t *)acc_mask
2532                                                  : (const uint8_t *)&nic_mask,
2533                                         sizeof(struct rte_flow_item_ipv4),
2534                                         range_accepted, error);
2535         if (ret < 0)
2536                 return ret;
2537         return 0;
2538 }
2539
2540 /**
2541  * Validate IPV6 item.
2542  *
2543  * @param[in] item
2544  *   Item specification.
2545  * @param[in] item_flags
2546  *   Bit-fields that holds the items detected until now.
2547  * @param[in] last_item
2548  *   Previous validated item in the pattern items.
2549  * @param[in] ether_type
2550  *   Type in the ethernet layer header (including dot1q).
2551  * @param[in] acc_mask
2552  *   Acceptable mask, if NULL default internal default mask
2553  *   will be used to check whether item fields are supported.
2554  * @param[out] error
2555  *   Pointer to error structure.
2556  *
2557  * @return
2558  *   0 on success, a negative errno value otherwise and rte_errno is set.
2559  */
2560 int
2561 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2562                              uint64_t item_flags,
2563                              uint64_t last_item,
2564                              uint16_t ether_type,
2565                              const struct rte_flow_item_ipv6 *acc_mask,
2566                              struct rte_flow_error *error)
2567 {
2568         const struct rte_flow_item_ipv6 *mask = item->mask;
2569         const struct rte_flow_item_ipv6 *spec = item->spec;
2570         const struct rte_flow_item_ipv6 nic_mask = {
2571                 .hdr = {
2572                         .src_addr =
2573                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2574                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2575                         .dst_addr =
2576                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2577                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2578                         .vtc_flow = RTE_BE32(0xffffffff),
2579                         .proto = 0xff,
2580                 },
2581         };
2582         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2583         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2584                                       MLX5_FLOW_LAYER_OUTER_L3;
2585         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2586                                       MLX5_FLOW_LAYER_OUTER_L4;
2587         int ret;
2588         uint8_t next_proto = 0xFF;
2589         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2590                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2591                                   MLX5_FLOW_LAYER_INNER_VLAN);
2592
2593         if ((last_item & l2_vlan) && ether_type &&
2594             ether_type != RTE_ETHER_TYPE_IPV6)
2595                 return rte_flow_error_set(error, EINVAL,
2596                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2597                                           "IPv6 cannot follow L2/VLAN layer "
2598                                           "which ether type is not IPv6");
2599         if (mask && mask->hdr.proto == UINT8_MAX && spec)
2600                 next_proto = spec->hdr.proto;
2601         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2602                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2603                         return rte_flow_error_set(error, EINVAL,
2604                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2605                                                   item,
2606                                                   "multiple tunnel "
2607                                                   "not supported");
2608         }
2609         if (next_proto == IPPROTO_HOPOPTS  ||
2610             next_proto == IPPROTO_ROUTING  ||
2611             next_proto == IPPROTO_FRAGMENT ||
2612             next_proto == IPPROTO_ESP      ||
2613             next_proto == IPPROTO_AH       ||
2614             next_proto == IPPROTO_DSTOPTS)
2615                 return rte_flow_error_set(error, EINVAL,
2616                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2617                                           "IPv6 proto (next header) should "
2618                                           "not be set as extension header");
2619         if (item_flags & MLX5_FLOW_LAYER_IPIP)
2620                 return rte_flow_error_set(error, EINVAL,
2621                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2622                                           "wrong tunnel type - IPv4 specified "
2623                                           "but IPv6 item provided");
2624         if (item_flags & l3m)
2625                 return rte_flow_error_set(error, ENOTSUP,
2626                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2627                                           "multiple L3 layers not supported");
2628         else if (item_flags & l4m)
2629                 return rte_flow_error_set(error, EINVAL,
2630                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2631                                           "L3 cannot follow an L4 layer.");
2632         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2633                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2634                 return rte_flow_error_set(error, EINVAL,
2635                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2636                                           "L3 cannot follow an NVGRE layer.");
2637         if (!mask)
2638                 mask = &rte_flow_item_ipv6_mask;
2639         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2640                                         acc_mask ? (const uint8_t *)acc_mask
2641                                                  : (const uint8_t *)&nic_mask,
2642                                         sizeof(struct rte_flow_item_ipv6),
2643                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2644         if (ret < 0)
2645                 return ret;
2646         return 0;
2647 }
2648
2649 /**
2650  * Validate ESP item.
2651  *
2652  * @param[in] item
2653  *   Item specification.
2654  * @param[in] item_flags
2655  *   Bit-fields that holds the items detected until now.
2656  * @param[in] target_protocol
2657  *   The next protocol in the previous item.
2658  * @param[out] error
2659  *   Pointer to error structure.
2660  *
2661  * @return
2662  *   0 on success, a negative errno value otherwise and rte_errno is set.
2663  */
2664 int
2665 mlx5_flow_validate_item_esp(const struct rte_flow_item *item,
2666                             uint64_t item_flags,
2667                             uint8_t target_protocol,
2668                             struct rte_flow_error *error)
2669 {
2670         const struct rte_flow_item_esp *mask = item->mask;
2671         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2672         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2673                                       MLX5_FLOW_LAYER_OUTER_L3;
2674         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2675                                       MLX5_FLOW_LAYER_OUTER_L4;
2676         int ret;
2677
2678         if (!(item_flags & l3m))
2679                 return rte_flow_error_set(error, EINVAL,
2680                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2681                                           "L3 is mandatory to filter on L4");
2682         if (item_flags & l4m)
2683                 return rte_flow_error_set(error, EINVAL,
2684                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2685                                           "multiple L4 layers not supported");
2686         if (target_protocol != 0xff && target_protocol != IPPROTO_ESP)
2687                 return rte_flow_error_set(error, EINVAL,
2688                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2689                                           "protocol filtering not compatible"
2690                                           " with ESP layer");
2691         if (!mask)
2692                 mask = &rte_flow_item_esp_mask;
2693         ret = mlx5_flow_item_acceptable
2694                 (item, (const uint8_t *)mask,
2695                  (const uint8_t *)&rte_flow_item_esp_mask,
2696                  sizeof(struct rte_flow_item_esp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2697                  error);
2698         if (ret < 0)
2699                 return ret;
2700         return 0;
2701 }
2702
2703 /**
2704  * Validate UDP item.
2705  *
2706  * @param[in] item
2707  *   Item specification.
2708  * @param[in] item_flags
2709  *   Bit-fields that holds the items detected until now.
2710  * @param[in] target_protocol
2711  *   The next protocol in the previous item.
2712  * @param[in] flow_mask
2713  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2714  * @param[out] error
2715  *   Pointer to error structure.
2716  *
2717  * @return
2718  *   0 on success, a negative errno value otherwise and rte_errno is set.
2719  */
2720 int
2721 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2722                             uint64_t item_flags,
2723                             uint8_t target_protocol,
2724                             struct rte_flow_error *error)
2725 {
2726         const struct rte_flow_item_udp *mask = item->mask;
2727         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2728         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2729                                       MLX5_FLOW_LAYER_OUTER_L3;
2730         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2731                                       MLX5_FLOW_LAYER_OUTER_L4;
2732         int ret;
2733
2734         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2735                 return rte_flow_error_set(error, EINVAL,
2736                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2737                                           "protocol filtering not compatible"
2738                                           " with UDP layer");
2739         if (!(item_flags & l3m))
2740                 return rte_flow_error_set(error, EINVAL,
2741                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2742                                           "L3 is mandatory to filter on L4");
2743         if (item_flags & l4m)
2744                 return rte_flow_error_set(error, EINVAL,
2745                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2746                                           "multiple L4 layers not supported");
2747         if (!mask)
2748                 mask = &rte_flow_item_udp_mask;
2749         ret = mlx5_flow_item_acceptable
2750                 (item, (const uint8_t *)mask,
2751                  (const uint8_t *)&rte_flow_item_udp_mask,
2752                  sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2753                  error);
2754         if (ret < 0)
2755                 return ret;
2756         return 0;
2757 }
2758
2759 /**
2760  * Validate TCP item.
2761  *
2762  * @param[in] item
2763  *   Item specification.
2764  * @param[in] item_flags
2765  *   Bit-fields that holds the items detected until now.
2766  * @param[in] target_protocol
2767  *   The next protocol in the previous item.
2768  * @param[out] error
2769  *   Pointer to error structure.
2770  *
2771  * @return
2772  *   0 on success, a negative errno value otherwise and rte_errno is set.
2773  */
2774 int
2775 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2776                             uint64_t item_flags,
2777                             uint8_t target_protocol,
2778                             const struct rte_flow_item_tcp *flow_mask,
2779                             struct rte_flow_error *error)
2780 {
2781         const struct rte_flow_item_tcp *mask = item->mask;
2782         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2783         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2784                                       MLX5_FLOW_LAYER_OUTER_L3;
2785         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2786                                       MLX5_FLOW_LAYER_OUTER_L4;
2787         int ret;
2788
2789         MLX5_ASSERT(flow_mask);
2790         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2791                 return rte_flow_error_set(error, EINVAL,
2792                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2793                                           "protocol filtering not compatible"
2794                                           " with TCP layer");
2795         if (!(item_flags & l3m))
2796                 return rte_flow_error_set(error, EINVAL,
2797                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2798                                           "L3 is mandatory to filter on L4");
2799         if (item_flags & l4m)
2800                 return rte_flow_error_set(error, EINVAL,
2801                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2802                                           "multiple L4 layers not supported");
2803         if (!mask)
2804                 mask = &rte_flow_item_tcp_mask;
2805         ret = mlx5_flow_item_acceptable
2806                 (item, (const uint8_t *)mask,
2807                  (const uint8_t *)flow_mask,
2808                  sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2809                  error);
2810         if (ret < 0)
2811                 return ret;
2812         return 0;
2813 }
2814
2815 /**
2816  * Validate VXLAN item.
2817  *
2818  * @param[in] dev
2819  *   Pointer to the Ethernet device structure.
2820  * @param[in] udp_dport
2821  *   UDP destination port
2822  * @param[in] item
2823  *   Item specification.
2824  * @param[in] item_flags
2825  *   Bit-fields that holds the items detected until now.
2826  * @param[in] attr
2827  *   Flow rule attributes.
2828  * @param[out] error
2829  *   Pointer to error structure.
2830  *
2831  * @return
2832  *   0 on success, a negative errno value otherwise and rte_errno is set.
2833  */
2834 int
2835 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev,
2836                               uint16_t udp_dport,
2837                               const struct rte_flow_item *item,
2838                               uint64_t item_flags,
2839                               const struct rte_flow_attr *attr,
2840                               struct rte_flow_error *error)
2841 {
2842         const struct rte_flow_item_vxlan *spec = item->spec;
2843         const struct rte_flow_item_vxlan *mask = item->mask;
2844         int ret;
2845         struct mlx5_priv *priv = dev->data->dev_private;
2846         union vni {
2847                 uint32_t vlan_id;
2848                 uint8_t vni[4];
2849         } id = { .vlan_id = 0, };
2850         const struct rte_flow_item_vxlan nic_mask = {
2851                 .vni = "\xff\xff\xff",
2852                 .rsvd1 = 0xff,
2853         };
2854         const struct rte_flow_item_vxlan *valid_mask;
2855
2856         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2857                 return rte_flow_error_set(error, ENOTSUP,
2858                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2859                                           "multiple tunnel layers not"
2860                                           " supported");
2861         valid_mask = &rte_flow_item_vxlan_mask;
2862         /*
2863          * Verify only UDPv4 is present as defined in
2864          * https://tools.ietf.org/html/rfc7348
2865          */
2866         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2867                 return rte_flow_error_set(error, EINVAL,
2868                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2869                                           "no outer UDP layer found");
2870         if (!mask)
2871                 mask = &rte_flow_item_vxlan_mask;
2872
2873         if (priv->sh->steering_format_version !=
2874             MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 ||
2875             !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) {
2876                 /* FDB domain & NIC domain non-zero group */
2877                 if ((attr->transfer || attr->group) && priv->sh->misc5_cap)
2878                         valid_mask = &nic_mask;
2879                 /* Group zero in NIC domain */
2880                 if (!attr->group && !attr->transfer &&
2881                     priv->sh->tunnel_header_0_1)
2882                         valid_mask = &nic_mask;
2883         }
2884         ret = mlx5_flow_item_acceptable
2885                 (item, (const uint8_t *)mask,
2886                  (const uint8_t *)valid_mask,
2887                  sizeof(struct rte_flow_item_vxlan),
2888                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2889         if (ret < 0)
2890                 return ret;
2891         if (spec) {
2892                 memcpy(&id.vni[1], spec->vni, 3);
2893                 memcpy(&id.vni[1], mask->vni, 3);
2894         }
2895         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2896                 return rte_flow_error_set(error, ENOTSUP,
2897                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2898                                           "VXLAN tunnel must be fully defined");
2899         return 0;
2900 }
2901
2902 /**
2903  * Validate VXLAN_GPE item.
2904  *
2905  * @param[in] item
2906  *   Item specification.
2907  * @param[in] item_flags
2908  *   Bit-fields that holds the items detected until now.
2909  * @param[in] priv
2910  *   Pointer to the private data structure.
2911  * @param[in] target_protocol
2912  *   The next protocol in the previous item.
2913  * @param[out] error
2914  *   Pointer to error structure.
2915  *
2916  * @return
2917  *   0 on success, a negative errno value otherwise and rte_errno is set.
2918  */
2919 int
2920 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2921                                   uint64_t item_flags,
2922                                   struct rte_eth_dev *dev,
2923                                   struct rte_flow_error *error)
2924 {
2925         struct mlx5_priv *priv = dev->data->dev_private;
2926         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2927         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2928         int ret;
2929         union vni {
2930                 uint32_t vlan_id;
2931                 uint8_t vni[4];
2932         } id = { .vlan_id = 0, };
2933
2934         if (!priv->sh->config.l3_vxlan_en)
2935                 return rte_flow_error_set(error, ENOTSUP,
2936                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2937                                           "L3 VXLAN is not enabled by device"
2938                                           " parameter and/or not configured in"
2939                                           " firmware");
2940         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2941                 return rte_flow_error_set(error, ENOTSUP,
2942                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2943                                           "multiple tunnel layers not"
2944                                           " supported");
2945         /*
2946          * Verify only UDPv4 is present as defined in
2947          * https://tools.ietf.org/html/rfc7348
2948          */
2949         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2950                 return rte_flow_error_set(error, EINVAL,
2951                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2952                                           "no outer UDP layer found");
2953         if (!mask)
2954                 mask = &rte_flow_item_vxlan_gpe_mask;
2955         ret = mlx5_flow_item_acceptable
2956                 (item, (const uint8_t *)mask,
2957                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2958                  sizeof(struct rte_flow_item_vxlan_gpe),
2959                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2960         if (ret < 0)
2961                 return ret;
2962         if (spec) {
2963                 if (spec->protocol)
2964                         return rte_flow_error_set(error, ENOTSUP,
2965                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2966                                                   item,
2967                                                   "VxLAN-GPE protocol"
2968                                                   " not supported");
2969                 memcpy(&id.vni[1], spec->vni, 3);
2970                 memcpy(&id.vni[1], mask->vni, 3);
2971         }
2972         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2973                 return rte_flow_error_set(error, ENOTSUP,
2974                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2975                                           "VXLAN-GPE tunnel must be fully"
2976                                           " defined");
2977         return 0;
2978 }
2979 /**
2980  * Validate GRE Key item.
2981  *
2982  * @param[in] item
2983  *   Item specification.
2984  * @param[in] item_flags
2985  *   Bit flags to mark detected items.
2986  * @param[in] gre_item
2987  *   Pointer to gre_item
2988  * @param[out] error
2989  *   Pointer to error structure.
2990  *
2991  * @return
2992  *   0 on success, a negative errno value otherwise and rte_errno is set.
2993  */
2994 int
2995 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2996                                 uint64_t item_flags,
2997                                 const struct rte_flow_item *gre_item,
2998                                 struct rte_flow_error *error)
2999 {
3000         const rte_be32_t *mask = item->mask;
3001         int ret = 0;
3002         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
3003         const struct rte_flow_item_gre *gre_spec;
3004         const struct rte_flow_item_gre *gre_mask;
3005
3006         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
3007                 return rte_flow_error_set(error, ENOTSUP,
3008                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3009                                           "Multiple GRE key not support");
3010         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
3011                 return rte_flow_error_set(error, ENOTSUP,
3012                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3013                                           "No preceding GRE header");
3014         if (item_flags & MLX5_FLOW_LAYER_INNER)
3015                 return rte_flow_error_set(error, ENOTSUP,
3016                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3017                                           "GRE key following a wrong item");
3018         gre_mask = gre_item->mask;
3019         if (!gre_mask)
3020                 gre_mask = &rte_flow_item_gre_mask;
3021         gre_spec = gre_item->spec;
3022         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
3023                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
3024                 return rte_flow_error_set(error, EINVAL,
3025                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3026                                           "Key bit must be on");
3027
3028         if (!mask)
3029                 mask = &gre_key_default_mask;
3030         ret = mlx5_flow_item_acceptable
3031                 (item, (const uint8_t *)mask,
3032                  (const uint8_t *)&gre_key_default_mask,
3033                  sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3034         return ret;
3035 }
3036
3037 /**
3038  * Validate GRE optional item.
3039  *
3040  * @param[in] dev
3041  *   Pointer to the Ethernet device structure.
3042  * @param[in] item
3043  *   Item specification.
3044  * @param[in] item_flags
3045  *   Bit flags to mark detected items.
3046  * @param[in] attr
3047  *   Flow rule attributes.
3048  * @param[in] gre_item
3049  *   Pointer to gre_item
3050  * @param[out] error
3051  *   Pointer to error structure.
3052  *
3053  * @return
3054  *   0 on success, a negative errno value otherwise and rte_errno is set.
3055  */
3056 int
3057 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev,
3058                                    const struct rte_flow_item *item,
3059                                    uint64_t item_flags,
3060                                    const struct rte_flow_attr *attr,
3061                                    const struct rte_flow_item *gre_item,
3062                                    struct rte_flow_error *error)
3063 {
3064         const struct rte_flow_item_gre *gre_spec = gre_item->spec;
3065         const struct rte_flow_item_gre *gre_mask = gre_item->mask;
3066         const struct rte_flow_item_gre_opt *spec = item->spec;
3067         const struct rte_flow_item_gre_opt *mask = item->mask;
3068         struct mlx5_priv *priv = dev->data->dev_private;
3069         int ret = 0;
3070         struct rte_flow_item_gre_opt nic_mask = {
3071                 .checksum_rsvd = {
3072                         .checksum = RTE_BE16(UINT16_MAX),
3073                         .reserved1 = 0x0,
3074                 },
3075                 .key = {
3076                         .key = RTE_BE32(UINT32_MAX),
3077                 },
3078                 .sequence = {
3079                         .sequence = RTE_BE32(UINT32_MAX),
3080                 },
3081         };
3082
3083         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
3084                 return rte_flow_error_set(error, ENOTSUP,
3085                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3086                                           "No preceding GRE header");
3087         if (item_flags & MLX5_FLOW_LAYER_INNER)
3088                 return rte_flow_error_set(error, ENOTSUP,
3089                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3090                                           "GRE option following a wrong item");
3091         if (!spec || !mask)
3092                 return rte_flow_error_set(error, EINVAL,
3093                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3094                                           "At least one field gre_option(checksum/key/sequence) must be specified");
3095         if (!gre_mask)
3096                 gre_mask = &rte_flow_item_gre_mask;
3097         if (mask->checksum_rsvd.checksum)
3098                 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) &&
3099                                  !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000)))
3100                         return rte_flow_error_set(error, EINVAL,
3101                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3102                                                   item,
3103                                                   "Checksum bit must be on");
3104         if (mask->key.key)
3105                 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
3106                                  !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
3107                         return rte_flow_error_set(error, EINVAL,
3108                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3109                                                   item, "Key bit must be on");
3110         if (mask->sequence.sequence)
3111                 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) &&
3112                                  !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000)))
3113                         return rte_flow_error_set(error, EINVAL,
3114                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3115                                                   item,
3116                                                   "Sequence bit must be on");
3117         if (mask->checksum_rsvd.checksum || mask->sequence.sequence) {
3118                 if (priv->sh->steering_format_version ==
3119                     MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 ||
3120                     ((attr->group || attr->transfer) &&
3121                      !priv->sh->misc5_cap) ||
3122                     (!(priv->sh->tunnel_header_0_1 &&
3123                        priv->sh->tunnel_header_2_3) &&
3124                     !attr->group && !attr->transfer))
3125                         return rte_flow_error_set(error, EINVAL,
3126                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3127                                                   item,
3128                                                   "Checksum/Sequence not supported");
3129         }
3130         ret = mlx5_flow_item_acceptable
3131                 (item, (const uint8_t *)mask,
3132                  (const uint8_t *)&nic_mask,
3133                  sizeof(struct rte_flow_item_gre_opt),
3134                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3135         return ret;
3136 }
3137
3138 /**
3139  * Validate GRE item.
3140  *
3141  * @param[in] item
3142  *   Item specification.
3143  * @param[in] item_flags
3144  *   Bit flags to mark detected items.
3145  * @param[in] target_protocol
3146  *   The next protocol in the previous item.
3147  * @param[out] error
3148  *   Pointer to error structure.
3149  *
3150  * @return
3151  *   0 on success, a negative errno value otherwise and rte_errno is set.
3152  */
3153 int
3154 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
3155                             uint64_t item_flags,
3156                             uint8_t target_protocol,
3157                             struct rte_flow_error *error)
3158 {
3159         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
3160         const struct rte_flow_item_gre *mask = item->mask;
3161         int ret;
3162         const struct rte_flow_item_gre nic_mask = {
3163                 .c_rsvd0_ver = RTE_BE16(0xB000),
3164                 .protocol = RTE_BE16(UINT16_MAX),
3165         };
3166
3167         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
3168                 return rte_flow_error_set(error, EINVAL,
3169                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3170                                           "protocol filtering not compatible"
3171                                           " with this GRE layer");
3172         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3173                 return rte_flow_error_set(error, ENOTSUP,
3174                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3175                                           "multiple tunnel layers not"
3176                                           " supported");
3177         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
3178                 return rte_flow_error_set(error, ENOTSUP,
3179                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3180                                           "L3 Layer is missing");
3181         if (!mask)
3182                 mask = &rte_flow_item_gre_mask;
3183         ret = mlx5_flow_item_acceptable
3184                 (item, (const uint8_t *)mask,
3185                  (const uint8_t *)&nic_mask,
3186                  sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
3187                  error);
3188         if (ret < 0)
3189                 return ret;
3190 #ifndef HAVE_MLX5DV_DR
3191 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
3192         if (spec && (spec->protocol & mask->protocol))
3193                 return rte_flow_error_set(error, ENOTSUP,
3194                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3195                                           "without MPLS support the"
3196                                           " specification cannot be used for"
3197                                           " filtering");
3198 #endif
3199 #endif
3200         return 0;
3201 }
3202
3203 /**
3204  * Validate Geneve item.
3205  *
3206  * @param[in] item
3207  *   Item specification.
3208  * @param[in] itemFlags
3209  *   Bit-fields that holds the items detected until now.
3210  * @param[in] enPriv
3211  *   Pointer to the private data structure.
3212  * @param[out] error
3213  *   Pointer to error structure.
3214  *
3215  * @return
3216  *   0 on success, a negative errno value otherwise and rte_errno is set.
3217  */
3218
3219 int
3220 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
3221                                uint64_t item_flags,
3222                                struct rte_eth_dev *dev,
3223                                struct rte_flow_error *error)
3224 {
3225         struct mlx5_priv *priv = dev->data->dev_private;
3226         const struct rte_flow_item_geneve *spec = item->spec;
3227         const struct rte_flow_item_geneve *mask = item->mask;
3228         int ret;
3229         uint16_t gbhdr;
3230         uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ?
3231                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
3232         const struct rte_flow_item_geneve nic_mask = {
3233                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
3234                 .vni = "\xff\xff\xff",
3235                 .protocol = RTE_BE16(UINT16_MAX),
3236         };
3237
3238         if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx)
3239                 return rte_flow_error_set(error, ENOTSUP,
3240                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3241                                           "L3 Geneve is not enabled by device"
3242                                           " parameter and/or not configured in"
3243                                           " firmware");
3244         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3245                 return rte_flow_error_set(error, ENOTSUP,
3246                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3247                                           "multiple tunnel layers not"
3248                                           " supported");
3249         /*
3250          * Verify only UDPv4 is present as defined in
3251          * https://tools.ietf.org/html/rfc7348
3252          */
3253         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
3254                 return rte_flow_error_set(error, EINVAL,
3255                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3256                                           "no outer UDP layer found");
3257         if (!mask)
3258                 mask = &rte_flow_item_geneve_mask;
3259         ret = mlx5_flow_item_acceptable
3260                                   (item, (const uint8_t *)mask,
3261                                    (const uint8_t *)&nic_mask,
3262                                    sizeof(struct rte_flow_item_geneve),
3263                                    MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3264         if (ret)
3265                 return ret;
3266         if (spec) {
3267                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
3268                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
3269                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
3270                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
3271                         return rte_flow_error_set(error, ENOTSUP,
3272                                                   RTE_FLOW_ERROR_TYPE_ITEM,
3273                                                   item,
3274                                                   "Geneve protocol unsupported"
3275                                                   " fields are being used");
3276                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
3277                         return rte_flow_error_set
3278                                         (error, ENOTSUP,
3279                                          RTE_FLOW_ERROR_TYPE_ITEM,
3280                                          item,
3281                                          "Unsupported Geneve options length");
3282         }
3283         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
3284                 return rte_flow_error_set
3285                                     (error, ENOTSUP,
3286                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
3287                                      "Geneve tunnel must be fully defined");
3288         return 0;
3289 }
3290
3291 /**
3292  * Validate Geneve TLV option item.
3293  *
3294  * @param[in] item
3295  *   Item specification.
3296  * @param[in] last_item
3297  *   Previous validated item in the pattern items.
3298  * @param[in] geneve_item
3299  *   Previous GENEVE item specification.
3300  * @param[in] dev
3301  *   Pointer to the rte_eth_dev structure.
3302  * @param[out] error
3303  *   Pointer to error structure.
3304  *
3305  * @return
3306  *   0 on success, a negative errno value otherwise and rte_errno is set.
3307  */
3308 int
3309 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item,
3310                                    uint64_t last_item,
3311                                    const struct rte_flow_item *geneve_item,
3312                                    struct rte_eth_dev *dev,
3313                                    struct rte_flow_error *error)
3314 {
3315         struct mlx5_priv *priv = dev->data->dev_private;
3316         struct mlx5_dev_ctx_shared *sh = priv->sh;
3317         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource;
3318         struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr;
3319         uint8_t data_max_supported =
3320                         hca_attr->max_geneve_tlv_option_data_len * 4;
3321         const struct rte_flow_item_geneve *geneve_spec;
3322         const struct rte_flow_item_geneve *geneve_mask;
3323         const struct rte_flow_item_geneve_opt *spec = item->spec;
3324         const struct rte_flow_item_geneve_opt *mask = item->mask;
3325         unsigned int i;
3326         unsigned int data_len;
3327         uint8_t tlv_option_len;
3328         uint16_t optlen_m, optlen_v;
3329         const struct rte_flow_item_geneve_opt full_mask = {
3330                 .option_class = RTE_BE16(0xffff),
3331                 .option_type = 0xff,
3332                 .option_len = 0x1f,
3333         };
3334
3335         if (!mask)
3336                 mask = &rte_flow_item_geneve_opt_mask;
3337         if (!spec)
3338                 return rte_flow_error_set
3339                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3340                         "Geneve TLV opt class/type/length must be specified");
3341         if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK)
3342                 return rte_flow_error_set
3343                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3344                         "Geneve TLV opt length exceeds the limit (31)");
3345         /* Check if class type and length masks are full. */
3346         if (full_mask.option_class != mask->option_class ||
3347             full_mask.option_type != mask->option_type ||
3348             full_mask.option_len != (mask->option_len & full_mask.option_len))
3349                 return rte_flow_error_set
3350                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3351                         "Geneve TLV opt class/type/length masks must be full");
3352         /* Check if length is supported */
3353         if ((uint32_t)spec->option_len >
3354                         hca_attr->max_geneve_tlv_option_data_len)
3355                 return rte_flow_error_set
3356                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3357                         "Geneve TLV opt length not supported");
3358         if (hca_attr->max_geneve_tlv_options > 1)
3359                 DRV_LOG(DEBUG,
3360                         "max_geneve_tlv_options supports more than 1 option");
3361         /* Check GENEVE item preceding. */
3362         if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE))
3363                 return rte_flow_error_set
3364                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3365                         "Geneve opt item must be preceded with Geneve item");
3366         geneve_spec = geneve_item->spec;
3367         geneve_mask = geneve_item->mask ? geneve_item->mask :
3368                                           &rte_flow_item_geneve_mask;
3369         /* Check if GENEVE TLV option size doesn't exceed option length */
3370         if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 ||
3371                             geneve_spec->ver_opt_len_o_c_rsvd0)) {
3372                 tlv_option_len = spec->option_len & mask->option_len;
3373                 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0);
3374                 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v);
3375                 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0);
3376                 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m);
3377                 if ((optlen_v & optlen_m) <= tlv_option_len)
3378                         return rte_flow_error_set
3379                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3380                                  "GENEVE TLV option length exceeds optlen");
3381         }
3382         /* Check if length is 0 or data is 0. */
3383         if (spec->data == NULL || spec->option_len == 0)
3384                 return rte_flow_error_set
3385                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3386                         "Geneve TLV opt with zero data/length not supported");
3387         /* Check not all data & mask are 0. */
3388         data_len = spec->option_len * 4;
3389         if (mask->data == NULL) {
3390                 for (i = 0; i < data_len; i++)
3391                         if (spec->data[i])
3392                                 break;
3393                 if (i == data_len)
3394                         return rte_flow_error_set(error, ENOTSUP,
3395                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
3396                                 "Can't match on Geneve option data 0");
3397         } else {
3398                 for (i = 0; i < data_len; i++)
3399                         if (spec->data[i] & mask->data[i])
3400                                 break;
3401                 if (i == data_len)
3402                         return rte_flow_error_set(error, ENOTSUP,
3403                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
3404                                 "Can't match on Geneve option data and mask 0");
3405                 /* Check data mask supported. */
3406                 for (i = data_max_supported; i < data_len ; i++)
3407                         if (mask->data[i])
3408                                 return rte_flow_error_set(error, ENOTSUP,
3409                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
3410                                         "Data mask is of unsupported size");
3411         }
3412         /* Check GENEVE option is supported in NIC. */
3413         if (!hca_attr->geneve_tlv_opt)
3414                 return rte_flow_error_set
3415                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
3416                         "Geneve TLV opt not supported");
3417         /* Check if we already have geneve option with different type/class. */
3418         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
3419         geneve_opt_resource = sh->geneve_tlv_option_resource;
3420         if (geneve_opt_resource != NULL)
3421                 if (geneve_opt_resource->option_class != spec->option_class ||
3422                     geneve_opt_resource->option_type != spec->option_type ||
3423                     geneve_opt_resource->length != spec->option_len) {
3424                         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
3425                         return rte_flow_error_set(error, ENOTSUP,
3426                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
3427                                 "Only one Geneve TLV option supported");
3428                 }
3429         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
3430         return 0;
3431 }
3432
3433 /**
3434  * Validate MPLS item.
3435  *
3436  * @param[in] dev
3437  *   Pointer to the rte_eth_dev structure.
3438  * @param[in] item
3439  *   Item specification.
3440  * @param[in] item_flags
3441  *   Bit-fields that holds the items detected until now.
3442  * @param[in] prev_layer
3443  *   The protocol layer indicated in previous item.
3444  * @param[out] error
3445  *   Pointer to error structure.
3446  *
3447  * @return
3448  *   0 on success, a negative errno value otherwise and rte_errno is set.
3449  */
3450 int
3451 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
3452                              const struct rte_flow_item *item __rte_unused,
3453                              uint64_t item_flags __rte_unused,
3454                              uint64_t prev_layer __rte_unused,
3455                              struct rte_flow_error *error)
3456 {
3457 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
3458         const struct rte_flow_item_mpls *mask = item->mask;
3459         struct mlx5_priv *priv = dev->data->dev_private;
3460         int ret;
3461
3462         if (!priv->sh->dev_cap.mpls_en)
3463                 return rte_flow_error_set(error, ENOTSUP,
3464                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3465                                           "MPLS not supported or"
3466                                           " disabled in firmware"
3467                                           " configuration.");
3468         /* MPLS over UDP, GRE is allowed */
3469         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP |
3470                             MLX5_FLOW_LAYER_GRE |
3471                             MLX5_FLOW_LAYER_GRE_KEY)))
3472                 return rte_flow_error_set(error, EINVAL,
3473                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3474                                           "protocol filtering not compatible"
3475                                           " with MPLS layer");
3476         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
3477         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
3478             !(item_flags & MLX5_FLOW_LAYER_GRE))
3479                 return rte_flow_error_set(error, ENOTSUP,
3480                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3481                                           "multiple tunnel layers not"
3482                                           " supported");
3483         if (!mask)
3484                 mask = &rte_flow_item_mpls_mask;
3485         ret = mlx5_flow_item_acceptable
3486                 (item, (const uint8_t *)mask,
3487                  (const uint8_t *)&rte_flow_item_mpls_mask,
3488                  sizeof(struct rte_flow_item_mpls),
3489                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3490         if (ret < 0)
3491                 return ret;
3492         return 0;
3493 #else
3494         return rte_flow_error_set(error, ENOTSUP,
3495                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
3496                                   "MPLS is not supported by Verbs, please"
3497                                   " update.");
3498 #endif
3499 }
3500
3501 /**
3502  * Validate NVGRE item.
3503  *
3504  * @param[in] item
3505  *   Item specification.
3506  * @param[in] item_flags
3507  *   Bit flags to mark detected items.
3508  * @param[in] target_protocol
3509  *   The next protocol in the previous item.
3510  * @param[out] error
3511  *   Pointer to error structure.
3512  *
3513  * @return
3514  *   0 on success, a negative errno value otherwise and rte_errno is set.
3515  */
3516 int
3517 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
3518                               uint64_t item_flags,
3519                               uint8_t target_protocol,
3520                               struct rte_flow_error *error)
3521 {
3522         const struct rte_flow_item_nvgre *mask = item->mask;
3523         int ret;
3524
3525         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
3526                 return rte_flow_error_set(error, EINVAL,
3527                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3528                                           "protocol filtering not compatible"
3529                                           " with this GRE layer");
3530         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3531                 return rte_flow_error_set(error, ENOTSUP,
3532                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3533                                           "multiple tunnel layers not"
3534                                           " supported");
3535         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
3536                 return rte_flow_error_set(error, ENOTSUP,
3537                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3538                                           "L3 Layer is missing");
3539         if (!mask)
3540                 mask = &rte_flow_item_nvgre_mask;
3541         ret = mlx5_flow_item_acceptable
3542                 (item, (const uint8_t *)mask,
3543                  (const uint8_t *)&rte_flow_item_nvgre_mask,
3544                  sizeof(struct rte_flow_item_nvgre),
3545                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3546         if (ret < 0)
3547                 return ret;
3548         return 0;
3549 }
3550
3551 /**
3552  * Validate eCPRI item.
3553  *
3554  * @param[in] item
3555  *   Item specification.
3556  * @param[in] item_flags
3557  *   Bit-fields that holds the items detected until now.
3558  * @param[in] last_item
3559  *   Previous validated item in the pattern items.
3560  * @param[in] ether_type
3561  *   Type in the ethernet layer header (including dot1q).
3562  * @param[in] acc_mask
3563  *   Acceptable mask, if NULL default internal default mask
3564  *   will be used to check whether item fields are supported.
3565  * @param[out] error
3566  *   Pointer to error structure.
3567  *
3568  * @return
3569  *   0 on success, a negative errno value otherwise and rte_errno is set.
3570  */
3571 int
3572 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
3573                               uint64_t item_flags,
3574                               uint64_t last_item,
3575                               uint16_t ether_type,
3576                               const struct rte_flow_item_ecpri *acc_mask,
3577                               struct rte_flow_error *error)
3578 {
3579         const struct rte_flow_item_ecpri *mask = item->mask;
3580         const struct rte_flow_item_ecpri nic_mask = {
3581                 .hdr = {
3582                         .common = {
3583                                 .u32 =
3584                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
3585                                         .type = 0xFF,
3586                                         }).u32),
3587                         },
3588                         .dummy[0] = 0xFFFFFFFF,
3589                 },
3590         };
3591         const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
3592                                         MLX5_FLOW_LAYER_OUTER_VLAN);
3593         struct rte_flow_item_ecpri mask_lo;
3594
3595         if (!(last_item & outer_l2_vlan) &&
3596             last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP)
3597                 return rte_flow_error_set(error, EINVAL,
3598                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3599                                           "eCPRI can only follow L2/VLAN layer or UDP layer");
3600         if ((last_item & outer_l2_vlan) && ether_type &&
3601             ether_type != RTE_ETHER_TYPE_ECPRI)
3602                 return rte_flow_error_set(error, EINVAL,
3603                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3604                                           "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE");
3605         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3606                 return rte_flow_error_set(error, EINVAL,
3607                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3608                                           "eCPRI with tunnel is not supported right now");
3609         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
3610                 return rte_flow_error_set(error, ENOTSUP,
3611                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3612                                           "multiple L3 layers not supported");
3613         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
3614                 return rte_flow_error_set(error, EINVAL,
3615                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3616                                           "eCPRI cannot coexist with a TCP layer");
3617         /* In specification, eCPRI could be over UDP layer. */
3618         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
3619                 return rte_flow_error_set(error, EINVAL,
3620                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3621                                           "eCPRI over UDP layer is not yet supported right now");
3622         /* Mask for type field in common header could be zero. */
3623         if (!mask)
3624                 mask = &rte_flow_item_ecpri_mask;
3625         mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
3626         /* Input mask is in big-endian format. */
3627         if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
3628                 return rte_flow_error_set(error, EINVAL,
3629                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3630                                           "partial mask is not supported for protocol");
3631         else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
3632                 return rte_flow_error_set(error, EINVAL,
3633                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3634                                           "message header mask must be after a type mask");
3635         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
3636                                          acc_mask ? (const uint8_t *)acc_mask
3637                                                   : (const uint8_t *)&nic_mask,
3638                                          sizeof(struct rte_flow_item_ecpri),
3639                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3640 }
3641
3642 static int
3643 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
3644                    const struct rte_flow_attr *attr __rte_unused,
3645                    const struct rte_flow_item items[] __rte_unused,
3646                    const struct rte_flow_action actions[] __rte_unused,
3647                    bool external __rte_unused,
3648                    int hairpin __rte_unused,
3649                    struct rte_flow_error *error)
3650 {
3651         return rte_flow_error_set(error, ENOTSUP,
3652                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3653 }
3654
3655 static struct mlx5_flow *
3656 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
3657                   const struct rte_flow_attr *attr __rte_unused,
3658                   const struct rte_flow_item items[] __rte_unused,
3659                   const struct rte_flow_action actions[] __rte_unused,
3660                   struct rte_flow_error *error)
3661 {
3662         rte_flow_error_set(error, ENOTSUP,
3663                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3664         return NULL;
3665 }
3666
3667 static int
3668 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
3669                     struct mlx5_flow *dev_flow __rte_unused,
3670                     const struct rte_flow_attr *attr __rte_unused,
3671                     const struct rte_flow_item items[] __rte_unused,
3672                     const struct rte_flow_action actions[] __rte_unused,
3673                     struct rte_flow_error *error)
3674 {
3675         return rte_flow_error_set(error, ENOTSUP,
3676                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3677 }
3678
3679 static int
3680 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
3681                 struct rte_flow *flow __rte_unused,
3682                 struct rte_flow_error *error)
3683 {
3684         return rte_flow_error_set(error, ENOTSUP,
3685                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3686 }
3687
3688 static void
3689 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
3690                  struct rte_flow *flow __rte_unused)
3691 {
3692 }
3693
3694 static void
3695 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
3696                   struct rte_flow *flow __rte_unused)
3697 {
3698 }
3699
3700 static int
3701 flow_null_query(struct rte_eth_dev *dev __rte_unused,
3702                 struct rte_flow *flow __rte_unused,
3703                 const struct rte_flow_action *actions __rte_unused,
3704                 void *data __rte_unused,
3705                 struct rte_flow_error *error)
3706 {
3707         return rte_flow_error_set(error, ENOTSUP,
3708                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3709 }
3710
3711 static int
3712 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused,
3713                       uint32_t domains __rte_unused,
3714                       uint32_t flags __rte_unused)
3715 {
3716         return 0;
3717 }
3718
3719 /* Void driver to protect from null pointer reference. */
3720 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
3721         .validate = flow_null_validate,
3722         .prepare = flow_null_prepare,
3723         .translate = flow_null_translate,
3724         .apply = flow_null_apply,
3725         .remove = flow_null_remove,
3726         .destroy = flow_null_destroy,
3727         .query = flow_null_query,
3728         .sync_domain = flow_null_sync_domain,
3729 };
3730
3731 /**
3732  * Select flow driver type according to flow attributes and device
3733  * configuration.
3734  *
3735  * @param[in] dev
3736  *   Pointer to the dev structure.
3737  * @param[in] attr
3738  *   Pointer to the flow attributes.
3739  *
3740  * @return
3741  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
3742  */
3743 static enum mlx5_flow_drv_type
3744 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
3745 {
3746         struct mlx5_priv *priv = dev->data->dev_private;
3747         /* The OS can determine first a specific flow type (DV, VERBS) */
3748         enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
3749
3750         if (type != MLX5_FLOW_TYPE_MAX)
3751                 return type;
3752         /*
3753          * Currently when dv_flow_en == 2, only HW steering engine is
3754          * supported. New engines can also be chosen here if ready.
3755          */
3756         if (priv->sh->config.dv_flow_en == 2)
3757                 return MLX5_FLOW_TYPE_HW;
3758         /* If no OS specific type - continue with DV/VERBS selection */
3759         if (attr->transfer && priv->sh->config.dv_esw_en)
3760                 type = MLX5_FLOW_TYPE_DV;
3761         if (!attr->transfer)
3762                 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
3763                                                      MLX5_FLOW_TYPE_VERBS;
3764         return type;
3765 }
3766
3767 #define flow_get_drv_ops(type) flow_drv_ops[type]
3768
3769 /**
3770  * Flow driver validation API. This abstracts calling driver specific functions.
3771  * The type of flow driver is determined according to flow attributes.
3772  *
3773  * @param[in] dev
3774  *   Pointer to the dev structure.
3775  * @param[in] attr
3776  *   Pointer to the flow attributes.
3777  * @param[in] items
3778  *   Pointer to the list of items.
3779  * @param[in] actions
3780  *   Pointer to the list of actions.
3781  * @param[in] external
3782  *   This flow rule is created by request external to PMD.
3783  * @param[in] hairpin
3784  *   Number of hairpin TX actions, 0 means classic flow.
3785  * @param[out] error
3786  *   Pointer to the error structure.
3787  *
3788  * @return
3789  *   0 on success, a negative errno value otherwise and rte_errno is set.
3790  */
3791 static inline int
3792 flow_drv_validate(struct rte_eth_dev *dev,
3793                   const struct rte_flow_attr *attr,
3794                   const struct rte_flow_item items[],
3795                   const struct rte_flow_action actions[],
3796                   bool external, int hairpin, struct rte_flow_error *error)
3797 {
3798         const struct mlx5_flow_driver_ops *fops;
3799         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
3800
3801         fops = flow_get_drv_ops(type);
3802         return fops->validate(dev, attr, items, actions, external,
3803                               hairpin, error);
3804 }
3805
3806 /**
3807  * Flow driver preparation API. This abstracts calling driver specific
3808  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3809  * calculates the size of memory required for device flow, allocates the memory,
3810  * initializes the device flow and returns the pointer.
3811  *
3812  * @note
3813  *   This function initializes device flow structure such as dv or verbs in
3814  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
3815  *   rest. For example, adding returning device flow to flow->dev_flow list and
3816  *   setting backward reference to the flow should be done out of this function.
3817  *   layers field is not filled either.
3818  *
3819  * @param[in] dev
3820  *   Pointer to the dev structure.
3821  * @param[in] attr
3822  *   Pointer to the flow attributes.
3823  * @param[in] items
3824  *   Pointer to the list of items.
3825  * @param[in] actions
3826  *   Pointer to the list of actions.
3827  * @param[in] flow_idx
3828  *   This memory pool index to the flow.
3829  * @param[out] error
3830  *   Pointer to the error structure.
3831  *
3832  * @return
3833  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
3834  */
3835 static inline struct mlx5_flow *
3836 flow_drv_prepare(struct rte_eth_dev *dev,
3837                  const struct rte_flow *flow,
3838                  const struct rte_flow_attr *attr,
3839                  const struct rte_flow_item items[],
3840                  const struct rte_flow_action actions[],
3841                  uint32_t flow_idx,
3842                  struct rte_flow_error *error)
3843 {
3844         const struct mlx5_flow_driver_ops *fops;
3845         enum mlx5_flow_drv_type type = flow->drv_type;
3846         struct mlx5_flow *mlx5_flow = NULL;
3847
3848         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3849         fops = flow_get_drv_ops(type);
3850         mlx5_flow = fops->prepare(dev, attr, items, actions, error);
3851         if (mlx5_flow)
3852                 mlx5_flow->flow_idx = flow_idx;
3853         return mlx5_flow;
3854 }
3855
3856 /**
3857  * Flow driver translation API. This abstracts calling driver specific
3858  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3859  * translates a generic flow into a driver flow. flow_drv_prepare() must
3860  * precede.
3861  *
3862  * @note
3863  *   dev_flow->layers could be filled as a result of parsing during translation
3864  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
3865  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
3866  *   flow->actions could be overwritten even though all the expanded dev_flows
3867  *   have the same actions.
3868  *
3869  * @param[in] dev
3870  *   Pointer to the rte dev structure.
3871  * @param[in, out] dev_flow
3872  *   Pointer to the mlx5 flow.
3873  * @param[in] attr
3874  *   Pointer to the flow attributes.
3875  * @param[in] items
3876  *   Pointer to the list of items.
3877  * @param[in] actions
3878  *   Pointer to the list of actions.
3879  * @param[out] error
3880  *   Pointer to the error structure.
3881  *
3882  * @return
3883  *   0 on success, a negative errno value otherwise and rte_errno is set.
3884  */
3885 static inline int
3886 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3887                    const struct rte_flow_attr *attr,
3888                    const struct rte_flow_item items[],
3889                    const struct rte_flow_action actions[],
3890                    struct rte_flow_error *error)
3891 {
3892         const struct mlx5_flow_driver_ops *fops;
3893         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
3894
3895         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3896         fops = flow_get_drv_ops(type);
3897         return fops->translate(dev, dev_flow, attr, items, actions, error);
3898 }
3899
3900 /**
3901  * Flow driver apply API. This abstracts calling driver specific functions.
3902  * Parent flow (rte_flow) should have driver type (drv_type). It applies
3903  * translated driver flows on to device. flow_drv_translate() must precede.
3904  *
3905  * @param[in] dev
3906  *   Pointer to Ethernet device structure.
3907  * @param[in, out] flow
3908  *   Pointer to flow structure.
3909  * @param[out] error
3910  *   Pointer to error structure.
3911  *
3912  * @return
3913  *   0 on success, a negative errno value otherwise and rte_errno is set.
3914  */
3915 static inline int
3916 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3917                struct rte_flow_error *error)
3918 {
3919         const struct mlx5_flow_driver_ops *fops;
3920         enum mlx5_flow_drv_type type = flow->drv_type;
3921
3922         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3923         fops = flow_get_drv_ops(type);
3924         return fops->apply(dev, flow, error);
3925 }
3926
3927 /**
3928  * Flow driver destroy API. This abstracts calling driver specific functions.
3929  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3930  * on device and releases resources of the flow.
3931  *
3932  * @param[in] dev
3933  *   Pointer to Ethernet device.
3934  * @param[in, out] flow
3935  *   Pointer to flow structure.
3936  */
3937 static inline void
3938 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3939 {
3940         const struct mlx5_flow_driver_ops *fops;
3941         enum mlx5_flow_drv_type type = flow->drv_type;
3942
3943         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3944         fops = flow_get_drv_ops(type);
3945         fops->destroy(dev, flow);
3946 }
3947
3948 /**
3949  * Flow driver find RSS policy tbl API. This abstracts calling driver
3950  * specific functions. Parent flow (rte_flow) should have driver
3951  * type (drv_type). It will find the RSS policy table that has the rss_desc.
3952  *
3953  * @param[in] dev
3954  *   Pointer to Ethernet device.
3955  * @param[in, out] flow
3956  *   Pointer to flow structure.
3957  * @param[in] policy
3958  *   Pointer to meter policy table.
3959  * @param[in] rss_desc
3960  *   Pointer to rss_desc
3961  */
3962 static struct mlx5_flow_meter_sub_policy *
3963 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
3964                 struct rte_flow *flow,
3965                 struct mlx5_flow_meter_policy *policy,
3966                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
3967 {
3968         const struct mlx5_flow_driver_ops *fops;
3969         enum mlx5_flow_drv_type type = flow->drv_type;
3970
3971         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3972         fops = flow_get_drv_ops(type);
3973         return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc);
3974 }
3975
3976 /**
3977  * Flow driver color tag rule API. This abstracts calling driver
3978  * specific functions. Parent flow (rte_flow) should have driver
3979  * type (drv_type). It will create the color tag rules in hierarchy meter.
3980  *
3981  * @param[in] dev
3982  *   Pointer to Ethernet device.
3983  * @param[in, out] flow
3984  *   Pointer to flow structure.
3985  * @param[in] fm
3986  *   Pointer to flow meter structure.
3987  * @param[in] src_port
3988  *   The src port this extra rule should use.
3989  * @param[in] item
3990  *   The src port id match item.
3991  * @param[out] error
3992  *   Pointer to error structure.
3993  */
3994 static int
3995 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev,
3996                 struct rte_flow *flow,
3997                 struct mlx5_flow_meter_info *fm,
3998                 int32_t src_port,
3999                 const struct rte_flow_item *item,
4000                 struct rte_flow_error *error)
4001 {
4002         const struct mlx5_flow_driver_ops *fops;
4003         enum mlx5_flow_drv_type type = flow->drv_type;
4004
4005         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
4006         fops = flow_get_drv_ops(type);
4007         return fops->meter_hierarchy_rule_create(dev, fm,
4008                                                 src_port, item, error);
4009 }
4010
4011 /**
4012  * Get RSS action from the action list.
4013  *
4014  * @param[in] dev
4015  *   Pointer to Ethernet device.
4016  * @param[in] actions
4017  *   Pointer to the list of actions.
4018  * @param[in] flow
4019  *   Parent flow structure pointer.
4020  *
4021  * @return
4022  *   Pointer to the RSS action if exist, else return NULL.
4023  */
4024 static const struct rte_flow_action_rss*
4025 flow_get_rss_action(struct rte_eth_dev *dev,
4026                     const struct rte_flow_action actions[])
4027 {
4028         struct mlx5_priv *priv = dev->data->dev_private;
4029         const struct rte_flow_action_rss *rss = NULL;
4030         struct mlx5_meter_policy_action_container *acg;
4031         struct mlx5_meter_policy_action_container *acy;
4032
4033         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4034                 switch (actions->type) {
4035                 case RTE_FLOW_ACTION_TYPE_RSS:
4036                         rss = actions->conf;
4037                         break;
4038                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
4039                 {
4040                         const struct rte_flow_action_sample *sample =
4041                                                                 actions->conf;
4042                         const struct rte_flow_action *act = sample->actions;
4043                         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++)
4044                                 if (act->type == RTE_FLOW_ACTION_TYPE_RSS)
4045                                         rss = act->conf;
4046                         break;
4047                 }
4048                 case RTE_FLOW_ACTION_TYPE_METER:
4049                 {
4050                         uint32_t mtr_idx;
4051                         struct mlx5_flow_meter_info *fm;
4052                         struct mlx5_flow_meter_policy *policy;
4053                         const struct rte_flow_action_meter *mtr = actions->conf;
4054
4055                         fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx);
4056                         if (fm && !fm->def_policy) {
4057                                 policy = mlx5_flow_meter_policy_find(dev,
4058                                                 fm->policy_id, NULL);
4059                                 MLX5_ASSERT(policy);
4060                                 if (policy->is_hierarchy) {
4061                                         policy =
4062                                 mlx5_flow_meter_hierarchy_get_final_policy(dev,
4063                                                                         policy);
4064                                         if (!policy)
4065                                                 return NULL;
4066                                 }
4067                                 if (policy->is_rss) {
4068                                         acg =
4069                                         &policy->act_cnt[RTE_COLOR_GREEN];
4070                                         acy =
4071                                         &policy->act_cnt[RTE_COLOR_YELLOW];
4072                                         if (acg->fate_action ==
4073                                             MLX5_FLOW_FATE_SHARED_RSS)
4074                                                 rss = acg->rss->conf;
4075                                         else if (acy->fate_action ==
4076                                                  MLX5_FLOW_FATE_SHARED_RSS)
4077                                                 rss = acy->rss->conf;
4078                                 }
4079                         }
4080                         break;
4081                 }
4082                 default:
4083                         break;
4084                 }
4085         }
4086         return rss;
4087 }
4088
4089 /**
4090  * Get ASO age action by index.
4091  *
4092  * @param[in] dev
4093  *   Pointer to the Ethernet device structure.
4094  * @param[in] age_idx
4095  *   Index to the ASO age action.
4096  *
4097  * @return
4098  *   The specified ASO age action.
4099  */
4100 struct mlx5_aso_age_action*
4101 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx)
4102 {
4103         uint16_t pool_idx = age_idx & UINT16_MAX;
4104         uint16_t offset = (age_idx >> 16) & UINT16_MAX;
4105         struct mlx5_priv *priv = dev->data->dev_private;
4106         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
4107         struct mlx5_aso_age_pool *pool;
4108
4109         rte_rwlock_read_lock(&mng->resize_rwl);
4110         pool = mng->pools[pool_idx];
4111         rte_rwlock_read_unlock(&mng->resize_rwl);
4112         return &pool->actions[offset - 1];
4113 }
4114
4115 /* maps indirect action to translated direct in some actions array */
4116 struct mlx5_translated_action_handle {
4117         struct rte_flow_action_handle *action; /**< Indirect action handle. */
4118         int index; /**< Index in related array of rte_flow_action. */
4119 };
4120
4121 /**
4122  * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related
4123  * direct action if translation possible.
4124  * This functionality used to run same execution path for both direct and
4125  * indirect actions on flow create. All necessary preparations for indirect
4126  * action handling should be performed on *handle* actions list returned
4127  * from this call.
4128  *
4129  * @param[in] dev
4130  *   Pointer to Ethernet device.
4131  * @param[in] actions
4132  *   List of actions to translate.
4133  * @param[out] handle
4134  *   List to store translated indirect action object handles.
4135  * @param[in, out] indir_n
4136  *   Size of *handle* array. On return should be updated with number of
4137  *   indirect actions retrieved from the *actions* list.
4138  * @param[out] translated_actions
4139  *   List of actions where all indirect actions were translated to direct
4140  *   if possible. NULL if no translation took place.
4141  * @param[out] error
4142  *   Pointer to the error structure.
4143  *
4144  * @return
4145  *   0 on success, a negative errno value otherwise and rte_errno is set.
4146  */
4147 static int
4148 flow_action_handles_translate(struct rte_eth_dev *dev,
4149                               const struct rte_flow_action actions[],
4150                               struct mlx5_translated_action_handle *handle,
4151                               int *indir_n,
4152                               struct rte_flow_action **translated_actions,
4153                               struct rte_flow_error *error)
4154 {
4155         struct mlx5_priv *priv = dev->data->dev_private;
4156         struct rte_flow_action *translated = NULL;
4157         size_t actions_size;
4158         int n;
4159         int copied_n = 0;
4160         struct mlx5_translated_action_handle *handle_end = NULL;
4161
4162         for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
4163                 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT)
4164                         continue;
4165                 if (copied_n == *indir_n) {
4166                         return rte_flow_error_set
4167                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
4168                                  NULL, "too many shared actions");
4169                 }
4170                 rte_memcpy(&handle[copied_n].action, &actions[n].conf,
4171                            sizeof(actions[n].conf));
4172                 handle[copied_n].index = n;
4173                 copied_n++;
4174         }
4175         n++;
4176         *indir_n = copied_n;
4177         if (!copied_n)
4178                 return 0;
4179         actions_size = sizeof(struct rte_flow_action) * n;
4180         translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
4181         if (!translated) {
4182                 rte_errno = ENOMEM;
4183                 return -ENOMEM;
4184         }
4185         memcpy(translated, actions, actions_size);
4186         for (handle_end = handle + copied_n; handle < handle_end; handle++) {
4187                 struct mlx5_shared_action_rss *shared_rss;
4188                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
4189                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
4190                 uint32_t idx = act_idx &
4191                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
4192
4193                 switch (type) {
4194                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
4195                         shared_rss = mlx5_ipool_get
4196                           (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
4197                         translated[handle->index].type =
4198                                 RTE_FLOW_ACTION_TYPE_RSS;
4199                         translated[handle->index].conf =
4200                                 &shared_rss->origin;
4201                         break;
4202                 case MLX5_INDIRECT_ACTION_TYPE_COUNT:
4203                         translated[handle->index].type =
4204                                                 (enum rte_flow_action_type)
4205                                                 MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
4206                         translated[handle->index].conf = (void *)(uintptr_t)idx;
4207                         break;
4208                 case MLX5_INDIRECT_ACTION_TYPE_AGE:
4209                         if (priv->sh->flow_hit_aso_en) {
4210                                 translated[handle->index].type =
4211                                         (enum rte_flow_action_type)
4212                                         MLX5_RTE_FLOW_ACTION_TYPE_AGE;
4213                                 translated[handle->index].conf =
4214                                                          (void *)(uintptr_t)idx;
4215                                 break;
4216                         }
4217                         /* Fall-through */
4218                 case MLX5_INDIRECT_ACTION_TYPE_CT:
4219                         if (priv->sh->ct_aso_en) {
4220                                 translated[handle->index].type =
4221                                         RTE_FLOW_ACTION_TYPE_CONNTRACK;
4222                                 translated[handle->index].conf =
4223                                                          (void *)(uintptr_t)idx;
4224                                 break;
4225                         }
4226                         /* Fall-through */
4227                 default:
4228                         mlx5_free(translated);
4229                         return rte_flow_error_set
4230                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
4231                                  NULL, "invalid indirect action type");
4232                 }
4233         }
4234         *translated_actions = translated;
4235         return 0;
4236 }
4237
4238 /**
4239  * Get Shared RSS action from the action list.
4240  *
4241  * @param[in] dev
4242  *   Pointer to Ethernet device.
4243  * @param[in] shared
4244  *   Pointer to the list of actions.
4245  * @param[in] shared_n
4246  *   Actions list length.
4247  *
4248  * @return
4249  *   The MLX5 RSS action ID if exists, otherwise return 0.
4250  */
4251 static uint32_t
4252 flow_get_shared_rss_action(struct rte_eth_dev *dev,
4253                            struct mlx5_translated_action_handle *handle,
4254                            int shared_n)
4255 {
4256         struct mlx5_translated_action_handle *handle_end;
4257         struct mlx5_priv *priv = dev->data->dev_private;
4258         struct mlx5_shared_action_rss *shared_rss;
4259
4260
4261         for (handle_end = handle + shared_n; handle < handle_end; handle++) {
4262                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
4263                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
4264                 uint32_t idx = act_idx &
4265                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
4266                 switch (type) {
4267                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
4268                         shared_rss = mlx5_ipool_get
4269                                 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
4270                                                                            idx);
4271                         __atomic_add_fetch(&shared_rss->refcnt, 1,
4272                                            __ATOMIC_RELAXED);
4273                         return idx;
4274                 default:
4275                         break;
4276                 }
4277         }
4278         return 0;
4279 }
4280
4281 static unsigned int
4282 find_graph_root(uint32_t rss_level)
4283 {
4284         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
4285                                MLX5_EXPANSION_ROOT_OUTER;
4286 }
4287
4288 /**
4289  *  Get layer flags from the prefix flow.
4290  *
4291  *  Some flows may be split to several subflows, the prefix subflow gets the
4292  *  match items and the suffix sub flow gets the actions.
4293  *  Some actions need the user defined match item flags to get the detail for
4294  *  the action.
4295  *  This function helps the suffix flow to get the item layer flags from prefix
4296  *  subflow.
4297  *
4298  * @param[in] dev_flow
4299  *   Pointer the created prefix subflow.
4300  *
4301  * @return
4302  *   The layers get from prefix subflow.
4303  */
4304 static inline uint64_t
4305 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
4306 {
4307         uint64_t layers = 0;
4308
4309         /*
4310          * Layers bits could be localization, but usually the compiler will
4311          * help to do the optimization work for source code.
4312          * If no decap actions, use the layers directly.
4313          */
4314         if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
4315                 return dev_flow->handle->layers;
4316         /* Convert L3 layers with decap action. */
4317         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
4318                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
4319         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
4320                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
4321         /* Convert L4 layers with decap action.  */
4322         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
4323                 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
4324         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
4325                 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
4326         return layers;
4327 }
4328
4329 /**
4330  * Get metadata split action information.
4331  *
4332  * @param[in] actions
4333  *   Pointer to the list of actions.
4334  * @param[out] qrss
4335  *   Pointer to the return pointer.
4336  * @param[out] qrss_type
4337  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
4338  *   if no QUEUE/RSS is found.
4339  * @param[out] encap_idx
4340  *   Pointer to the index of the encap action if exists, otherwise the last
4341  *   action index.
4342  *
4343  * @return
4344  *   Total number of actions.
4345  */
4346 static int
4347 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
4348                                        const struct rte_flow_action **qrss,
4349                                        int *encap_idx)
4350 {
4351         const struct rte_flow_action_raw_encap *raw_encap;
4352         int actions_n = 0;
4353         int raw_decap_idx = -1;
4354
4355         *encap_idx = -1;
4356         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4357                 switch (actions->type) {
4358                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4359                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4360                         *encap_idx = actions_n;
4361                         break;
4362                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4363                         raw_decap_idx = actions_n;
4364                         break;
4365                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4366                         raw_encap = actions->conf;
4367                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4368                                 *encap_idx = raw_decap_idx != -1 ?
4369                                                       raw_decap_idx : actions_n;
4370                         break;
4371                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4372                 case RTE_FLOW_ACTION_TYPE_RSS:
4373                         *qrss = actions;
4374                         break;
4375                 default:
4376                         break;
4377                 }
4378                 actions_n++;
4379         }
4380         if (*encap_idx == -1)
4381                 *encap_idx = actions_n;
4382         /* Count RTE_FLOW_ACTION_TYPE_END. */
4383         return actions_n + 1;
4384 }
4385
4386 /**
4387  * Check if the action will change packet.
4388  *
4389  * @param dev
4390  *   Pointer to Ethernet device.
4391  * @param[in] type
4392  *   action type.
4393  *
4394  * @return
4395  *   true if action will change packet, false otherwise.
4396  */
4397 static bool flow_check_modify_action_type(struct rte_eth_dev *dev,
4398                                           enum rte_flow_action_type type)
4399 {
4400         struct mlx5_priv *priv = dev->data->dev_private;
4401
4402         switch (type) {
4403         case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
4404         case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
4405         case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
4406         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
4407         case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
4408         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
4409         case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
4410         case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
4411         case RTE_FLOW_ACTION_TYPE_DEC_TTL:
4412         case RTE_FLOW_ACTION_TYPE_SET_TTL:
4413         case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
4414         case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
4415         case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
4416         case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
4417         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
4418         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
4419         case RTE_FLOW_ACTION_TYPE_SET_META:
4420         case RTE_FLOW_ACTION_TYPE_SET_TAG:
4421         case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
4422         case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4423         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4424         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4425         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4426         case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4427         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4428         case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4429         case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4430         case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4431         case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
4432                 return true;
4433         case RTE_FLOW_ACTION_TYPE_FLAG:
4434         case RTE_FLOW_ACTION_TYPE_MARK:
4435                 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
4436                         return true;
4437                 else
4438                         return false;
4439         default:
4440                 return false;
4441         }
4442 }
4443
4444 /**
4445  * Check meter action from the action list.
4446  *
4447  * @param dev
4448  *   Pointer to Ethernet device.
4449  * @param[in] actions
4450  *   Pointer to the list of actions.
4451  * @param[out] has_mtr
4452  *   Pointer to the meter exist flag.
4453  * @param[out] has_modify
4454  *   Pointer to the flag showing there's packet change action.
4455  * @param[out] meter_id
4456  *   Pointer to the meter id.
4457  *
4458  * @return
4459  *   Total number of actions.
4460  */
4461 static int
4462 flow_check_meter_action(struct rte_eth_dev *dev,
4463                         const struct rte_flow_action actions[],
4464                         bool *has_mtr, bool *has_modify, uint32_t *meter_id)
4465 {
4466         const struct rte_flow_action_meter *mtr = NULL;
4467         int actions_n = 0;
4468
4469         MLX5_ASSERT(has_mtr);
4470         *has_mtr = false;
4471         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4472                 switch (actions->type) {
4473                 case RTE_FLOW_ACTION_TYPE_METER:
4474                         mtr = actions->conf;
4475                         *meter_id = mtr->mtr_id;
4476                         *has_mtr = true;
4477                         break;
4478                 default:
4479                         break;
4480                 }
4481                 if (!*has_mtr)
4482                         *has_modify |= flow_check_modify_action_type(dev,
4483                                                                 actions->type);
4484                 actions_n++;
4485         }
4486         /* Count RTE_FLOW_ACTION_TYPE_END. */
4487         return actions_n + 1;
4488 }
4489
4490 /**
4491  * Check if the flow should be split due to hairpin.
4492  * The reason for the split is that in current HW we can't
4493  * support encap and push-vlan on Rx, so if a flow contains
4494  * these actions we move it to Tx.
4495  *
4496  * @param dev
4497  *   Pointer to Ethernet device.
4498  * @param[in] attr
4499  *   Flow rule attributes.
4500  * @param[in] actions
4501  *   Associated actions (list terminated by the END action).
4502  *
4503  * @return
4504  *   > 0 the number of actions and the flow should be split,
4505  *   0 when no split required.
4506  */
4507 static int
4508 flow_check_hairpin_split(struct rte_eth_dev *dev,
4509                          const struct rte_flow_attr *attr,
4510                          const struct rte_flow_action actions[])
4511 {
4512         int queue_action = 0;
4513         int action_n = 0;
4514         int split = 0;
4515         const struct rte_flow_action_queue *queue;
4516         const struct rte_flow_action_rss *rss;
4517         const struct rte_flow_action_raw_encap *raw_encap;
4518         const struct rte_eth_hairpin_conf *conf;
4519
4520         if (!attr->ingress)
4521                 return 0;
4522         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4523                 switch (actions->type) {
4524                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4525                         queue = actions->conf;
4526                         if (queue == NULL)
4527                                 return 0;
4528                         conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
4529                         if (conf == NULL || conf->tx_explicit != 0)
4530                                 return 0;
4531                         queue_action = 1;
4532                         action_n++;
4533                         break;
4534                 case RTE_FLOW_ACTION_TYPE_RSS:
4535                         rss = actions->conf;
4536                         if (rss == NULL || rss->queue_num == 0)
4537                                 return 0;
4538                         conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
4539                         if (conf == NULL || conf->tx_explicit != 0)
4540                                 return 0;
4541                         queue_action = 1;
4542                         action_n++;
4543                         break;
4544                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4545                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4546                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4547                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4548                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4549                         split++;
4550                         action_n++;
4551                         break;
4552                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4553                         raw_encap = actions->conf;
4554                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4555                                 split++;
4556                         action_n++;
4557                         break;
4558                 default:
4559                         action_n++;
4560                         break;
4561                 }
4562         }
4563         if (split && queue_action)
4564                 return action_n;
4565         return 0;
4566 }
4567
4568 /* Declare flow create/destroy prototype in advance. */
4569 static uint32_t
4570 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
4571                  const struct rte_flow_attr *attr,
4572                  const struct rte_flow_item items[],
4573                  const struct rte_flow_action actions[],
4574                  bool external, struct rte_flow_error *error);
4575
4576 static void
4577 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
4578                   uint32_t flow_idx);
4579
4580 int
4581 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused,
4582                       struct mlx5_list_entry *entry, void *cb_ctx)
4583 {
4584         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4585         struct mlx5_flow_mreg_copy_resource *mcp_res =
4586                                container_of(entry, typeof(*mcp_res), hlist_ent);
4587
4588         return mcp_res->mark_id != *(uint32_t *)(ctx->data);
4589 }
4590
4591 struct mlx5_list_entry *
4592 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx)
4593 {
4594         struct rte_eth_dev *dev = tool_ctx;
4595         struct mlx5_priv *priv = dev->data->dev_private;
4596         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4597         struct mlx5_flow_mreg_copy_resource *mcp_res;
4598         struct rte_flow_error *error = ctx->error;
4599         uint32_t idx = 0;
4600         int ret;
4601         uint32_t mark_id = *(uint32_t *)(ctx->data);
4602         struct rte_flow_attr attr = {
4603                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4604                 .ingress = 1,
4605         };
4606         struct mlx5_rte_flow_item_tag tag_spec = {
4607                 .data = mark_id,
4608         };
4609         struct rte_flow_item items[] = {
4610                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
4611         };
4612         struct rte_flow_action_mark ftag = {
4613                 .id = mark_id,
4614         };
4615         struct mlx5_flow_action_copy_mreg cp_mreg = {
4616                 .dst = REG_B,
4617                 .src = REG_NON,
4618         };
4619         struct rte_flow_action_jump jump = {
4620                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4621         };
4622         struct rte_flow_action actions[] = {
4623                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
4624         };
4625
4626         /* Fill the register fields in the flow. */
4627         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
4628         if (ret < 0)
4629                 return NULL;
4630         tag_spec.id = ret;
4631         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4632         if (ret < 0)
4633                 return NULL;
4634         cp_mreg.src = ret;
4635         /* Provide the full width of FLAG specific value. */
4636         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
4637                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
4638         /* Build a new flow. */
4639         if (mark_id != MLX5_DEFAULT_COPY_ID) {
4640                 items[0] = (struct rte_flow_item){
4641                         .type = (enum rte_flow_item_type)
4642                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4643                         .spec = &tag_spec,
4644                 };
4645                 items[1] = (struct rte_flow_item){
4646                         .type = RTE_FLOW_ITEM_TYPE_END,
4647                 };
4648                 actions[0] = (struct rte_flow_action){
4649                         .type = (enum rte_flow_action_type)
4650                                 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
4651                         .conf = &ftag,
4652                 };
4653                 actions[1] = (struct rte_flow_action){
4654                         .type = (enum rte_flow_action_type)
4655                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4656                         .conf = &cp_mreg,
4657                 };
4658                 actions[2] = (struct rte_flow_action){
4659                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4660                         .conf = &jump,
4661                 };
4662                 actions[3] = (struct rte_flow_action){
4663                         .type = RTE_FLOW_ACTION_TYPE_END,
4664                 };
4665         } else {
4666                 /* Default rule, wildcard match. */
4667                 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR;
4668                 items[0] = (struct rte_flow_item){
4669                         .type = RTE_FLOW_ITEM_TYPE_END,
4670                 };
4671                 actions[0] = (struct rte_flow_action){
4672                         .type = (enum rte_flow_action_type)
4673                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4674                         .conf = &cp_mreg,
4675                 };
4676                 actions[1] = (struct rte_flow_action){
4677                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4678                         .conf = &jump,
4679                 };
4680                 actions[2] = (struct rte_flow_action){
4681                         .type = RTE_FLOW_ACTION_TYPE_END,
4682                 };
4683         }
4684         /* Build a new entry. */
4685         mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
4686         if (!mcp_res) {
4687                 rte_errno = ENOMEM;
4688                 return NULL;
4689         }
4690         mcp_res->idx = idx;
4691         mcp_res->mark_id = mark_id;
4692         /*
4693          * The copy Flows are not included in any list. There
4694          * ones are referenced from other Flows and can not
4695          * be applied, removed, deleted in arbitrary order
4696          * by list traversing.
4697          */
4698         mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP,
4699                                         &attr, items, actions, false, error);
4700         if (!mcp_res->rix_flow) {
4701                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
4702                 return NULL;
4703         }
4704         return &mcp_res->hlist_ent;
4705 }
4706
4707 struct mlx5_list_entry *
4708 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
4709                       void *cb_ctx __rte_unused)
4710 {
4711         struct rte_eth_dev *dev = tool_ctx;
4712         struct mlx5_priv *priv = dev->data->dev_private;
4713         struct mlx5_flow_mreg_copy_resource *mcp_res;
4714         uint32_t idx = 0;
4715
4716         mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
4717         if (!mcp_res) {
4718                 rte_errno = ENOMEM;
4719                 return NULL;
4720         }
4721         memcpy(mcp_res, oentry, sizeof(*mcp_res));
4722         mcp_res->idx = idx;
4723         return &mcp_res->hlist_ent;
4724 }
4725
4726 void
4727 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
4728 {
4729         struct mlx5_flow_mreg_copy_resource *mcp_res =
4730                                container_of(entry, typeof(*mcp_res), hlist_ent);
4731         struct rte_eth_dev *dev = tool_ctx;
4732         struct mlx5_priv *priv = dev->data->dev_private;
4733
4734         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
4735 }
4736
4737 /**
4738  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4739  *
4740  * As mark_id is unique, if there's already a registered flow for the mark_id,
4741  * return by increasing the reference counter of the resource. Otherwise, create
4742  * the resource (mcp_res) and flow.
4743  *
4744  * Flow looks like,
4745  *   - If ingress port is ANY and reg_c[1] is mark_id,
4746  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4747  *
4748  * For default flow (zero mark_id), flow is like,
4749  *   - If ingress port is ANY,
4750  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
4751  *
4752  * @param dev
4753  *   Pointer to Ethernet device.
4754  * @param mark_id
4755  *   ID of MARK action, zero means default flow for META.
4756  * @param[out] error
4757  *   Perform verbose error reporting if not NULL.
4758  *
4759  * @return
4760  *   Associated resource on success, NULL otherwise and rte_errno is set.
4761  */
4762 static struct mlx5_flow_mreg_copy_resource *
4763 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
4764                           struct rte_flow_error *error)
4765 {
4766         struct mlx5_priv *priv = dev->data->dev_private;
4767         struct mlx5_list_entry *entry;
4768         struct mlx5_flow_cb_ctx ctx = {
4769                 .dev = dev,
4770                 .error = error,
4771                 .data = &mark_id,
4772         };
4773
4774         /* Check if already registered. */
4775         MLX5_ASSERT(priv->mreg_cp_tbl);
4776         entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx);
4777         if (!entry)
4778                 return NULL;
4779         return container_of(entry, struct mlx5_flow_mreg_copy_resource,
4780                             hlist_ent);
4781 }
4782
4783 void
4784 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
4785 {
4786         struct mlx5_flow_mreg_copy_resource *mcp_res =
4787                                container_of(entry, typeof(*mcp_res), hlist_ent);
4788         struct rte_eth_dev *dev = tool_ctx;
4789         struct mlx5_priv *priv = dev->data->dev_private;
4790
4791         MLX5_ASSERT(mcp_res->rix_flow);
4792         flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow);
4793         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
4794 }
4795
4796 /**
4797  * Release flow in RX_CP_TBL.
4798  *
4799  * @param dev
4800  *   Pointer to Ethernet device.
4801  * @flow
4802  *   Parent flow for wich copying is provided.
4803  */
4804 static void
4805 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
4806                           struct rte_flow *flow)
4807 {
4808         struct mlx5_flow_mreg_copy_resource *mcp_res;
4809         struct mlx5_priv *priv = dev->data->dev_private;
4810
4811         if (!flow->rix_mreg_copy)
4812                 return;
4813         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
4814                                  flow->rix_mreg_copy);
4815         if (!mcp_res || !priv->mreg_cp_tbl)
4816                 return;
4817         MLX5_ASSERT(mcp_res->rix_flow);
4818         mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
4819         flow->rix_mreg_copy = 0;
4820 }
4821
4822 /**
4823  * Remove the default copy action from RX_CP_TBL.
4824  *
4825  * This functions is called in the mlx5_dev_start(). No thread safe
4826  * is guaranteed.
4827  *
4828  * @param dev
4829  *   Pointer to Ethernet device.
4830  */
4831 static void
4832 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
4833 {
4834         struct mlx5_list_entry *entry;
4835         struct mlx5_priv *priv = dev->data->dev_private;
4836         struct mlx5_flow_cb_ctx ctx;
4837         uint32_t mark_id;
4838
4839         /* Check if default flow is registered. */
4840         if (!priv->mreg_cp_tbl)
4841                 return;
4842         mark_id = MLX5_DEFAULT_COPY_ID;
4843         ctx.data = &mark_id;
4844         entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx);
4845         if (!entry)
4846                 return;
4847         mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
4848 }
4849
4850 /**
4851  * Add the default copy action in in RX_CP_TBL.
4852  *
4853  * This functions is called in the mlx5_dev_start(). No thread safe
4854  * is guaranteed.
4855  *
4856  * @param dev
4857  *   Pointer to Ethernet device.
4858  * @param[out] error
4859  *   Perform verbose error reporting if not NULL.
4860  *
4861  * @return
4862  *   0 for success, negative value otherwise and rte_errno is set.
4863  */
4864 static int
4865 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
4866                                   struct rte_flow_error *error)
4867 {
4868         struct mlx5_priv *priv = dev->data->dev_private;
4869         struct mlx5_flow_mreg_copy_resource *mcp_res;
4870         struct mlx5_flow_cb_ctx ctx;
4871         uint32_t mark_id;
4872
4873         /* Check whether extensive metadata feature is engaged. */
4874         if (!priv->sh->config.dv_flow_en ||
4875             priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4876             !mlx5_flow_ext_mreg_supported(dev) ||
4877             !priv->sh->dv_regc0_mask)
4878                 return 0;
4879         /*
4880          * Add default mreg copy flow may be called multiple time, but
4881          * only be called once in stop. Avoid register it twice.
4882          */
4883         mark_id = MLX5_DEFAULT_COPY_ID;
4884         ctx.data = &mark_id;
4885         if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx))
4886                 return 0;
4887         mcp_res = flow_mreg_add_copy_action(dev, mark_id, error);
4888         if (!mcp_res)
4889                 return -rte_errno;
4890         return 0;
4891 }
4892
4893 /**
4894  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4895  *
4896  * All the flow having Q/RSS action should be split by
4897  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
4898  * performs the following,
4899  *   - CQE->flow_tag := reg_c[1] (MARK)
4900  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4901  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
4902  * but there should be a flow per each MARK ID set by MARK action.
4903  *
4904  * For the aforementioned reason, if there's a MARK action in flow's action
4905  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
4906  * the MARK ID to CQE's flow_tag like,
4907  *   - If reg_c[1] is mark_id,
4908  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4909  *
4910  * For SET_META action which stores value in reg_c[0], as the destination is
4911  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
4912  * MARK ID means the default flow. The default flow looks like,
4913  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4914  *
4915  * @param dev
4916  *   Pointer to Ethernet device.
4917  * @param flow
4918  *   Pointer to flow structure.
4919  * @param[in] actions
4920  *   Pointer to the list of actions.
4921  * @param[out] error
4922  *   Perform verbose error reporting if not NULL.
4923  *
4924  * @return
4925  *   0 on success, negative value otherwise and rte_errno is set.
4926  */
4927 static int
4928 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
4929                             struct rte_flow *flow,
4930                             const struct rte_flow_action *actions,
4931                             struct rte_flow_error *error)
4932 {
4933         struct mlx5_priv *priv = dev->data->dev_private;
4934         struct mlx5_sh_config *config = &priv->sh->config;
4935         struct mlx5_flow_mreg_copy_resource *mcp_res;
4936         const struct rte_flow_action_mark *mark;
4937
4938         /* Check whether extensive metadata feature is engaged. */
4939         if (!config->dv_flow_en ||
4940             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4941             !mlx5_flow_ext_mreg_supported(dev) ||
4942             !priv->sh->dv_regc0_mask)
4943                 return 0;
4944         /* Find MARK action. */
4945         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4946                 switch (actions->type) {
4947                 case RTE_FLOW_ACTION_TYPE_FLAG:
4948                         mcp_res = flow_mreg_add_copy_action
4949                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
4950                         if (!mcp_res)
4951                                 return -rte_errno;
4952                         flow->rix_mreg_copy = mcp_res->idx;
4953                         return 0;
4954                 case RTE_FLOW_ACTION_TYPE_MARK:
4955                         mark = (const struct rte_flow_action_mark *)
4956                                 actions->conf;
4957                         mcp_res =
4958                                 flow_mreg_add_copy_action(dev, mark->id, error);
4959                         if (!mcp_res)
4960                                 return -rte_errno;
4961                         flow->rix_mreg_copy = mcp_res->idx;
4962                         return 0;
4963                 default:
4964                         break;
4965                 }
4966         }
4967         return 0;
4968 }
4969
4970 #define MLX5_MAX_SPLIT_ACTIONS 24
4971 #define MLX5_MAX_SPLIT_ITEMS 24
4972
4973 /**
4974  * Split the hairpin flow.
4975  * Since HW can't support encap and push-vlan on Rx, we move these
4976  * actions to Tx.
4977  * If the count action is after the encap then we also
4978  * move the count action. in this case the count will also measure
4979  * the outer bytes.
4980  *
4981  * @param dev
4982  *   Pointer to Ethernet device.
4983  * @param[in] actions
4984  *   Associated actions (list terminated by the END action).
4985  * @param[out] actions_rx
4986  *   Rx flow actions.
4987  * @param[out] actions_tx
4988  *   Tx flow actions..
4989  * @param[out] pattern_tx
4990  *   The pattern items for the Tx flow.
4991  * @param[out] flow_id
4992  *   The flow ID connected to this flow.
4993  *
4994  * @return
4995  *   0 on success.
4996  */
4997 static int
4998 flow_hairpin_split(struct rte_eth_dev *dev,
4999                    const struct rte_flow_action actions[],
5000                    struct rte_flow_action actions_rx[],
5001                    struct rte_flow_action actions_tx[],
5002                    struct rte_flow_item pattern_tx[],
5003                    uint32_t flow_id)
5004 {
5005         const struct rte_flow_action_raw_encap *raw_encap;
5006         const struct rte_flow_action_raw_decap *raw_decap;
5007         struct mlx5_rte_flow_action_set_tag *set_tag;
5008         struct rte_flow_action *tag_action;
5009         struct mlx5_rte_flow_item_tag *tag_item;
5010         struct rte_flow_item *item;
5011         char *addr;
5012         int encap = 0;
5013
5014         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5015                 switch (actions->type) {
5016                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5017                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5018                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5019                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5020                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5021                         rte_memcpy(actions_tx, actions,
5022                                sizeof(struct rte_flow_action));
5023                         actions_tx++;
5024                         break;
5025                 case RTE_FLOW_ACTION_TYPE_COUNT:
5026                         if (encap) {
5027                                 rte_memcpy(actions_tx, actions,
5028                                            sizeof(struct rte_flow_action));
5029                                 actions_tx++;
5030                         } else {
5031                                 rte_memcpy(actions_rx, actions,
5032                                            sizeof(struct rte_flow_action));
5033                                 actions_rx++;
5034                         }
5035                         break;
5036                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5037                         raw_encap = actions->conf;
5038                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) {
5039                                 memcpy(actions_tx, actions,
5040                                        sizeof(struct rte_flow_action));
5041                                 actions_tx++;
5042                                 encap = 1;
5043                         } else {
5044                                 rte_memcpy(actions_rx, actions,
5045                                            sizeof(struct rte_flow_action));
5046                                 actions_rx++;
5047                         }
5048                         break;
5049                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5050                         raw_decap = actions->conf;
5051                         if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) {
5052                                 memcpy(actions_tx, actions,
5053                                        sizeof(struct rte_flow_action));
5054                                 actions_tx++;
5055                         } else {
5056                                 rte_memcpy(actions_rx, actions,
5057                                            sizeof(struct rte_flow_action));
5058                                 actions_rx++;
5059                         }
5060                         break;
5061                 default:
5062                         rte_memcpy(actions_rx, actions,
5063                                    sizeof(struct rte_flow_action));
5064                         actions_rx++;
5065                         break;
5066                 }
5067         }
5068         /* Add set meta action and end action for the Rx flow. */
5069         tag_action = actions_rx;
5070         tag_action->type = (enum rte_flow_action_type)
5071                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5072         actions_rx++;
5073         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
5074         actions_rx++;
5075         set_tag = (void *)actions_rx;
5076         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
5077                 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL),
5078                 .data = flow_id,
5079         };
5080         MLX5_ASSERT(set_tag->id > REG_NON);
5081         tag_action->conf = set_tag;
5082         /* Create Tx item list. */
5083         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
5084         addr = (void *)&pattern_tx[2];
5085         item = pattern_tx;
5086         item->type = (enum rte_flow_item_type)
5087                      MLX5_RTE_FLOW_ITEM_TYPE_TAG;
5088         tag_item = (void *)addr;
5089         tag_item->data = flow_id;
5090         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
5091         MLX5_ASSERT(set_tag->id > REG_NON);
5092         item->spec = tag_item;
5093         addr += sizeof(struct mlx5_rte_flow_item_tag);
5094         tag_item = (void *)addr;
5095         tag_item->data = UINT32_MAX;
5096         tag_item->id = UINT16_MAX;
5097         item->mask = tag_item;
5098         item->last = NULL;
5099         item++;
5100         item->type = RTE_FLOW_ITEM_TYPE_END;
5101         return 0;
5102 }
5103
5104 /**
5105  * The last stage of splitting chain, just creates the subflow
5106  * without any modification.
5107  *
5108  * @param[in] dev
5109  *   Pointer to Ethernet device.
5110  * @param[in] flow
5111  *   Parent flow structure pointer.
5112  * @param[in, out] sub_flow
5113  *   Pointer to return the created subflow, may be NULL.
5114  * @param[in] attr
5115  *   Flow rule attributes.
5116  * @param[in] items
5117  *   Pattern specification (list terminated by the END pattern item).
5118  * @param[in] actions
5119  *   Associated actions (list terminated by the END action).
5120  * @param[in] flow_split_info
5121  *   Pointer to flow split info structure.
5122  * @param[out] error
5123  *   Perform verbose error reporting if not NULL.
5124  * @return
5125  *   0 on success, negative value otherwise
5126  */
5127 static int
5128 flow_create_split_inner(struct rte_eth_dev *dev,
5129                         struct rte_flow *flow,
5130                         struct mlx5_flow **sub_flow,
5131                         const struct rte_flow_attr *attr,
5132                         const struct rte_flow_item items[],
5133                         const struct rte_flow_action actions[],
5134                         struct mlx5_flow_split_info *flow_split_info,
5135                         struct rte_flow_error *error)
5136 {
5137         struct mlx5_flow *dev_flow;
5138         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
5139
5140         dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
5141                                     flow_split_info->flow_idx, error);
5142         if (!dev_flow)
5143                 return -rte_errno;
5144         dev_flow->flow = flow;
5145         dev_flow->external = flow_split_info->external;
5146         dev_flow->skip_scale = flow_split_info->skip_scale;
5147         /* Subflow object was created, we must include one in the list. */
5148         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
5149                       dev_flow->handle, next);
5150         /*
5151          * If dev_flow is as one of the suffix flow, some actions in suffix
5152          * flow may need some user defined item layer flags, and pass the
5153          * Metadata rxq mark flag to suffix flow as well.
5154          */
5155         if (flow_split_info->prefix_layers)
5156                 dev_flow->handle->layers = flow_split_info->prefix_layers;
5157         if (flow_split_info->prefix_mark) {
5158                 MLX5_ASSERT(wks);
5159                 wks->mark = 1;
5160         }
5161         if (sub_flow)
5162                 *sub_flow = dev_flow;
5163 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5164         dev_flow->dv.table_id = flow_split_info->table_id;
5165 #endif
5166         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
5167 }
5168
5169 /**
5170  * Get the sub policy of a meter.
5171  *
5172  * @param[in] dev
5173  *   Pointer to Ethernet device.
5174  * @param[in] flow
5175  *   Parent flow structure pointer.
5176  * @param wks
5177  *   Pointer to thread flow work space.
5178  * @param[in] attr
5179  *   Flow rule attributes.
5180  * @param[in] items
5181  *   Pattern specification (list terminated by the END pattern item).
5182  * @param[out] error
5183  *   Perform verbose error reporting if not NULL.
5184  *
5185  * @return
5186  *   Pointer to the meter sub policy, NULL otherwise and rte_errno is set.
5187  */
5188 static struct mlx5_flow_meter_sub_policy *
5189 get_meter_sub_policy(struct rte_eth_dev *dev,
5190                      struct rte_flow *flow,
5191                      struct mlx5_flow_workspace *wks,
5192                      const struct rte_flow_attr *attr,
5193                      const struct rte_flow_item items[],
5194                      struct rte_flow_error *error)
5195 {
5196         struct mlx5_flow_meter_policy *policy;
5197         struct mlx5_flow_meter_policy *final_policy;
5198         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
5199
5200         policy = wks->policy;
5201         final_policy = policy->is_hierarchy ? wks->final_policy : policy;
5202         if (final_policy->is_rss || final_policy->is_queue) {
5203                 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS];
5204                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0};
5205                 uint32_t i;
5206
5207                 /*
5208                  * This is a tmp dev_flow,
5209                  * no need to register any matcher for it in translate.
5210                  */
5211                 wks->skip_matcher_reg = 1;
5212                 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
5213                         struct mlx5_flow dev_flow = {0};
5214                         struct mlx5_flow_handle dev_handle = { {0} };
5215                         uint8_t fate = final_policy->act_cnt[i].fate_action;
5216
5217                         if (fate == MLX5_FLOW_FATE_SHARED_RSS) {
5218                                 const struct rte_flow_action_rss *rss_act =
5219                                         final_policy->act_cnt[i].rss->conf;
5220                                 struct rte_flow_action rss_actions[2] = {
5221                                         [0] = {
5222                                         .type = RTE_FLOW_ACTION_TYPE_RSS,
5223                                         .conf = rss_act,
5224                                         },
5225                                         [1] = {
5226                                         .type = RTE_FLOW_ACTION_TYPE_END,
5227                                         .conf = NULL,
5228                                         }
5229                                 };
5230
5231                                 dev_flow.handle = &dev_handle;
5232                                 dev_flow.ingress = attr->ingress;
5233                                 dev_flow.flow = flow;
5234                                 dev_flow.external = 0;
5235 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
5236                                 dev_flow.dv.transfer = attr->transfer;
5237 #endif
5238                                 /**
5239                                  * Translate RSS action to get rss hash fields.
5240                                  */
5241                                 if (flow_drv_translate(dev, &dev_flow, attr,
5242                                                 items, rss_actions, error))
5243                                         goto exit;
5244                                 rss_desc_v[i] = wks->rss_desc;
5245                                 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN;
5246                                 rss_desc_v[i].hash_fields =
5247                                                 dev_flow.hash_fields;
5248                                 rss_desc_v[i].queue_num =
5249                                                 rss_desc_v[i].hash_fields ?
5250                                                 rss_desc_v[i].queue_num : 1;
5251                                 rss_desc_v[i].tunnel =
5252                                                 !!(dev_flow.handle->layers &
5253                                                    MLX5_FLOW_LAYER_TUNNEL);
5254                                 /* Use the RSS queues in the containers. */
5255                                 rss_desc_v[i].queue =
5256                                         (uint16_t *)(uintptr_t)rss_act->queue;
5257                                 rss_desc[i] = &rss_desc_v[i];
5258                         } else if (fate == MLX5_FLOW_FATE_QUEUE) {
5259                                 /* This is queue action. */
5260                                 rss_desc_v[i] = wks->rss_desc;
5261                                 rss_desc_v[i].key_len = 0;
5262                                 rss_desc_v[i].hash_fields = 0;
5263                                 rss_desc_v[i].queue =
5264                                         &final_policy->act_cnt[i].queue;
5265                                 rss_desc_v[i].queue_num = 1;
5266                                 rss_desc[i] = &rss_desc_v[i];
5267                         } else {
5268                                 rss_desc[i] = NULL;
5269                         }
5270                 }
5271                 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev,
5272                                                 flow, policy, rss_desc);
5273         } else {
5274                 enum mlx5_meter_domain mtr_domain =
5275                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
5276                                 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
5277                                                 MLX5_MTR_DOMAIN_INGRESS);
5278                 sub_policy = policy->sub_policys[mtr_domain][0];
5279         }
5280         if (!sub_policy)
5281                 rte_flow_error_set(error, EINVAL,
5282                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5283                                    "Failed to get meter sub-policy.");
5284 exit:
5285         return sub_policy;
5286 }
5287
5288 /**
5289  * Split the meter flow.
5290  *
5291  * As meter flow will split to three sub flow, other than meter
5292  * action, the other actions make sense to only meter accepts
5293  * the packet. If it need to be dropped, no other additional
5294  * actions should be take.
5295  *
5296  * One kind of special action which decapsulates the L3 tunnel
5297  * header will be in the prefix sub flow, as not to take the
5298  * L3 tunnel header into account.
5299  *
5300  * @param[in] dev
5301  *   Pointer to Ethernet device.
5302  * @param[in] flow
5303  *   Parent flow structure pointer.
5304  * @param wks
5305  *   Pointer to thread flow work space.
5306  * @param[in] attr
5307  *   Flow rule attributes.
5308  * @param[in] items
5309  *   Pattern specification (list terminated by the END pattern item).
5310  * @param[out] sfx_items
5311  *   Suffix flow match items (list terminated by the END pattern item).
5312  * @param[in] actions
5313  *   Associated actions (list terminated by the END action).
5314  * @param[out] actions_sfx
5315  *   Suffix flow actions.
5316  * @param[out] actions_pre
5317  *   Prefix flow actions.
5318  * @param[out] mtr_flow_id
5319  *   Pointer to meter flow id.
5320  * @param[out] error
5321  *   Perform verbose error reporting if not NULL.
5322  *
5323  * @return
5324  *   0 on success, a negative errno value otherwise and rte_errno is set.
5325  */
5326 static int
5327 flow_meter_split_prep(struct rte_eth_dev *dev,
5328                       struct rte_flow *flow,
5329                       struct mlx5_flow_workspace *wks,
5330                       const struct rte_flow_attr *attr,
5331                       const struct rte_flow_item items[],
5332                       struct rte_flow_item sfx_items[],
5333                       const struct rte_flow_action actions[],
5334                       struct rte_flow_action actions_sfx[],
5335                       struct rte_flow_action actions_pre[],
5336                       uint32_t *mtr_flow_id,
5337                       struct rte_flow_error *error)
5338 {
5339         struct mlx5_priv *priv = dev->data->dev_private;
5340         struct mlx5_flow_meter_info *fm = wks->fm;
5341         struct rte_flow_action *tag_action = NULL;
5342         struct rte_flow_item *tag_item;
5343         struct mlx5_rte_flow_action_set_tag *set_tag;
5344         const struct rte_flow_action_raw_encap *raw_encap;
5345         const struct rte_flow_action_raw_decap *raw_decap;
5346         struct mlx5_rte_flow_item_tag *tag_item_spec;
5347         struct mlx5_rte_flow_item_tag *tag_item_mask;
5348         uint32_t tag_id = 0;
5349         struct rte_flow_item *vlan_item_dst = NULL;
5350         const struct rte_flow_item *vlan_item_src = NULL;
5351         const struct rte_flow_item *orig_items = items;
5352         struct rte_flow_action *hw_mtr_action;
5353         struct rte_flow_action *action_pre_head = NULL;
5354         int32_t flow_src_port = priv->representor_id;
5355         bool mtr_first;
5356         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
5357         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
5358                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
5359         uint32_t flow_id = 0;
5360         uint32_t flow_id_reversed = 0;
5361         uint8_t flow_id_bits = 0;
5362         bool after_meter = false;
5363         int shift;
5364
5365         /* Prepare the suffix subflow items. */
5366         tag_item = sfx_items++;
5367         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5368                 struct mlx5_priv *port_priv;
5369                 const struct rte_flow_item_port_id *pid_v;
5370                 int item_type = items->type;
5371
5372                 switch (item_type) {
5373                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
5374                         pid_v = items->spec;
5375                         MLX5_ASSERT(pid_v);
5376                         port_priv = mlx5_port_to_eswitch_info(pid_v->id, false);
5377                         if (!port_priv)
5378                                 return rte_flow_error_set(error,
5379                                                 rte_errno,
5380                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5381                                                 pid_v,
5382                                                 "Failed to get port info.");
5383                         flow_src_port = port_priv->representor_id;
5384                         if (!fm->def_policy && wks->policy->is_hierarchy &&
5385                             flow_src_port != priv->representor_id) {
5386                                 if (flow_drv_mtr_hierarchy_rule_create(dev,
5387                                                                 flow, fm,
5388                                                                 flow_src_port,
5389                                                                 items,
5390                                                                 error))
5391                                         return -rte_errno;
5392                         }
5393                         memcpy(sfx_items, items, sizeof(*sfx_items));
5394                         sfx_items++;
5395                         break;
5396                 case RTE_FLOW_ITEM_TYPE_VLAN:
5397                         /* Determine if copy vlan item below. */
5398                         vlan_item_src = items;
5399                         vlan_item_dst = sfx_items++;
5400                         vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID;
5401                         break;
5402                 default:
5403                         break;
5404                 }
5405         }
5406         sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
5407         sfx_items++;
5408         mtr_first = priv->sh->meter_aso_en &&
5409                 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX));
5410         /* For ASO meter, meter must be before tag in TX direction. */
5411         if (mtr_first) {
5412                 action_pre_head = actions_pre++;
5413                 /* Leave space for tag action. */
5414                 tag_action = actions_pre++;
5415         }
5416         /* Prepare the actions for prefix and suffix flow. */
5417         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5418                 struct rte_flow_action *action_cur = NULL;
5419
5420                 switch (actions->type) {
5421                 case RTE_FLOW_ACTION_TYPE_METER:
5422                         if (mtr_first) {
5423                                 action_cur = action_pre_head;
5424                         } else {
5425                                 /* Leave space for tag action. */
5426                                 tag_action = actions_pre++;
5427                                 action_cur = actions_pre++;
5428                         }
5429                         after_meter = true;
5430                         break;
5431                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5432                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5433                         action_cur = actions_pre++;
5434                         break;
5435                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5436                         raw_encap = actions->conf;
5437                         if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
5438                                 action_cur = actions_pre++;
5439                         break;
5440                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5441                         raw_decap = actions->conf;
5442                         if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
5443                                 action_cur = actions_pre++;
5444                         break;
5445                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5446                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5447                         if (vlan_item_dst && vlan_item_src) {
5448                                 memcpy(vlan_item_dst, vlan_item_src,
5449                                         sizeof(*vlan_item_dst));
5450                                 /*
5451                                  * Convert to internal match item, it is used
5452                                  * for vlan push and set vid.
5453                                  */
5454                                 vlan_item_dst->type = (enum rte_flow_item_type)
5455                                                 MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
5456                         }
5457                         break;
5458                 case RTE_FLOW_ACTION_TYPE_COUNT:
5459                         if (fm->def_policy)
5460                                 action_cur = after_meter ?
5461                                                 actions_sfx++ : actions_pre++;
5462                         break;
5463                 default:
5464                         break;
5465                 }
5466                 if (!action_cur)
5467                         action_cur = (fm->def_policy) ?
5468                                         actions_sfx++ : actions_pre++;
5469                 memcpy(action_cur, actions, sizeof(struct rte_flow_action));
5470         }
5471         /* Add end action to the actions. */
5472         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
5473         if (priv->sh->meter_aso_en) {
5474                 /**
5475                  * For ASO meter, need to add an extra jump action explicitly,
5476                  * to jump from meter to policer table.
5477                  */
5478                 struct mlx5_flow_meter_sub_policy *sub_policy;
5479                 struct mlx5_flow_tbl_data_entry *tbl_data;
5480
5481                 if (!fm->def_policy) {
5482                         sub_policy = get_meter_sub_policy(dev, flow, wks,
5483                                                           attr, orig_items,
5484                                                           error);
5485                         if (!sub_policy)
5486                                 return -rte_errno;
5487                 } else {
5488                         enum mlx5_meter_domain mtr_domain =
5489                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
5490                                 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
5491                                                 MLX5_MTR_DOMAIN_INGRESS);
5492
5493                         sub_policy =
5494                         &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy;
5495                 }
5496                 tbl_data = container_of(sub_policy->tbl_rsc,
5497                                         struct mlx5_flow_tbl_data_entry, tbl);
5498                 hw_mtr_action = actions_pre++;
5499                 hw_mtr_action->type = (enum rte_flow_action_type)
5500                                       MLX5_RTE_FLOW_ACTION_TYPE_JUMP;
5501                 hw_mtr_action->conf = tbl_data->jump.action;
5502         }
5503         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
5504         actions_pre++;
5505         if (!tag_action)
5506                 return rte_flow_error_set(error, ENOMEM,
5507                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5508                                           NULL, "No tag action space.");
5509         if (!mtr_flow_id) {
5510                 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID;
5511                 goto exit;
5512         }
5513         /* Only default-policy Meter creates mtr flow id. */
5514         if (fm->def_policy) {
5515                 mlx5_ipool_malloc(fm->flow_ipool, &tag_id);
5516                 if (!tag_id)
5517                         return rte_flow_error_set(error, ENOMEM,
5518                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5519                                         "Failed to allocate meter flow id.");
5520                 flow_id = tag_id - 1;
5521                 flow_id_bits = (!flow_id) ? 1 :
5522                                 (MLX5_REG_BITS - __builtin_clz(flow_id));
5523                 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) >
5524                     mtr_reg_bits) {
5525                         mlx5_ipool_free(fm->flow_ipool, tag_id);
5526                         return rte_flow_error_set(error, EINVAL,
5527                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5528                                         "Meter flow id exceeds max limit.");
5529                 }
5530                 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits)
5531                         priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits;
5532         }
5533         /* Build tag actions and items for meter_id/meter flow_id. */
5534         set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre;
5535         tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
5536         tag_item_mask = tag_item_spec + 1;
5537         /* Both flow_id and meter_id share the same register. */
5538         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
5539                 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
5540                                                             0, error),
5541                 .offset = mtr_id_offset,
5542                 .length = mtr_reg_bits,
5543                 .data = flow->meter,
5544         };
5545         /*
5546          * The color Reg bits used by flow_id are growing from
5547          * msb to lsb, so must do bit reverse for flow_id val in RegC.
5548          */
5549         for (shift = 0; shift < flow_id_bits; shift++)
5550                 flow_id_reversed = (flow_id_reversed << 1) |
5551                                 ((flow_id >> shift) & 0x1);
5552         set_tag->data |=
5553                 flow_id_reversed << (mtr_reg_bits - flow_id_bits);
5554         tag_item_spec->id = set_tag->id;
5555         tag_item_spec->data = set_tag->data << mtr_id_offset;
5556         tag_item_mask->data = UINT32_MAX << mtr_id_offset;
5557         tag_action->type = (enum rte_flow_action_type)
5558                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5559         tag_action->conf = set_tag;
5560         tag_item->type = (enum rte_flow_item_type)
5561                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG;
5562         tag_item->spec = tag_item_spec;
5563         tag_item->last = NULL;
5564         tag_item->mask = tag_item_mask;
5565 exit:
5566         if (mtr_flow_id)
5567                 *mtr_flow_id = tag_id;
5568         return 0;
5569 }
5570
5571 /**
5572  * Split action list having QUEUE/RSS for metadata register copy.
5573  *
5574  * Once Q/RSS action is detected in user's action list, the flow action
5575  * should be split in order to copy metadata registers, which will happen in
5576  * RX_CP_TBL like,
5577  *   - CQE->flow_tag := reg_c[1] (MARK)
5578  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
5579  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
5580  * This is because the last action of each flow must be a terminal action
5581  * (QUEUE, RSS or DROP).
5582  *
5583  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
5584  * stored and kept in the mlx5_flow structure per each sub_flow.
5585  *
5586  * The Q/RSS action is replaced with,
5587  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
5588  * And the following JUMP action is added at the end,
5589  *   - JUMP, to RX_CP_TBL.
5590  *
5591  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
5592  * flow_create_split_metadata() routine. The flow will look like,
5593  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
5594  *
5595  * @param dev
5596  *   Pointer to Ethernet device.
5597  * @param[out] split_actions
5598  *   Pointer to store split actions to jump to CP_TBL.
5599  * @param[in] actions
5600  *   Pointer to the list of original flow actions.
5601  * @param[in] qrss
5602  *   Pointer to the Q/RSS action.
5603  * @param[in] actions_n
5604  *   Number of original actions.
5605  * @param[in] mtr_sfx
5606  *   Check if it is in meter suffix table.
5607  * @param[out] error
5608  *   Perform verbose error reporting if not NULL.
5609  *
5610  * @return
5611  *   non-zero unique flow_id on success, otherwise 0 and
5612  *   error/rte_error are set.
5613  */
5614 static uint32_t
5615 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
5616                           struct rte_flow_action *split_actions,
5617                           const struct rte_flow_action *actions,
5618                           const struct rte_flow_action *qrss,
5619                           int actions_n, int mtr_sfx,
5620                           struct rte_flow_error *error)
5621 {
5622         struct mlx5_priv *priv = dev->data->dev_private;
5623         struct mlx5_rte_flow_action_set_tag *set_tag;
5624         struct rte_flow_action_jump *jump;
5625         const int qrss_idx = qrss - actions;
5626         uint32_t flow_id = 0;
5627         int ret = 0;
5628
5629         /*
5630          * Given actions will be split
5631          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
5632          * - Add jump to mreg CP_TBL.
5633          * As a result, there will be one more action.
5634          */
5635         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
5636         /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */
5637         ++actions_n;
5638         set_tag = (void *)(split_actions + actions_n);
5639         /*
5640          * If we are not the meter suffix flow, add the tag action.
5641          * Since meter suffix flow already has the tag added.
5642          */
5643         if (!mtr_sfx) {
5644                 /*
5645                  * Allocate the new subflow ID. This one is unique within
5646                  * device and not shared with representors. Otherwise,
5647                  * we would have to resolve multi-thread access synch
5648                  * issue. Each flow on the shared device is appended
5649                  * with source vport identifier, so the resulting
5650                  * flows will be unique in the shared (by master and
5651                  * representors) domain even if they have coinciding
5652                  * IDs.
5653                  */
5654                 mlx5_ipool_malloc(priv->sh->ipool
5655                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id);
5656                 if (!flow_id)
5657                         return rte_flow_error_set(error, ENOMEM,
5658                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5659                                                   NULL, "can't allocate id "
5660                                                   "for split Q/RSS subflow");
5661                 /* Internal SET_TAG action to set flow ID. */
5662                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
5663                         .data = flow_id,
5664                 };
5665                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
5666                 if (ret < 0)
5667                         return ret;
5668                 set_tag->id = ret;
5669                 /* Construct new actions array. */
5670                 /* Replace QUEUE/RSS action. */
5671                 split_actions[qrss_idx] = (struct rte_flow_action){
5672                         .type = (enum rte_flow_action_type)
5673                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
5674                         .conf = set_tag,
5675                 };
5676         } else {
5677                 /*
5678                  * If we are the suffix flow of meter, tag already exist.
5679                  * Set the QUEUE/RSS action to void.
5680                  */
5681                 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID;
5682         }
5683         /* JUMP action to jump to mreg copy table (CP_TBL). */
5684         jump = (void *)(set_tag + 1);
5685         *jump = (struct rte_flow_action_jump){
5686                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
5687         };
5688         split_actions[actions_n - 2] = (struct rte_flow_action){
5689                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
5690                 .conf = jump,
5691         };
5692         split_actions[actions_n - 1] = (struct rte_flow_action){
5693                 .type = RTE_FLOW_ACTION_TYPE_END,
5694         };
5695         return flow_id;
5696 }
5697
5698 /**
5699  * Extend the given action list for Tx metadata copy.
5700  *
5701  * Copy the given action list to the ext_actions and add flow metadata register
5702  * copy action in order to copy reg_a set by WQE to reg_c[0].
5703  *
5704  * @param[out] ext_actions
5705  *   Pointer to the extended action list.
5706  * @param[in] actions
5707  *   Pointer to the list of actions.
5708  * @param[in] actions_n
5709  *   Number of actions in the list.
5710  * @param[out] error
5711  *   Perform verbose error reporting if not NULL.
5712  * @param[in] encap_idx
5713  *   The encap action index.
5714  *
5715  * @return
5716  *   0 on success, negative value otherwise
5717  */
5718 static int
5719 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
5720                        struct rte_flow_action *ext_actions,
5721                        const struct rte_flow_action *actions,
5722                        int actions_n, struct rte_flow_error *error,
5723                        int encap_idx)
5724 {
5725         struct mlx5_flow_action_copy_mreg *cp_mreg =
5726                 (struct mlx5_flow_action_copy_mreg *)
5727                         (ext_actions + actions_n + 1);
5728         int ret;
5729
5730         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
5731         if (ret < 0)
5732                 return ret;
5733         cp_mreg->dst = ret;
5734         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
5735         if (ret < 0)
5736                 return ret;
5737         cp_mreg->src = ret;
5738         if (encap_idx != 0)
5739                 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
5740         if (encap_idx == actions_n - 1) {
5741                 ext_actions[actions_n - 1] = (struct rte_flow_action){
5742                         .type = (enum rte_flow_action_type)
5743                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5744                         .conf = cp_mreg,
5745                 };
5746                 ext_actions[actions_n] = (struct rte_flow_action){
5747                         .type = RTE_FLOW_ACTION_TYPE_END,
5748                 };
5749         } else {
5750                 ext_actions[encap_idx] = (struct rte_flow_action){
5751                         .type = (enum rte_flow_action_type)
5752                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5753                         .conf = cp_mreg,
5754                 };
5755                 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
5756                                 sizeof(*ext_actions) * (actions_n - encap_idx));
5757         }
5758         return 0;
5759 }
5760
5761 /**
5762  * Check the match action from the action list.
5763  *
5764  * @param[in] actions
5765  *   Pointer to the list of actions.
5766  * @param[in] attr
5767  *   Flow rule attributes.
5768  * @param[in] action
5769  *   The action to be check if exist.
5770  * @param[out] match_action_pos
5771  *   Pointer to the position of the matched action if exists, otherwise is -1.
5772  * @param[out] qrss_action_pos
5773  *   Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
5774  * @param[out] modify_after_mirror
5775  *   Pointer to the flag of modify action after FDB mirroring.
5776  *
5777  * @return
5778  *   > 0 the total number of actions.
5779  *   0 if not found match action in action list.
5780  */
5781 static int
5782 flow_check_match_action(const struct rte_flow_action actions[],
5783                         const struct rte_flow_attr *attr,
5784                         enum rte_flow_action_type action,
5785                         int *match_action_pos, int *qrss_action_pos,
5786                         int *modify_after_mirror)
5787 {
5788         const struct rte_flow_action_sample *sample;
5789         const struct rte_flow_action_raw_decap *decap;
5790         int actions_n = 0;
5791         uint32_t ratio = 0;
5792         int sub_type = 0;
5793         int flag = 0;
5794         int fdb_mirror = 0;
5795
5796         *match_action_pos = -1;
5797         *qrss_action_pos = -1;
5798         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5799                 if (actions->type == action) {
5800                         flag = 1;
5801                         *match_action_pos = actions_n;
5802                 }
5803                 switch (actions->type) {
5804                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5805                 case RTE_FLOW_ACTION_TYPE_RSS:
5806                         *qrss_action_pos = actions_n;
5807                         break;
5808                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
5809                         sample = actions->conf;
5810                         ratio = sample->ratio;
5811                         sub_type = ((const struct rte_flow_action *)
5812                                         (sample->actions))->type;
5813                         if (ratio == 1 && attr->transfer)
5814                                 fdb_mirror = 1;
5815                         break;
5816                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5817                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5818                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5819                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5820                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5821                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5822                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5823                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5824                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5825                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5826                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5827                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5828                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5829                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5830                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5831                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5832                 case RTE_FLOW_ACTION_TYPE_FLAG:
5833                 case RTE_FLOW_ACTION_TYPE_MARK:
5834                 case RTE_FLOW_ACTION_TYPE_SET_META:
5835                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5836                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5837                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5838                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5839                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5840                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5841                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5842                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
5843                 case RTE_FLOW_ACTION_TYPE_METER:
5844                         if (fdb_mirror)
5845                                 *modify_after_mirror = 1;
5846                         break;
5847                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5848                         decap = actions->conf;
5849                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5850                                 ;
5851                         actions_n++;
5852                         if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5853                                 const struct rte_flow_action_raw_encap *encap =
5854                                                                 actions->conf;
5855                                 if (decap->size <=
5856                                         MLX5_ENCAPSULATION_DECISION_SIZE &&
5857                                     encap->size >
5858                                         MLX5_ENCAPSULATION_DECISION_SIZE)
5859                                         /* L3 encap. */
5860                                         break;
5861                         }
5862                         if (fdb_mirror)
5863                                 *modify_after_mirror = 1;
5864                         break;
5865                 default:
5866                         break;
5867                 }
5868                 actions_n++;
5869         }
5870         if (flag && fdb_mirror && !*modify_after_mirror) {
5871                 /* FDB mirroring uses the destination array to implement
5872                  * instead of FLOW_SAMPLER object.
5873                  */
5874                 if (sub_type != RTE_FLOW_ACTION_TYPE_END)
5875                         flag = 0;
5876         }
5877         /* Count RTE_FLOW_ACTION_TYPE_END. */
5878         return flag ? actions_n + 1 : 0;
5879 }
5880
5881 #define SAMPLE_SUFFIX_ITEM 3
5882
5883 /**
5884  * Split the sample flow.
5885  *
5886  * As sample flow will split to two sub flow, sample flow with
5887  * sample action, the other actions will move to new suffix flow.
5888  *
5889  * Also add unique tag id with tag action in the sample flow,
5890  * the same tag id will be as match in the suffix flow.
5891  *
5892  * @param dev
5893  *   Pointer to Ethernet device.
5894  * @param[in] add_tag
5895  *   Add extra tag action flag.
5896  * @param[out] sfx_items
5897  *   Suffix flow match items (list terminated by the END pattern item).
5898  * @param[in] actions
5899  *   Associated actions (list terminated by the END action).
5900  * @param[out] actions_sfx
5901  *   Suffix flow actions.
5902  * @param[out] actions_pre
5903  *   Prefix flow actions.
5904  * @param[in] actions_n
5905  *  The total number of actions.
5906  * @param[in] sample_action_pos
5907  *   The sample action position.
5908  * @param[in] qrss_action_pos
5909  *   The Queue/RSS action position.
5910  * @param[in] jump_table
5911  *   Add extra jump action flag.
5912  * @param[out] error
5913  *   Perform verbose error reporting if not NULL.
5914  *
5915  * @return
5916  *   0 on success, or unique flow_id, a negative errno value
5917  *   otherwise and rte_errno is set.
5918  */
5919 static int
5920 flow_sample_split_prep(struct rte_eth_dev *dev,
5921                        int add_tag,
5922                        const struct rte_flow_item items[],
5923                        struct rte_flow_item sfx_items[],
5924                        const struct rte_flow_action actions[],
5925                        struct rte_flow_action actions_sfx[],
5926                        struct rte_flow_action actions_pre[],
5927                        int actions_n,
5928                        int sample_action_pos,
5929                        int qrss_action_pos,
5930                        int jump_table,
5931                        struct rte_flow_error *error)
5932 {
5933         struct mlx5_priv *priv = dev->data->dev_private;
5934         struct mlx5_rte_flow_action_set_tag *set_tag;
5935         struct mlx5_rte_flow_item_tag *tag_spec;
5936         struct mlx5_rte_flow_item_tag *tag_mask;
5937         struct rte_flow_action_jump *jump_action;
5938         uint32_t tag_id = 0;
5939         int append_index = 0;
5940         int set_tag_idx = -1;
5941         int index;
5942         int ret;
5943
5944         if (sample_action_pos < 0)
5945                 return rte_flow_error_set(error, EINVAL,
5946                                           RTE_FLOW_ERROR_TYPE_ACTION,
5947                                           NULL, "invalid position of sample "
5948                                           "action in list");
5949         /* Prepare the actions for prefix and suffix flow. */
5950         if (add_tag) {
5951                 /* Update the new added tag action index preceding
5952                  * the PUSH_VLAN or ENCAP action.
5953                  */
5954                 const struct rte_flow_action_raw_encap *raw_encap;
5955                 const struct rte_flow_action *action = actions;
5956                 int encap_idx;
5957                 int action_idx = 0;
5958                 int raw_decap_idx = -1;
5959                 int push_vlan_idx = -1;
5960                 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) {
5961                         switch (action->type) {
5962                         case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5963                                 raw_decap_idx = action_idx;
5964                                 break;
5965                         case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5966                                 raw_encap = action->conf;
5967                                 if (raw_encap->size >
5968                                         MLX5_ENCAPSULATION_DECISION_SIZE) {
5969                                         encap_idx = raw_decap_idx != -1 ?
5970                                                     raw_decap_idx : action_idx;
5971                                         if (encap_idx < sample_action_pos &&
5972                                             push_vlan_idx == -1)
5973                                                 set_tag_idx = encap_idx;
5974                                 }
5975                                 break;
5976                         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5977                         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5978                                 encap_idx = action_idx;
5979                                 if (encap_idx < sample_action_pos &&
5980                                     push_vlan_idx == -1)
5981                                         set_tag_idx = encap_idx;
5982                                 break;
5983                         case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5984                         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5985                                 push_vlan_idx = action_idx;
5986                                 if (push_vlan_idx < sample_action_pos)
5987                                         set_tag_idx = action_idx;
5988                                 break;
5989                         default:
5990                                 break;
5991                         }
5992                         action_idx++;
5993                 }
5994         }
5995         /* Prepare the actions for prefix and suffix flow. */
5996         if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
5997                 index = qrss_action_pos;
5998                 /* Put the preceding the Queue/RSS action into prefix flow. */
5999                 if (index != 0)
6000                         memcpy(actions_pre, actions,
6001                                sizeof(struct rte_flow_action) * index);
6002                 /* Put others preceding the sample action into prefix flow. */
6003                 if (sample_action_pos > index + 1)
6004                         memcpy(actions_pre + index, actions + index + 1,
6005                                sizeof(struct rte_flow_action) *
6006                                (sample_action_pos - index - 1));
6007                 index = sample_action_pos - 1;
6008                 /* Put Queue/RSS action into Suffix flow. */
6009                 memcpy(actions_sfx, actions + qrss_action_pos,
6010                        sizeof(struct rte_flow_action));
6011                 actions_sfx++;
6012         } else if (add_tag && set_tag_idx >= 0) {
6013                 if (set_tag_idx > 0)
6014                         memcpy(actions_pre, actions,
6015                                sizeof(struct rte_flow_action) * set_tag_idx);
6016                 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx,
6017                        sizeof(struct rte_flow_action) *
6018                        (sample_action_pos - set_tag_idx));
6019                 index = sample_action_pos;
6020         } else {
6021                 index = sample_action_pos;
6022                 if (index != 0)
6023                         memcpy(actions_pre, actions,
6024                                sizeof(struct rte_flow_action) * index);
6025         }
6026         /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress.
6027          * For CX6DX and above, metadata registers Cx preserve their value,
6028          * add an extra tag action for NIC-RX and E-Switch Domain.
6029          */
6030         if (add_tag) {
6031                 /* Prepare the prefix tag action. */
6032                 append_index++;
6033                 set_tag = (void *)(actions_pre + actions_n + append_index);
6034                 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error);
6035                 /* Trust VF/SF on CX5 not supported meter so that the reserved
6036                  * metadata regC is REG_NON, back to use application tag
6037                  * index 0.
6038                  */
6039                 if (unlikely(ret == REG_NON))
6040                         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
6041                 if (ret < 0)
6042                         return ret;
6043                 mlx5_ipool_malloc(priv->sh->ipool
6044                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id);
6045                 *set_tag = (struct mlx5_rte_flow_action_set_tag) {
6046                         .id = ret,
6047                         .data = tag_id,
6048                 };
6049                 /* Prepare the suffix subflow items. */
6050                 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
6051                         if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID) {
6052                                 memcpy(sfx_items, items, sizeof(*sfx_items));
6053                                 sfx_items++;
6054                         }
6055                 }
6056                 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
6057                 tag_spec->data = tag_id;
6058                 tag_spec->id = set_tag->id;
6059                 tag_mask = tag_spec + 1;
6060                 tag_mask->data = UINT32_MAX;
6061                 sfx_items[0] = (struct rte_flow_item){
6062                         .type = (enum rte_flow_item_type)
6063                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
6064                         .spec = tag_spec,
6065                         .last = NULL,
6066                         .mask = tag_mask,
6067                 };
6068                 sfx_items[1] = (struct rte_flow_item){
6069                         .type = (enum rte_flow_item_type)
6070                                 RTE_FLOW_ITEM_TYPE_END,
6071                 };
6072                 /* Prepare the tag action in prefix subflow. */
6073                 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx;
6074                 actions_pre[set_tag_idx] =
6075                         (struct rte_flow_action){
6076                         .type = (enum rte_flow_action_type)
6077                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
6078                         .conf = set_tag,
6079                 };
6080                 /* Update next sample position due to add one tag action */
6081                 index += 1;
6082         }
6083         /* Copy the sample action into prefix flow. */
6084         memcpy(actions_pre + index, actions + sample_action_pos,
6085                sizeof(struct rte_flow_action));
6086         index += 1;
6087         /* For the modify action after the sample action in E-Switch mirroring,
6088          * Add the extra jump action in prefix subflow and jump into the next
6089          * table, then do the modify action in the new table.
6090          */
6091         if (jump_table) {
6092                 /* Prepare the prefix jump action. */
6093                 append_index++;
6094                 jump_action = (void *)(actions_pre + actions_n + append_index);
6095                 jump_action->group = jump_table;
6096                 actions_pre[index++] =
6097                         (struct rte_flow_action){
6098                         .type = (enum rte_flow_action_type)
6099                                 RTE_FLOW_ACTION_TYPE_JUMP,
6100                         .conf = jump_action,
6101                 };
6102         }
6103         actions_pre[index] = (struct rte_flow_action){
6104                 .type = (enum rte_flow_action_type)
6105                         RTE_FLOW_ACTION_TYPE_END,
6106         };
6107         /* Put the actions after sample into Suffix flow. */
6108         memcpy(actions_sfx, actions + sample_action_pos + 1,
6109                sizeof(struct rte_flow_action) *
6110                (actions_n - sample_action_pos - 1));
6111         return tag_id;
6112 }
6113
6114 /**
6115  * The splitting for metadata feature.
6116  *
6117  * - Q/RSS action on NIC Rx should be split in order to pass by
6118  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
6119  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
6120  *
6121  * - All the actions on NIC Tx should have a mreg copy action to
6122  *   copy reg_a from WQE to reg_c[0].
6123  *
6124  * @param dev
6125  *   Pointer to Ethernet device.
6126  * @param[in] flow
6127  *   Parent flow structure pointer.
6128  * @param[in] attr
6129  *   Flow rule attributes.
6130  * @param[in] items
6131  *   Pattern specification (list terminated by the END pattern item).
6132  * @param[in] actions
6133  *   Associated actions (list terminated by the END action).
6134  * @param[in] flow_split_info
6135  *   Pointer to flow split info structure.
6136  * @param[out] error
6137  *   Perform verbose error reporting if not NULL.
6138  * @return
6139  *   0 on success, negative value otherwise
6140  */
6141 static int
6142 flow_create_split_metadata(struct rte_eth_dev *dev,
6143                            struct rte_flow *flow,
6144                            const struct rte_flow_attr *attr,
6145                            const struct rte_flow_item items[],
6146                            const struct rte_flow_action actions[],
6147                            struct mlx5_flow_split_info *flow_split_info,
6148                            struct rte_flow_error *error)
6149 {
6150         struct mlx5_priv *priv = dev->data->dev_private;
6151         struct mlx5_sh_config *config = &priv->sh->config;
6152         const struct rte_flow_action *qrss = NULL;
6153         struct rte_flow_action *ext_actions = NULL;
6154         struct mlx5_flow *dev_flow = NULL;
6155         uint32_t qrss_id = 0;
6156         int mtr_sfx = 0;
6157         size_t act_size;
6158         int actions_n;
6159         int encap_idx;
6160         int ret;
6161
6162         /* Check whether extensive metadata feature is engaged. */
6163         if (!config->dv_flow_en ||
6164             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
6165             !mlx5_flow_ext_mreg_supported(dev))
6166                 return flow_create_split_inner(dev, flow, NULL, attr, items,
6167                                                actions, flow_split_info, error);
6168         actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
6169                                                            &encap_idx);
6170         if (qrss) {
6171                 /* Exclude hairpin flows from splitting. */
6172                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
6173                         const struct rte_flow_action_queue *queue;
6174
6175                         queue = qrss->conf;
6176                         if (mlx5_rxq_is_hairpin(dev, queue->index))
6177                                 qrss = NULL;
6178                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
6179                         const struct rte_flow_action_rss *rss;
6180
6181                         rss = qrss->conf;
6182                         if (mlx5_rxq_is_hairpin(dev, rss->queue[0]))
6183                                 qrss = NULL;
6184                 }
6185         }
6186         if (qrss) {
6187                 /* Check if it is in meter suffix table. */
6188                 mtr_sfx = attr->group == (attr->transfer ?
6189                           (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
6190                           MLX5_FLOW_TABLE_LEVEL_METER);
6191                 /*
6192                  * Q/RSS action on NIC Rx should be split in order to pass by
6193                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
6194                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
6195                  */
6196                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
6197                            sizeof(struct rte_flow_action_set_tag) +
6198                            sizeof(struct rte_flow_action_jump);
6199                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
6200                                           SOCKET_ID_ANY);
6201                 if (!ext_actions)
6202                         return rte_flow_error_set(error, ENOMEM,
6203                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6204                                                   NULL, "no memory to split "
6205                                                   "metadata flow");
6206                 /*
6207                  * Create the new actions list with removed Q/RSS action
6208                  * and appended set tag and jump to register copy table
6209                  * (RX_CP_TBL). We should preallocate unique tag ID here
6210                  * in advance, because it is needed for set tag action.
6211                  */
6212                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
6213                                                     qrss, actions_n,
6214                                                     mtr_sfx, error);
6215                 if (!mtr_sfx && !qrss_id) {
6216                         ret = -rte_errno;
6217                         goto exit;
6218                 }
6219         } else if (attr->egress && !attr->transfer) {
6220                 /*
6221                  * All the actions on NIC Tx should have a metadata register
6222                  * copy action to copy reg_a from WQE to reg_c[meta]
6223                  */
6224                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
6225                            sizeof(struct mlx5_flow_action_copy_mreg);
6226                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
6227                                           SOCKET_ID_ANY);
6228                 if (!ext_actions)
6229                         return rte_flow_error_set(error, ENOMEM,
6230                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6231                                                   NULL, "no memory to split "
6232                                                   "metadata flow");
6233                 /* Create the action list appended with copy register. */
6234                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
6235                                              actions_n, error, encap_idx);
6236                 if (ret < 0)
6237                         goto exit;
6238         }
6239         /* Add the unmodified original or prefix subflow. */
6240         ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
6241                                       items, ext_actions ? ext_actions :
6242                                       actions, flow_split_info, error);
6243         if (ret < 0)
6244                 goto exit;
6245         MLX5_ASSERT(dev_flow);
6246         if (qrss) {
6247                 const struct rte_flow_attr q_attr = {
6248                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
6249                         .ingress = 1,
6250                 };
6251                 /* Internal PMD action to set register. */
6252                 struct mlx5_rte_flow_item_tag q_tag_spec = {
6253                         .data = qrss_id,
6254                         .id = REG_NON,
6255                 };
6256                 struct rte_flow_item q_items[] = {
6257                         {
6258                                 .type = (enum rte_flow_item_type)
6259                                         MLX5_RTE_FLOW_ITEM_TYPE_TAG,
6260                                 .spec = &q_tag_spec,
6261                                 .last = NULL,
6262                                 .mask = NULL,
6263                         },
6264                         {
6265                                 .type = RTE_FLOW_ITEM_TYPE_END,
6266                         },
6267                 };
6268                 struct rte_flow_action q_actions[] = {
6269                         {
6270                                 .type = qrss->type,
6271                                 .conf = qrss->conf,
6272                         },
6273                         {
6274                                 .type = RTE_FLOW_ACTION_TYPE_END,
6275                         },
6276                 };
6277                 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
6278
6279                 /*
6280                  * Configure the tag item only if there is no meter subflow.
6281                  * Since tag is already marked in the meter suffix subflow
6282                  * we can just use the meter suffix items as is.
6283                  */
6284                 if (qrss_id) {
6285                         /* Not meter subflow. */
6286                         MLX5_ASSERT(!mtr_sfx);
6287                         /*
6288                          * Put unique id in prefix flow due to it is destroyed
6289                          * after suffix flow and id will be freed after there
6290                          * is no actual flows with this id and identifier
6291                          * reallocation becomes possible (for example, for
6292                          * other flows in other threads).
6293                          */
6294                         dev_flow->handle->split_flow_id = qrss_id;
6295                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
6296                                                    error);
6297                         if (ret < 0)
6298                                 goto exit;
6299                         q_tag_spec.id = ret;
6300                 }
6301                 dev_flow = NULL;
6302                 /* Add suffix subflow to execute Q/RSS. */
6303                 flow_split_info->prefix_layers = layers;
6304                 flow_split_info->prefix_mark = 0;
6305                 flow_split_info->table_id = 0;
6306                 ret = flow_create_split_inner(dev, flow, &dev_flow,
6307                                               &q_attr, mtr_sfx ? items :
6308                                               q_items, q_actions,
6309                                               flow_split_info, error);
6310                 if (ret < 0)
6311                         goto exit;
6312                 /* qrss ID should be freed if failed. */
6313                 qrss_id = 0;
6314                 MLX5_ASSERT(dev_flow);
6315         }
6316
6317 exit:
6318         /*
6319          * We do not destroy the partially created sub_flows in case of error.
6320          * These ones are included into parent flow list and will be destroyed
6321          * by flow_drv_destroy.
6322          */
6323         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
6324                         qrss_id);
6325         mlx5_free(ext_actions);
6326         return ret;
6327 }
6328
6329 /**
6330  * Create meter internal drop flow with the original pattern.
6331  *
6332  * @param dev
6333  *   Pointer to Ethernet device.
6334  * @param[in] flow
6335  *   Parent flow structure pointer.
6336  * @param[in] attr
6337  *   Flow rule attributes.
6338  * @param[in] items
6339  *   Pattern specification (list terminated by the END pattern item).
6340  * @param[in] flow_split_info
6341  *   Pointer to flow split info structure.
6342  * @param[in] fm
6343  *   Pointer to flow meter structure.
6344  * @param[out] error
6345  *   Perform verbose error reporting if not NULL.
6346  * @return
6347  *   0 on success, negative value otherwise
6348  */
6349 static uint32_t
6350 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev,
6351                         struct rte_flow *flow,
6352                         const struct rte_flow_attr *attr,
6353                         const struct rte_flow_item items[],
6354                         struct mlx5_flow_split_info *flow_split_info,
6355                         struct mlx5_flow_meter_info *fm,
6356                         struct rte_flow_error *error)
6357 {
6358         struct mlx5_flow *dev_flow = NULL;
6359         struct rte_flow_attr drop_attr = *attr;
6360         struct rte_flow_action drop_actions[3];
6361         struct mlx5_flow_split_info drop_split_info = *flow_split_info;
6362
6363         MLX5_ASSERT(fm->drop_cnt);
6364         drop_actions[0].type =
6365                 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
6366         drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt;
6367         drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP;
6368         drop_actions[1].conf = NULL;
6369         drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END;
6370         drop_actions[2].conf = NULL;
6371         drop_split_info.external = false;
6372         drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT;
6373         drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP;
6374         drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER;
6375         return flow_create_split_inner(dev, flow, &dev_flow,
6376                                 &drop_attr, items, drop_actions,
6377                                 &drop_split_info, error);
6378 }
6379
6380 /**
6381  * The splitting for meter feature.
6382  *
6383  * - The meter flow will be split to two flows as prefix and
6384  *   suffix flow. The packets make sense only it pass the prefix
6385  *   meter action.
6386  *
6387  * - Reg_C_5 is used for the packet to match betweend prefix and
6388  *   suffix flow.
6389  *
6390  * @param dev
6391  *   Pointer to Ethernet device.
6392  * @param[in] flow
6393  *   Parent flow structure pointer.
6394  * @param[in] attr
6395  *   Flow rule attributes.
6396  * @param[in] items
6397  *   Pattern specification (list terminated by the END pattern item).
6398  * @param[in] actions
6399  *   Associated actions (list terminated by the END action).
6400  * @param[in] flow_split_info
6401  *   Pointer to flow split info structure.
6402  * @param[out] error
6403  *   Perform verbose error reporting if not NULL.
6404  * @return
6405  *   0 on success, negative value otherwise
6406  */
6407 static int
6408 flow_create_split_meter(struct rte_eth_dev *dev,
6409                         struct rte_flow *flow,
6410                         const struct rte_flow_attr *attr,
6411                         const struct rte_flow_item items[],
6412                         const struct rte_flow_action actions[],
6413                         struct mlx5_flow_split_info *flow_split_info,
6414                         struct rte_flow_error *error)
6415 {
6416         struct mlx5_priv *priv = dev->data->dev_private;
6417         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
6418         struct rte_flow_action *sfx_actions = NULL;
6419         struct rte_flow_action *pre_actions = NULL;
6420         struct rte_flow_item *sfx_items = NULL;
6421         struct mlx5_flow *dev_flow = NULL;
6422         struct rte_flow_attr sfx_attr = *attr;
6423         struct mlx5_flow_meter_info *fm = NULL;
6424         uint8_t skip_scale_restore;
6425         bool has_mtr = false;
6426         bool has_modify = false;
6427         bool set_mtr_reg = true;
6428         bool is_mtr_hierarchy = false;
6429         uint32_t meter_id = 0;
6430         uint32_t mtr_idx = 0;
6431         uint32_t mtr_flow_id = 0;
6432         size_t act_size;
6433         size_t item_size;
6434         int actions_n = 0;
6435         int ret = 0;
6436
6437         if (priv->mtr_en)
6438                 actions_n = flow_check_meter_action(dev, actions, &has_mtr,
6439                                                     &has_modify, &meter_id);
6440         if (has_mtr) {
6441                 if (flow->meter) {
6442                         fm = flow_dv_meter_find_by_idx(priv, flow->meter);
6443                         if (!fm)
6444                                 return rte_flow_error_set(error, EINVAL,
6445                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6446                                                 NULL, "Meter not found.");
6447                 } else {
6448                         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
6449                         if (!fm)
6450                                 return rte_flow_error_set(error, EINVAL,
6451                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6452                                                 NULL, "Meter not found.");
6453                         ret = mlx5_flow_meter_attach(priv, fm,
6454                                                      &sfx_attr, error);
6455                         if (ret)
6456                                 return -rte_errno;
6457                         flow->meter = mtr_idx;
6458                 }
6459                 MLX5_ASSERT(wks);
6460                 wks->fm = fm;
6461                 if (!fm->def_policy) {
6462                         wks->policy = mlx5_flow_meter_policy_find(dev,
6463                                                                   fm->policy_id,
6464                                                                   NULL);
6465                         MLX5_ASSERT(wks->policy);
6466                         if (wks->policy->mark)
6467                                 wks->mark = 1;
6468                         if (wks->policy->is_hierarchy) {
6469                                 wks->final_policy =
6470                                 mlx5_flow_meter_hierarchy_get_final_policy(dev,
6471                                                                 wks->policy);
6472                                 if (!wks->final_policy)
6473                                         return rte_flow_error_set(error,
6474                                         EINVAL,
6475                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
6476                                 "Failed to find terminal policy of hierarchy.");
6477                                 is_mtr_hierarchy = true;
6478                         }
6479                 }
6480                 /*
6481                  * If it isn't default-policy Meter, and
6482                  * 1. There's no action in flow to change
6483                  *    packet (modify/encap/decap etc.), OR
6484                  * 2. No drop count needed for this meter.
6485                  * 3. It's not meter hierarchy.
6486                  * Then no need to use regC to save meter id anymore.
6487                  */
6488                 if (!fm->def_policy && !is_mtr_hierarchy &&
6489                     (!has_modify || !fm->drop_cnt))
6490                         set_mtr_reg = false;
6491                 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */
6492 #define METER_PREFIX_ACTION 7
6493                 act_size = (sizeof(struct rte_flow_action) *
6494                             (actions_n + METER_PREFIX_ACTION)) +
6495                            sizeof(struct mlx5_rte_flow_action_set_tag);
6496                 /* Suffix items: tag, vlan, port id, end. */
6497 #define METER_SUFFIX_ITEM 4
6498                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
6499                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
6500                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
6501                                           0, SOCKET_ID_ANY);
6502                 if (!sfx_actions)
6503                         return rte_flow_error_set(error, ENOMEM,
6504                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6505                                                   NULL, "no memory to split "
6506                                                   "meter flow");
6507                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
6508                              act_size);
6509                 /* There's no suffix flow for meter of non-default policy. */
6510                 if (!fm->def_policy)
6511                         pre_actions = sfx_actions + 1;
6512                 else
6513                         pre_actions = sfx_actions + actions_n;
6514                 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr,
6515                                             items, sfx_items, actions,
6516                                             sfx_actions, pre_actions,
6517                                             (set_mtr_reg ? &mtr_flow_id : NULL),
6518                                             error);
6519                 if (ret) {
6520                         ret = -rte_errno;
6521                         goto exit;
6522                 }
6523                 /* Add the prefix subflow. */
6524                 skip_scale_restore = flow_split_info->skip_scale;
6525                 flow_split_info->skip_scale |=
6526                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
6527                 ret = flow_create_split_inner(dev, flow, &dev_flow,
6528                                               attr, items, pre_actions,
6529                                               flow_split_info, error);
6530                 flow_split_info->skip_scale = skip_scale_restore;
6531                 if (ret) {
6532                         if (mtr_flow_id)
6533                                 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id);
6534                         ret = -rte_errno;
6535                         goto exit;
6536                 }
6537                 if (mtr_flow_id) {
6538                         dev_flow->handle->split_flow_id = mtr_flow_id;
6539                         dev_flow->handle->is_meter_flow_id = 1;
6540                 }
6541                 if (!fm->def_policy) {
6542                         if (!set_mtr_reg && fm->drop_cnt)
6543                                 ret =
6544                         flow_meter_create_drop_flow_with_org_pattern(dev, flow,
6545                                                         &sfx_attr, items,
6546                                                         flow_split_info,
6547                                                         fm, error);
6548                         goto exit;
6549                 }
6550                 /* Setting the sfx group atrr. */
6551                 sfx_attr.group = sfx_attr.transfer ?
6552                                 (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
6553                                  MLX5_FLOW_TABLE_LEVEL_METER;
6554                 flow_split_info->prefix_layers =
6555                                 flow_get_prefix_layer_flags(dev_flow);
6556                 flow_split_info->prefix_mark |= wks->mark;
6557                 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX;
6558         }
6559         /* Add the prefix subflow. */
6560         ret = flow_create_split_metadata(dev, flow,
6561                                          &sfx_attr, sfx_items ?
6562                                          sfx_items : items,
6563                                          sfx_actions ? sfx_actions : actions,
6564                                          flow_split_info, error);
6565 exit:
6566         if (sfx_actions)
6567                 mlx5_free(sfx_actions);
6568         return ret;
6569 }
6570
6571 /**
6572  * The splitting for sample feature.
6573  *
6574  * Once Sample action is detected in the action list, the flow actions should
6575  * be split into prefix sub flow and suffix sub flow.
6576  *
6577  * The original items remain in the prefix sub flow, all actions preceding the
6578  * sample action and the sample action itself will be copied to the prefix
6579  * sub flow, the actions following the sample action will be copied to the
6580  * suffix sub flow, Queue action always be located in the suffix sub flow.
6581  *
6582  * In order to make the packet from prefix sub flow matches with suffix sub
6583  * flow, an extra tag action be added into prefix sub flow, and the suffix sub
6584  * flow uses tag item with the unique flow id.
6585  *
6586  * @param dev
6587  *   Pointer to Ethernet device.
6588  * @param[in] flow
6589  *   Parent flow structure pointer.
6590  * @param[in] attr
6591  *   Flow rule attributes.
6592  * @param[in] items
6593  *   Pattern specification (list terminated by the END pattern item).
6594  * @param[in] actions
6595  *   Associated actions (list terminated by the END action).
6596  * @param[in] flow_split_info
6597  *   Pointer to flow split info structure.
6598  * @param[out] error
6599  *   Perform verbose error reporting if not NULL.
6600  * @return
6601  *   0 on success, negative value otherwise
6602  */
6603 static int
6604 flow_create_split_sample(struct rte_eth_dev *dev,
6605                          struct rte_flow *flow,
6606                          const struct rte_flow_attr *attr,
6607                          const struct rte_flow_item items[],
6608                          const struct rte_flow_action actions[],
6609                          struct mlx5_flow_split_info *flow_split_info,
6610                          struct rte_flow_error *error)
6611 {
6612         struct mlx5_priv *priv = dev->data->dev_private;
6613         struct rte_flow_action *sfx_actions = NULL;
6614         struct rte_flow_action *pre_actions = NULL;
6615         struct rte_flow_item *sfx_items = NULL;
6616         struct mlx5_flow *dev_flow = NULL;
6617         struct rte_flow_attr sfx_attr = *attr;
6618 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
6619         struct mlx5_flow_dv_sample_resource *sample_res;
6620         struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
6621         struct mlx5_flow_tbl_resource *sfx_tbl;
6622         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
6623 #endif
6624         size_t act_size;
6625         size_t item_size;
6626         uint32_t fdb_tx = 0;
6627         int32_t tag_id = 0;
6628         int actions_n = 0;
6629         int sample_action_pos;
6630         int qrss_action_pos;
6631         int add_tag = 0;
6632         int modify_after_mirror = 0;
6633         uint16_t jump_table = 0;
6634         const uint32_t next_ft_step = 1;
6635         int ret = 0;
6636
6637         if (priv->sampler_en)
6638                 actions_n = flow_check_match_action(actions, attr,
6639                                         RTE_FLOW_ACTION_TYPE_SAMPLE,
6640                                         &sample_action_pos, &qrss_action_pos,
6641                                         &modify_after_mirror);
6642         if (actions_n) {
6643                 /* The prefix actions must includes sample, tag, end. */
6644                 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
6645                            + sizeof(struct mlx5_rte_flow_action_set_tag);
6646                 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
6647                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
6648                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
6649                                           item_size), 0, SOCKET_ID_ANY);
6650                 if (!sfx_actions)
6651                         return rte_flow_error_set(error, ENOMEM,
6652                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6653                                                   NULL, "no memory to split "
6654                                                   "sample flow");
6655                 /* The representor_id is UINT16_MAX for uplink. */
6656                 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX);
6657                 /*
6658                  * When reg_c_preserve is set, metadata registers Cx preserve
6659                  * their value even through packet duplication.
6660                  */
6661                 add_tag = (!fdb_tx ||
6662                            priv->sh->cdev->config.hca_attr.reg_c_preserve);
6663                 if (add_tag)
6664                         sfx_items = (struct rte_flow_item *)((char *)sfx_actions
6665                                         + act_size);
6666                 if (modify_after_mirror)
6667                         jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR +
6668                                      next_ft_step;
6669                 pre_actions = sfx_actions + actions_n;
6670                 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items,
6671                                                 actions, sfx_actions,
6672                                                 pre_actions, actions_n,
6673                                                 sample_action_pos,
6674                                                 qrss_action_pos, jump_table,
6675                                                 error);
6676                 if (tag_id < 0 || (add_tag && !tag_id)) {
6677                         ret = -rte_errno;
6678                         goto exit;
6679                 }
6680                 if (modify_after_mirror)
6681                         flow_split_info->skip_scale =
6682                                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
6683                 /* Add the prefix subflow. */
6684                 ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
6685                                               items, pre_actions,
6686                                               flow_split_info, error);
6687                 if (ret) {
6688                         ret = -rte_errno;
6689                         goto exit;
6690                 }
6691                 dev_flow->handle->split_flow_id = tag_id;
6692 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
6693                 if (!modify_after_mirror) {
6694                         /* Set the sfx group attr. */
6695                         sample_res = (struct mlx5_flow_dv_sample_resource *)
6696                                                 dev_flow->dv.sample_res;
6697                         sfx_tbl = (struct mlx5_flow_tbl_resource *)
6698                                                 sample_res->normal_path_tbl;
6699                         sfx_tbl_data = container_of(sfx_tbl,
6700                                                 struct mlx5_flow_tbl_data_entry,
6701                                                 tbl);
6702                         sfx_attr.group = sfx_attr.transfer ?
6703                         (sfx_tbl_data->level - 1) : sfx_tbl_data->level;
6704                 } else {
6705                         MLX5_ASSERT(attr->transfer);
6706                         sfx_attr.group = jump_table;
6707                 }
6708                 flow_split_info->prefix_layers =
6709                                 flow_get_prefix_layer_flags(dev_flow);
6710                 MLX5_ASSERT(wks);
6711                 flow_split_info->prefix_mark |= wks->mark;
6712                 /* Suffix group level already be scaled with factor, set
6713                  * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale
6714                  * again in translation.
6715                  */
6716                 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT;
6717 #endif
6718         }
6719         /* Add the suffix subflow. */
6720         ret = flow_create_split_meter(dev, flow, &sfx_attr,
6721                                       sfx_items ? sfx_items : items,
6722                                       sfx_actions ? sfx_actions : actions,
6723                                       flow_split_info, error);
6724 exit:
6725         if (sfx_actions)
6726                 mlx5_free(sfx_actions);
6727         return ret;
6728 }
6729
6730 /**
6731  * Split the flow to subflow set. The splitters might be linked
6732  * in the chain, like this:
6733  * flow_create_split_outer() calls:
6734  *   flow_create_split_meter() calls:
6735  *     flow_create_split_metadata(meter_subflow_0) calls:
6736  *       flow_create_split_inner(metadata_subflow_0)
6737  *       flow_create_split_inner(metadata_subflow_1)
6738  *       flow_create_split_inner(metadata_subflow_2)
6739  *     flow_create_split_metadata(meter_subflow_1) calls:
6740  *       flow_create_split_inner(metadata_subflow_0)
6741  *       flow_create_split_inner(metadata_subflow_1)
6742  *       flow_create_split_inner(metadata_subflow_2)
6743  *
6744  * This provide flexible way to add new levels of flow splitting.
6745  * The all of successfully created subflows are included to the
6746  * parent flow dev_flow list.
6747  *
6748  * @param dev
6749  *   Pointer to Ethernet device.
6750  * @param[in] flow
6751  *   Parent flow structure pointer.
6752  * @param[in] attr
6753  *   Flow rule attributes.
6754  * @param[in] items
6755  *   Pattern specification (list terminated by the END pattern item).
6756  * @param[in] actions
6757  *   Associated actions (list terminated by the END action).
6758  * @param[in] flow_split_info
6759  *   Pointer to flow split info structure.
6760  * @param[out] error
6761  *   Perform verbose error reporting if not NULL.
6762  * @return
6763  *   0 on success, negative value otherwise
6764  */
6765 static int
6766 flow_create_split_outer(struct rte_eth_dev *dev,
6767                         struct rte_flow *flow,
6768                         const struct rte_flow_attr *attr,
6769                         const struct rte_flow_item items[],
6770                         const struct rte_flow_action actions[],
6771                         struct mlx5_flow_split_info *flow_split_info,
6772                         struct rte_flow_error *error)
6773 {
6774         int ret;
6775
6776         ret = flow_create_split_sample(dev, flow, attr, items,
6777                                        actions, flow_split_info, error);
6778         MLX5_ASSERT(ret <= 0);
6779         return ret;
6780 }
6781
6782 static inline struct mlx5_flow_tunnel *
6783 flow_tunnel_from_rule(const struct mlx5_flow *flow)
6784 {
6785         struct mlx5_flow_tunnel *tunnel;
6786
6787 #pragma GCC diagnostic push
6788 #pragma GCC diagnostic ignored "-Wcast-qual"
6789         tunnel = (typeof(tunnel))flow->tunnel;
6790 #pragma GCC diagnostic pop
6791
6792         return tunnel;
6793 }
6794
6795 /**
6796  * Adjust flow RSS workspace if needed.
6797  *
6798  * @param wks
6799  *   Pointer to thread flow work space.
6800  * @param rss_desc
6801  *   Pointer to RSS descriptor.
6802  * @param[in] nrssq_num
6803  *   New RSS queue number.
6804  *
6805  * @return
6806  *   0 on success, -1 otherwise and rte_errno is set.
6807  */
6808 static int
6809 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
6810                           struct mlx5_flow_rss_desc *rss_desc,
6811                           uint32_t nrssq_num)
6812 {
6813         if (likely(nrssq_num <= wks->rssq_num))
6814                 return 0;
6815         rss_desc->queue = realloc(rss_desc->queue,
6816                           sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2));
6817         if (!rss_desc->queue) {
6818                 rte_errno = ENOMEM;
6819                 return -1;
6820         }
6821         wks->rssq_num = RTE_ALIGN(nrssq_num, 2);
6822         return 0;
6823 }
6824
6825 /**
6826  * Create a flow and add it to @p list.
6827  *
6828  * @param dev
6829  *   Pointer to Ethernet device.
6830  * @param list
6831  *   Pointer to a TAILQ flow list. If this parameter NULL,
6832  *   no list insertion occurred, flow is just created,
6833  *   this is caller's responsibility to track the
6834  *   created flow.
6835  * @param[in] attr
6836  *   Flow rule attributes.
6837  * @param[in] items
6838  *   Pattern specification (list terminated by the END pattern item).
6839  * @param[in] actions
6840  *   Associated actions (list terminated by the END action).
6841  * @param[in] external
6842  *   This flow rule is created by request external to PMD.
6843  * @param[out] error
6844  *   Perform verbose error reporting if not NULL.
6845  *
6846  * @return
6847  *   A flow index on success, 0 otherwise and rte_errno is set.
6848  */
6849 static uint32_t
6850 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
6851                  const struct rte_flow_attr *attr,
6852                  const struct rte_flow_item items[],
6853                  const struct rte_flow_action original_actions[],
6854                  bool external, struct rte_flow_error *error)
6855 {
6856         struct mlx5_priv *priv = dev->data->dev_private;
6857         struct rte_flow *flow = NULL;
6858         struct mlx5_flow *dev_flow;
6859         const struct rte_flow_action_rss *rss = NULL;
6860         struct mlx5_translated_action_handle
6861                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
6862         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
6863         union {
6864                 struct mlx5_flow_expand_rss buf;
6865                 uint8_t buffer[4096];
6866         } expand_buffer;
6867         union {
6868                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6869                 uint8_t buffer[2048];
6870         } actions_rx;
6871         union {
6872                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6873                 uint8_t buffer[2048];
6874         } actions_hairpin_tx;
6875         union {
6876                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
6877                 uint8_t buffer[2048];
6878         } items_tx;
6879         struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
6880         struct mlx5_flow_rss_desc *rss_desc;
6881         const struct rte_flow_action *p_actions_rx;
6882         uint32_t i;
6883         uint32_t idx = 0;
6884         int hairpin_flow;
6885         struct rte_flow_attr attr_tx = { .priority = 0 };
6886         const struct rte_flow_action *actions;
6887         struct rte_flow_action *translated_actions = NULL;
6888         struct mlx5_flow_tunnel *tunnel;
6889         struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
6890         struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace();
6891         struct mlx5_flow_split_info flow_split_info = {
6892                 .external = !!external,
6893                 .skip_scale = 0,
6894                 .flow_idx = 0,
6895                 .prefix_mark = 0,
6896                 .prefix_layers = 0,
6897                 .table_id = 0
6898         };
6899         int ret;
6900
6901         MLX5_ASSERT(wks);
6902         rss_desc = &wks->rss_desc;
6903         ret = flow_action_handles_translate(dev, original_actions,
6904                                             indir_actions,
6905                                             &indir_actions_n,
6906                                             &translated_actions, error);
6907         if (ret < 0) {
6908                 MLX5_ASSERT(translated_actions == NULL);
6909                 return 0;
6910         }
6911         actions = translated_actions ? translated_actions : original_actions;
6912         p_actions_rx = actions;
6913         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
6914         ret = flow_drv_validate(dev, attr, items, p_actions_rx,
6915                                 external, hairpin_flow, error);
6916         if (ret < 0)
6917                 goto error_before_hairpin_split;
6918         flow = mlx5_ipool_zmalloc(priv->flows[type], &idx);
6919         if (!flow) {
6920                 rte_errno = ENOMEM;
6921                 goto error_before_hairpin_split;
6922         }
6923         if (hairpin_flow > 0) {
6924                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
6925                         rte_errno = EINVAL;
6926                         goto error_before_hairpin_split;
6927                 }
6928                 flow_hairpin_split(dev, actions, actions_rx.actions,
6929                                    actions_hairpin_tx.actions, items_tx.items,
6930                                    idx);
6931                 p_actions_rx = actions_rx.actions;
6932         }
6933         flow_split_info.flow_idx = idx;
6934         flow->drv_type = flow_get_drv_type(dev, attr);
6935         MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
6936                     flow->drv_type < MLX5_FLOW_TYPE_MAX);
6937         memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
6938         /* RSS Action only works on NIC RX domain */
6939         if (attr->ingress && !attr->transfer)
6940                 rss = flow_get_rss_action(dev, p_actions_rx);
6941         if (rss) {
6942                 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num))
6943                         return 0;
6944                 /*
6945                  * The following information is required by
6946                  * mlx5_flow_hashfields_adjust() in advance.
6947                  */
6948                 rss_desc->level = rss->level;
6949                 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */
6950                 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types;
6951         }
6952         flow->dev_handles = 0;
6953         if (rss && rss->types) {
6954                 unsigned int graph_root;
6955
6956                 graph_root = find_graph_root(rss->level);
6957                 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
6958                                            items, rss->types,
6959                                            mlx5_support_expansion, graph_root);
6960                 MLX5_ASSERT(ret > 0 &&
6961                        (unsigned int)ret < sizeof(expand_buffer.buffer));
6962                 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) {
6963                         for (i = 0; i < buf->entries; ++i)
6964                                 mlx5_dbg__print_pattern(buf->entry[i].pattern);
6965                 }
6966         } else {
6967                 buf->entries = 1;
6968                 buf->entry[0].pattern = (void *)(uintptr_t)items;
6969         }
6970         rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions,
6971                                                       indir_actions_n);
6972         for (i = 0; i < buf->entries; ++i) {
6973                 /* Initialize flow split data. */
6974                 flow_split_info.prefix_layers = 0;
6975                 flow_split_info.prefix_mark = 0;
6976                 flow_split_info.skip_scale = 0;
6977                 /*
6978                  * The splitter may create multiple dev_flows,
6979                  * depending on configuration. In the simplest
6980                  * case it just creates unmodified original flow.
6981                  */
6982                 ret = flow_create_split_outer(dev, flow, attr,
6983                                               buf->entry[i].pattern,
6984                                               p_actions_rx, &flow_split_info,
6985                                               error);
6986                 if (ret < 0)
6987                         goto error;
6988                 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) {
6989                         ret = flow_tunnel_add_default_miss(dev, flow, attr,
6990                                                            p_actions_rx,
6991                                                            idx,
6992                                                            wks->flows[0].tunnel,
6993                                                            &default_miss_ctx,
6994                                                            error);
6995                         if (ret < 0) {
6996                                 mlx5_free(default_miss_ctx.queue);
6997                                 goto error;
6998                         }
6999                 }
7000         }
7001         /* Create the tx flow. */
7002         if (hairpin_flow) {
7003                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
7004                 attr_tx.ingress = 0;
7005                 attr_tx.egress = 1;
7006                 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
7007                                          actions_hairpin_tx.actions,
7008                                          idx, error);
7009                 if (!dev_flow)
7010                         goto error;
7011                 dev_flow->flow = flow;
7012                 dev_flow->external = 0;
7013                 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
7014                               dev_flow->handle, next);
7015                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
7016                                          items_tx.items,
7017                                          actions_hairpin_tx.actions, error);
7018                 if (ret < 0)
7019                         goto error;
7020         }
7021         /*
7022          * Update the metadata register copy table. If extensive
7023          * metadata feature is enabled and registers are supported
7024          * we might create the extra rte_flow for each unique
7025          * MARK/FLAG action ID.
7026          *
7027          * The table is updated for ingress Flows only, because
7028          * the egress Flows belong to the different device and
7029          * copy table should be updated in peer NIC Rx domain.
7030          */
7031         if (attr->ingress &&
7032             (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
7033                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
7034                 if (ret)
7035                         goto error;
7036         }
7037         /*
7038          * If the flow is external (from application) OR device is started,
7039          * OR mreg discover, then apply immediately.
7040          */
7041         if (external || dev->data->dev_started ||
7042             (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP &&
7043              attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) {
7044                 ret = flow_drv_apply(dev, flow, error);
7045                 if (ret < 0)
7046                         goto error;
7047         }
7048         flow->type = type;
7049         flow_rxq_flags_set(dev, flow);
7050         rte_free(translated_actions);
7051         tunnel = flow_tunnel_from_rule(wks->flows);
7052         if (tunnel) {
7053                 flow->tunnel = 1;
7054                 flow->tunnel_id = tunnel->tunnel_id;
7055                 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED);
7056                 mlx5_free(default_miss_ctx.queue);
7057         }
7058         mlx5_flow_pop_thread_workspace();
7059         return idx;
7060 error:
7061         MLX5_ASSERT(flow);
7062         ret = rte_errno; /* Save rte_errno before cleanup. */
7063         flow_mreg_del_copy_action(dev, flow);
7064         flow_drv_destroy(dev, flow);
7065         if (rss_desc->shared_rss)
7066                 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *)
7067                         mlx5_ipool_get
7068                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
7069                         rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
7070         mlx5_ipool_free(priv->flows[type], idx);
7071         rte_errno = ret; /* Restore rte_errno. */
7072         ret = rte_errno;
7073         rte_errno = ret;
7074         mlx5_flow_pop_thread_workspace();
7075 error_before_hairpin_split:
7076         rte_free(translated_actions);
7077         return 0;
7078 }
7079
7080 /**
7081  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
7082  * incoming packets to table 1.
7083  *
7084  * Other flow rules, requested for group n, will be created in
7085  * e-switch table n+1.
7086  * Jump action to e-switch group n will be created to group n+1.
7087  *
7088  * Used when working in switchdev mode, to utilise advantages of table 1
7089  * and above.
7090  *
7091  * @param dev
7092  *   Pointer to Ethernet device.
7093  *
7094  * @return
7095  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
7096  */
7097 struct rte_flow *
7098 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
7099 {
7100         const struct rte_flow_attr attr = {
7101                 .group = 0,
7102                 .priority = 0,
7103                 .ingress = 1,
7104                 .egress = 0,
7105                 .transfer = 1,
7106         };
7107         const struct rte_flow_item pattern = {
7108                 .type = RTE_FLOW_ITEM_TYPE_END,
7109         };
7110         struct rte_flow_action_jump jump = {
7111                 .group = 1,
7112         };
7113         const struct rte_flow_action actions[] = {
7114                 {
7115                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
7116                         .conf = &jump,
7117                 },
7118                 {
7119                         .type = RTE_FLOW_ACTION_TYPE_END,
7120                 },
7121         };
7122         struct rte_flow_error error;
7123
7124         return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7125                                                    &attr, &pattern,
7126                                                    actions, false, &error);
7127 }
7128
7129 /**
7130  * Create a dedicated flow rule on e-switch table 1, matches ESW manager
7131  * and sq number, directs all packets to peer vport.
7132  *
7133  * @param dev
7134  *   Pointer to Ethernet device.
7135  * @param txq
7136  *   Txq index.
7137  *
7138  * @return
7139  *   Flow ID on success, 0 otherwise and rte_errno is set.
7140  */
7141 uint32_t
7142 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq)
7143 {
7144         struct rte_flow_attr attr = {
7145                 .group = 0,
7146                 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
7147                 .ingress = 1,
7148                 .egress = 0,
7149                 .transfer = 1,
7150         };
7151         struct rte_flow_item_port_id port_spec = {
7152                 .id = MLX5_PORT_ESW_MGR,
7153         };
7154         struct mlx5_rte_flow_item_tx_queue txq_spec = {
7155                 .queue = txq,
7156         };
7157         struct rte_flow_item pattern[] = {
7158                 {
7159                         .type = RTE_FLOW_ITEM_TYPE_PORT_ID,
7160                         .spec = &port_spec,
7161                 },
7162                 {
7163                         .type = (enum rte_flow_item_type)
7164                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
7165                         .spec = &txq_spec,
7166                 },
7167                 {
7168                         .type = RTE_FLOW_ITEM_TYPE_END,
7169                 },
7170         };
7171         struct rte_flow_action_jump jump = {
7172                 .group = 1,
7173         };
7174         struct rte_flow_action_port_id port = {
7175                 .id = dev->data->port_id,
7176         };
7177         struct rte_flow_action actions[] = {
7178                 {
7179                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
7180                         .conf = &jump,
7181                 },
7182                 {
7183                         .type = RTE_FLOW_ACTION_TYPE_END,
7184                 },
7185         };
7186         struct rte_flow_error error;
7187
7188         /*
7189          * Creates group 0, highest priority jump flow.
7190          * Matches txq to bypass kernel packets.
7191          */
7192         if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions,
7193                              false, &error) == 0)
7194                 return 0;
7195         /* Create group 1, lowest priority redirect flow for txq. */
7196         attr.group = 1;
7197         actions[0].conf = &port;
7198         actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID;
7199         return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern,
7200                                 actions, false, &error);
7201 }
7202
7203 /**
7204  * Validate a flow supported by the NIC.
7205  *
7206  * @see rte_flow_validate()
7207  * @see rte_flow_ops
7208  */
7209 int
7210 mlx5_flow_validate(struct rte_eth_dev *dev,
7211                    const struct rte_flow_attr *attr,
7212                    const struct rte_flow_item items[],
7213                    const struct rte_flow_action original_actions[],
7214                    struct rte_flow_error *error)
7215 {
7216         int hairpin_flow;
7217         struct mlx5_translated_action_handle
7218                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
7219         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
7220         const struct rte_flow_action *actions;
7221         struct rte_flow_action *translated_actions = NULL;
7222         int ret = flow_action_handles_translate(dev, original_actions,
7223                                                 indir_actions,
7224                                                 &indir_actions_n,
7225                                                 &translated_actions, error);
7226
7227         if (ret)
7228                 return ret;
7229         actions = translated_actions ? translated_actions : original_actions;
7230         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
7231         ret = flow_drv_validate(dev, attr, items, actions,
7232                                 true, hairpin_flow, error);
7233         rte_free(translated_actions);
7234         return ret;
7235 }
7236
7237 /**
7238  * Create a flow.
7239  *
7240  * @see rte_flow_create()
7241  * @see rte_flow_ops
7242  */
7243 struct rte_flow *
7244 mlx5_flow_create(struct rte_eth_dev *dev,
7245                  const struct rte_flow_attr *attr,
7246                  const struct rte_flow_item items[],
7247                  const struct rte_flow_action actions[],
7248                  struct rte_flow_error *error)
7249 {
7250         struct mlx5_priv *priv = dev->data->dev_private;
7251
7252         if (priv->sh->config.dv_flow_en == 2) {
7253                 rte_flow_error_set(error, ENOTSUP,
7254                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7255                           NULL,
7256                           "Flow non-Q creation not supported");
7257                 return NULL;
7258         }
7259         /*
7260          * If the device is not started yet, it is not allowed to created a
7261          * flow from application. PMD default flows and traffic control flows
7262          * are not affected.
7263          */
7264         if (unlikely(!dev->data->dev_started)) {
7265                 DRV_LOG(DEBUG, "port %u is not started when "
7266                         "inserting a flow", dev->data->port_id);
7267                 rte_flow_error_set(error, ENODEV,
7268                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7269                                    NULL,
7270                                    "port not started");
7271                 return NULL;
7272         }
7273
7274         return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN,
7275                                                    attr, items, actions,
7276                                                    true, error);
7277 }
7278
7279 /**
7280  * Destroy a flow in a list.
7281  *
7282  * @param dev
7283  *   Pointer to Ethernet device.
7284  * @param[in] flow_idx
7285  *   Index of flow to destroy.
7286  */
7287 static void
7288 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
7289                   uint32_t flow_idx)
7290 {
7291         struct mlx5_priv *priv = dev->data->dev_private;
7292         struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx);
7293
7294         if (!flow)
7295                 return;
7296         MLX5_ASSERT(flow->type == type);
7297         /*
7298          * Update RX queue flags only if port is started, otherwise it is
7299          * already clean.
7300          */
7301         if (dev->data->dev_started)
7302                 flow_rxq_flags_trim(dev, flow);
7303         flow_drv_destroy(dev, flow);
7304         if (flow->tunnel) {
7305                 struct mlx5_flow_tunnel *tunnel;
7306
7307                 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
7308                 RTE_VERIFY(tunnel);
7309                 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
7310                         mlx5_flow_tunnel_free(dev, tunnel);
7311         }
7312         flow_mreg_del_copy_action(dev, flow);
7313         mlx5_ipool_free(priv->flows[type], flow_idx);
7314 }
7315
7316 /**
7317  * Destroy all flows.
7318  *
7319  * @param dev
7320  *   Pointer to Ethernet device.
7321  * @param type
7322  *   Flow type to be flushed.
7323  * @param active
7324  *   If flushing is called actively.
7325  */
7326 void
7327 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
7328                      bool active)
7329 {
7330         struct mlx5_priv *priv = dev->data->dev_private;
7331         uint32_t num_flushed = 0, fidx = 1;
7332         struct rte_flow *flow;
7333
7334 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
7335         if (priv->sh->config.dv_flow_en == 2 &&
7336             type == MLX5_FLOW_TYPE_GEN) {
7337                 flow_hw_q_flow_flush(dev, NULL);
7338                 return;
7339         }
7340 #endif
7341
7342         MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) {
7343                 flow_list_destroy(dev, type, fidx);
7344                 num_flushed++;
7345         }
7346         if (active) {
7347                 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
7348                         dev->data->port_id, num_flushed);
7349         }
7350 }
7351
7352 /**
7353  * Stop all default actions for flows.
7354  *
7355  * @param dev
7356  *   Pointer to Ethernet device.
7357  */
7358 void
7359 mlx5_flow_stop_default(struct rte_eth_dev *dev)
7360 {
7361         flow_mreg_del_default_copy_action(dev);
7362         flow_rxq_flags_clear(dev);
7363 }
7364
7365 /**
7366  * Start all default actions for flows.
7367  *
7368  * @param dev
7369  *   Pointer to Ethernet device.
7370  * @return
7371  *   0 on success, a negative errno value otherwise and rte_errno is set.
7372  */
7373 int
7374 mlx5_flow_start_default(struct rte_eth_dev *dev)
7375 {
7376         struct rte_flow_error error;
7377
7378         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
7379         return flow_mreg_add_default_copy_action(dev, &error);
7380 }
7381
7382 /**
7383  * Release key of thread specific flow workspace data.
7384  */
7385 void
7386 flow_release_workspace(void *data)
7387 {
7388         struct mlx5_flow_workspace *wks = data;
7389         struct mlx5_flow_workspace *next;
7390
7391         while (wks) {
7392                 next = wks->next;
7393                 free(wks->rss_desc.queue);
7394                 free(wks);
7395                 wks = next;
7396         }
7397 }
7398
7399 /**
7400  * Get thread specific current flow workspace.
7401  *
7402  * @return pointer to thread specific flow workspace data, NULL on error.
7403  */
7404 struct mlx5_flow_workspace*
7405 mlx5_flow_get_thread_workspace(void)
7406 {
7407         struct mlx5_flow_workspace *data;
7408
7409         data = mlx5_flow_os_get_specific_workspace();
7410         MLX5_ASSERT(data && data->inuse);
7411         if (!data || !data->inuse)
7412                 DRV_LOG(ERR, "flow workspace not initialized.");
7413         return data;
7414 }
7415
7416 /**
7417  * Allocate and init new flow workspace.
7418  *
7419  * @return pointer to flow workspace data, NULL on error.
7420  */
7421 static struct mlx5_flow_workspace*
7422 flow_alloc_thread_workspace(void)
7423 {
7424         struct mlx5_flow_workspace *data = calloc(1, sizeof(*data));
7425
7426         if (!data) {
7427                 DRV_LOG(ERR, "Failed to allocate flow workspace "
7428                         "memory.");
7429                 return NULL;
7430         }
7431         data->rss_desc.queue = calloc(1,
7432                         sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM);
7433         if (!data->rss_desc.queue)
7434                 goto err;
7435         data->rssq_num = MLX5_RSSQ_DEFAULT_NUM;
7436         return data;
7437 err:
7438         free(data->rss_desc.queue);
7439         free(data);
7440         return NULL;
7441 }
7442
7443 /**
7444  * Get new thread specific flow workspace.
7445  *
7446  * If current workspace inuse, create new one and set as current.
7447  *
7448  * @return pointer to thread specific flow workspace data, NULL on error.
7449  */
7450 static struct mlx5_flow_workspace*
7451 mlx5_flow_push_thread_workspace(void)
7452 {
7453         struct mlx5_flow_workspace *curr;
7454         struct mlx5_flow_workspace *data;
7455
7456         curr = mlx5_flow_os_get_specific_workspace();
7457         if (!curr) {
7458                 data = flow_alloc_thread_workspace();
7459                 if (!data)
7460                         return NULL;
7461         } else if (!curr->inuse) {
7462                 data = curr;
7463         } else if (curr->next) {
7464                 data = curr->next;
7465         } else {
7466                 data = flow_alloc_thread_workspace();
7467                 if (!data)
7468                         return NULL;
7469                 curr->next = data;
7470                 data->prev = curr;
7471         }
7472         data->inuse = 1;
7473         data->flow_idx = 0;
7474         /* Set as current workspace */
7475         if (mlx5_flow_os_set_specific_workspace(data))
7476                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
7477         return data;
7478 }
7479
7480 /**
7481  * Close current thread specific flow workspace.
7482  *
7483  * If previous workspace available, set it as current.
7484  *
7485  * @return pointer to thread specific flow workspace data, NULL on error.
7486  */
7487 static void
7488 mlx5_flow_pop_thread_workspace(void)
7489 {
7490         struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace();
7491
7492         if (!data)
7493                 return;
7494         if (!data->inuse) {
7495                 DRV_LOG(ERR, "Failed to close unused flow workspace.");
7496                 return;
7497         }
7498         data->inuse = 0;
7499         if (!data->prev)
7500                 return;
7501         if (mlx5_flow_os_set_specific_workspace(data->prev))
7502                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
7503 }
7504
7505 /**
7506  * Verify the flow list is empty
7507  *
7508  * @param dev
7509  *  Pointer to Ethernet device.
7510  *
7511  * @return the number of flows not released.
7512  */
7513 int
7514 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused)
7515 {
7516         struct mlx5_priv *priv = dev->data->dev_private;
7517         struct rte_flow *flow;
7518         uint32_t idx = 0;
7519         int ret = 0, i;
7520
7521         for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
7522                 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) {
7523                         DRV_LOG(DEBUG, "port %u flow %p still referenced",
7524                                 dev->data->port_id, (void *)flow);
7525                         ret++;
7526                 }
7527         }
7528         return ret;
7529 }
7530
7531 /**
7532  * Enable default hairpin egress flow.
7533  *
7534  * @param dev
7535  *   Pointer to Ethernet device.
7536  * @param queue
7537  *   The queue index.
7538  *
7539  * @return
7540  *   0 on success, a negative errno value otherwise and rte_errno is set.
7541  */
7542 int
7543 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
7544                             uint32_t queue)
7545 {
7546         const struct rte_flow_attr attr = {
7547                 .egress = 1,
7548                 .priority = 0,
7549         };
7550         struct mlx5_rte_flow_item_tx_queue queue_spec = {
7551                 .queue = queue,
7552         };
7553         struct mlx5_rte_flow_item_tx_queue queue_mask = {
7554                 .queue = UINT32_MAX,
7555         };
7556         struct rte_flow_item items[] = {
7557                 {
7558                         .type = (enum rte_flow_item_type)
7559                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
7560                         .spec = &queue_spec,
7561                         .last = NULL,
7562                         .mask = &queue_mask,
7563                 },
7564                 {
7565                         .type = RTE_FLOW_ITEM_TYPE_END,
7566                 },
7567         };
7568         struct rte_flow_action_jump jump = {
7569                 .group = MLX5_HAIRPIN_TX_TABLE,
7570         };
7571         struct rte_flow_action actions[2];
7572         uint32_t flow_idx;
7573         struct rte_flow_error error;
7574
7575         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
7576         actions[0].conf = &jump;
7577         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
7578         flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7579                                     &attr, items, actions, false, &error);
7580         if (!flow_idx) {
7581                 DRV_LOG(DEBUG,
7582                         "Failed to create ctrl flow: rte_errno(%d),"
7583                         " type(%d), message(%s)",
7584                         rte_errno, error.type,
7585                         error.message ? error.message : " (no stated reason)");
7586                 return -rte_errno;
7587         }
7588         return 0;
7589 }
7590
7591 /**
7592  * Enable a control flow configured from the control plane.
7593  *
7594  * @param dev
7595  *   Pointer to Ethernet device.
7596  * @param eth_spec
7597  *   An Ethernet flow spec to apply.
7598  * @param eth_mask
7599  *   An Ethernet flow mask to apply.
7600  * @param vlan_spec
7601  *   A VLAN flow spec to apply.
7602  * @param vlan_mask
7603  *   A VLAN flow mask to apply.
7604  *
7605  * @return
7606  *   0 on success, a negative errno value otherwise and rte_errno is set.
7607  */
7608 int
7609 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
7610                     struct rte_flow_item_eth *eth_spec,
7611                     struct rte_flow_item_eth *eth_mask,
7612                     struct rte_flow_item_vlan *vlan_spec,
7613                     struct rte_flow_item_vlan *vlan_mask)
7614 {
7615         struct mlx5_priv *priv = dev->data->dev_private;
7616         const struct rte_flow_attr attr = {
7617                 .ingress = 1,
7618                 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
7619         };
7620         struct rte_flow_item items[] = {
7621                 {
7622                         .type = RTE_FLOW_ITEM_TYPE_ETH,
7623                         .spec = eth_spec,
7624                         .last = NULL,
7625                         .mask = eth_mask,
7626                 },
7627                 {
7628                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
7629                                               RTE_FLOW_ITEM_TYPE_END,
7630                         .spec = vlan_spec,
7631                         .last = NULL,
7632                         .mask = vlan_mask,
7633                 },
7634                 {
7635                         .type = RTE_FLOW_ITEM_TYPE_END,
7636                 },
7637         };
7638         uint16_t queue[priv->reta_idx_n];
7639         struct rte_flow_action_rss action_rss = {
7640                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
7641                 .level = 0,
7642                 .types = priv->rss_conf.rss_hf,
7643                 .key_len = priv->rss_conf.rss_key_len,
7644                 .queue_num = priv->reta_idx_n,
7645                 .key = priv->rss_conf.rss_key,
7646                 .queue = queue,
7647         };
7648         struct rte_flow_action actions[] = {
7649                 {
7650                         .type = RTE_FLOW_ACTION_TYPE_RSS,
7651                         .conf = &action_rss,
7652                 },
7653                 {
7654                         .type = RTE_FLOW_ACTION_TYPE_END,
7655                 },
7656         };
7657         uint32_t flow_idx;
7658         struct rte_flow_error error;
7659         unsigned int i;
7660
7661         if (!priv->reta_idx_n || !priv->rxqs_n) {
7662                 return 0;
7663         }
7664         if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG))
7665                 action_rss.types = 0;
7666         for (i = 0; i != priv->reta_idx_n; ++i)
7667                 queue[i] = (*priv->reta_idx)[i];
7668         flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7669                                     &attr, items, actions, false, &error);
7670         if (!flow_idx)
7671                 return -rte_errno;
7672         return 0;
7673 }
7674
7675 /**
7676  * Enable a flow control configured from the control plane.
7677  *
7678  * @param dev
7679  *   Pointer to Ethernet device.
7680  * @param eth_spec
7681  *   An Ethernet flow spec to apply.
7682  * @param eth_mask
7683  *   An Ethernet flow mask to apply.
7684  *
7685  * @return
7686  *   0 on success, a negative errno value otherwise and rte_errno is set.
7687  */
7688 int
7689 mlx5_ctrl_flow(struct rte_eth_dev *dev,
7690                struct rte_flow_item_eth *eth_spec,
7691                struct rte_flow_item_eth *eth_mask)
7692 {
7693         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
7694 }
7695
7696 /**
7697  * Create default miss flow rule matching lacp traffic
7698  *
7699  * @param dev
7700  *   Pointer to Ethernet device.
7701  * @param eth_spec
7702  *   An Ethernet flow spec to apply.
7703  *
7704  * @return
7705  *   0 on success, a negative errno value otherwise and rte_errno is set.
7706  */
7707 int
7708 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
7709 {
7710         /*
7711          * The LACP matching is done by only using ether type since using
7712          * a multicast dst mac causes kernel to give low priority to this flow.
7713          */
7714         static const struct rte_flow_item_eth lacp_spec = {
7715                 .type = RTE_BE16(0x8809),
7716         };
7717         static const struct rte_flow_item_eth lacp_mask = {
7718                 .type = 0xffff,
7719         };
7720         const struct rte_flow_attr attr = {
7721                 .ingress = 1,
7722         };
7723         struct rte_flow_item items[] = {
7724                 {
7725                         .type = RTE_FLOW_ITEM_TYPE_ETH,
7726                         .spec = &lacp_spec,
7727                         .mask = &lacp_mask,
7728                 },
7729                 {
7730                         .type = RTE_FLOW_ITEM_TYPE_END,
7731                 },
7732         };
7733         struct rte_flow_action actions[] = {
7734                 {
7735                         .type = (enum rte_flow_action_type)
7736                                 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
7737                 },
7738                 {
7739                         .type = RTE_FLOW_ACTION_TYPE_END,
7740                 },
7741         };
7742         struct rte_flow_error error;
7743         uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7744                                         &attr, items, actions,
7745                                         false, &error);
7746
7747         if (!flow_idx)
7748                 return -rte_errno;
7749         return 0;
7750 }
7751
7752 /**
7753  * Destroy a flow.
7754  *
7755  * @see rte_flow_destroy()
7756  * @see rte_flow_ops
7757  */
7758 int
7759 mlx5_flow_destroy(struct rte_eth_dev *dev,
7760                   struct rte_flow *flow,
7761                   struct rte_flow_error *error __rte_unused)
7762 {
7763         struct mlx5_priv *priv = dev->data->dev_private;
7764
7765         if (priv->sh->config.dv_flow_en == 2)
7766                 return rte_flow_error_set(error, ENOTSUP,
7767                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7768                           NULL,
7769                           "Flow non-Q destruction not supported");
7770         flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
7771                                 (uintptr_t)(void *)flow);
7772         return 0;
7773 }
7774
7775 /**
7776  * Destroy all flows.
7777  *
7778  * @see rte_flow_flush()
7779  * @see rte_flow_ops
7780  */
7781 int
7782 mlx5_flow_flush(struct rte_eth_dev *dev,
7783                 struct rte_flow_error *error __rte_unused)
7784 {
7785         mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false);
7786         return 0;
7787 }
7788
7789 /**
7790  * Isolated mode.
7791  *
7792  * @see rte_flow_isolate()
7793  * @see rte_flow_ops
7794  */
7795 int
7796 mlx5_flow_isolate(struct rte_eth_dev *dev,
7797                   int enable,
7798                   struct rte_flow_error *error)
7799 {
7800         struct mlx5_priv *priv = dev->data->dev_private;
7801
7802         if (dev->data->dev_started) {
7803                 rte_flow_error_set(error, EBUSY,
7804                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7805                                    NULL,
7806                                    "port must be stopped first");
7807                 return -rte_errno;
7808         }
7809         priv->isolated = !!enable;
7810         if (enable)
7811                 dev->dev_ops = &mlx5_dev_ops_isolate;
7812         else
7813                 dev->dev_ops = &mlx5_dev_ops;
7814
7815         dev->rx_descriptor_status = mlx5_rx_descriptor_status;
7816         dev->tx_descriptor_status = mlx5_tx_descriptor_status;
7817
7818         return 0;
7819 }
7820
7821 /**
7822  * Query a flow.
7823  *
7824  * @see rte_flow_query()
7825  * @see rte_flow_ops
7826  */
7827 static int
7828 flow_drv_query(struct rte_eth_dev *dev,
7829                uint32_t flow_idx,
7830                const struct rte_flow_action *actions,
7831                void *data,
7832                struct rte_flow_error *error)
7833 {
7834         struct mlx5_priv *priv = dev->data->dev_private;
7835         const struct mlx5_flow_driver_ops *fops;
7836         struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
7837                                                flow_idx);
7838         enum mlx5_flow_drv_type ftype;
7839
7840         if (!flow) {
7841                 return rte_flow_error_set(error, ENOENT,
7842                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7843                           NULL,
7844                           "invalid flow handle");
7845         }
7846         ftype = flow->drv_type;
7847         MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
7848         fops = flow_get_drv_ops(ftype);
7849
7850         return fops->query(dev, flow, actions, data, error);
7851 }
7852
7853 /**
7854  * Query a flow.
7855  *
7856  * @see rte_flow_query()
7857  * @see rte_flow_ops
7858  */
7859 int
7860 mlx5_flow_query(struct rte_eth_dev *dev,
7861                 struct rte_flow *flow,
7862                 const struct rte_flow_action *actions,
7863                 void *data,
7864                 struct rte_flow_error *error)
7865 {
7866         int ret;
7867         struct mlx5_priv *priv = dev->data->dev_private;
7868
7869         if (priv->sh->config.dv_flow_en == 2)
7870                 return rte_flow_error_set(error, ENOTSUP,
7871                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7872                           NULL,
7873                           "Flow non-Q query not supported");
7874         ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
7875                              error);
7876         if (ret < 0)
7877                 return ret;
7878         return 0;
7879 }
7880
7881 /**
7882  * Get rte_flow callbacks.
7883  *
7884  * @param dev
7885  *   Pointer to Ethernet device structure.
7886  * @param ops
7887  *   Pointer to operation-specific structure.
7888  *
7889  * @return 0
7890  */
7891 int
7892 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused,
7893                   const struct rte_flow_ops **ops)
7894 {
7895         *ops = &mlx5_flow_ops;
7896         return 0;
7897 }
7898
7899 /**
7900  * Validate meter policy actions.
7901  * Dispatcher for action type specific validation.
7902  *
7903  * @param[in] dev
7904  *   Pointer to the Ethernet device structure.
7905  * @param[in] action
7906  *   The meter policy action object to validate.
7907  * @param[in] attr
7908  *   Attributes of flow to determine steering domain.
7909  * @param[out] is_rss
7910  *   Is RSS or not.
7911  * @param[out] domain_bitmap
7912  *   Domain bitmap.
7913  * @param[out] is_def_policy
7914  *   Is default policy or not.
7915  * @param[out] error
7916  *   Perform verbose error reporting if not NULL. Initialized in case of
7917  *   error only.
7918  *
7919  * @return
7920  *   0 on success, otherwise negative errno value.
7921  */
7922 int
7923 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev,
7924                         const struct rte_flow_action *actions[RTE_COLORS],
7925                         struct rte_flow_attr *attr,
7926                         bool *is_rss,
7927                         uint8_t *domain_bitmap,
7928                         uint8_t *policy_mode,
7929                         struct rte_mtr_error *error)
7930 {
7931         const struct mlx5_flow_driver_ops *fops;
7932
7933         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7934         return fops->validate_mtr_acts(dev, actions, attr, is_rss,
7935                                        domain_bitmap, policy_mode, error);
7936 }
7937
7938 /**
7939  * Destroy the meter table set.
7940  *
7941  * @param[in] dev
7942  *   Pointer to Ethernet device.
7943  * @param[in] mtr_policy
7944  *   Meter policy struct.
7945  */
7946 void
7947 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev,
7948                       struct mlx5_flow_meter_policy *mtr_policy)
7949 {
7950         const struct mlx5_flow_driver_ops *fops;
7951
7952         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7953         fops->destroy_mtr_acts(dev, mtr_policy);
7954 }
7955
7956 /**
7957  * Create policy action, lock free,
7958  * (mutex should be acquired by caller).
7959  * Dispatcher for action type specific call.
7960  *
7961  * @param[in] dev
7962  *   Pointer to the Ethernet device structure.
7963  * @param[in] mtr_policy
7964  *   Meter policy struct.
7965  * @param[in] action
7966  *   Action specification used to create meter actions.
7967  * @param[out] error
7968  *   Perform verbose error reporting if not NULL. Initialized in case of
7969  *   error only.
7970  *
7971  * @return
7972  *   0 on success, otherwise negative errno value.
7973  */
7974 int
7975 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev,
7976                       struct mlx5_flow_meter_policy *mtr_policy,
7977                       const struct rte_flow_action *actions[RTE_COLORS],
7978                       struct rte_mtr_error *error)
7979 {
7980         const struct mlx5_flow_driver_ops *fops;
7981
7982         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7983         return fops->create_mtr_acts(dev, mtr_policy, actions, error);
7984 }
7985
7986 /**
7987  * Create policy rules, lock free,
7988  * (mutex should be acquired by caller).
7989  * Dispatcher for action type specific call.
7990  *
7991  * @param[in] dev
7992  *   Pointer to the Ethernet device structure.
7993  * @param[in] mtr_policy
7994  *   Meter policy struct.
7995  *
7996  * @return
7997  *   0 on success, -1 otherwise.
7998  */
7999 int
8000 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev,
8001                              struct mlx5_flow_meter_policy *mtr_policy)
8002 {
8003         const struct mlx5_flow_driver_ops *fops;
8004
8005         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8006         return fops->create_policy_rules(dev, mtr_policy);
8007 }
8008
8009 /**
8010  * Destroy policy rules, lock free,
8011  * (mutex should be acquired by caller).
8012  * Dispatcher for action type specific call.
8013  *
8014  * @param[in] dev
8015  *   Pointer to the Ethernet device structure.
8016  * @param[in] mtr_policy
8017  *   Meter policy struct.
8018  */
8019 void
8020 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev,
8021                              struct mlx5_flow_meter_policy *mtr_policy)
8022 {
8023         const struct mlx5_flow_driver_ops *fops;
8024
8025         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8026         fops->destroy_policy_rules(dev, mtr_policy);
8027 }
8028
8029 /**
8030  * Destroy the default policy table set.
8031  *
8032  * @param[in] dev
8033  *   Pointer to Ethernet device.
8034  */
8035 void
8036 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev)
8037 {
8038         const struct mlx5_flow_driver_ops *fops;
8039
8040         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8041         fops->destroy_def_policy(dev);
8042 }
8043
8044 /**
8045  * Destroy the default policy table set.
8046  *
8047  * @param[in] dev
8048  *   Pointer to Ethernet device.
8049  *
8050  * @return
8051  *   0 on success, -1 otherwise.
8052  */
8053 int
8054 mlx5_flow_create_def_policy(struct rte_eth_dev *dev)
8055 {
8056         const struct mlx5_flow_driver_ops *fops;
8057
8058         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8059         return fops->create_def_policy(dev);
8060 }
8061
8062 /**
8063  * Create the needed meter and suffix tables.
8064  *
8065  * @param[in] dev
8066  *   Pointer to Ethernet device.
8067  *
8068  * @return
8069  *   0 on success, -1 otherwise.
8070  */
8071 int
8072 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
8073                         struct mlx5_flow_meter_info *fm,
8074                         uint32_t mtr_idx,
8075                         uint8_t domain_bitmap)
8076 {
8077         const struct mlx5_flow_driver_ops *fops;
8078
8079         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8080         return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap);
8081 }
8082
8083 /**
8084  * Destroy the meter table set.
8085  *
8086  * @param[in] dev
8087  *   Pointer to Ethernet device.
8088  * @param[in] tbl
8089  *   Pointer to the meter table set.
8090  */
8091 void
8092 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
8093                            struct mlx5_flow_meter_info *fm)
8094 {
8095         const struct mlx5_flow_driver_ops *fops;
8096
8097         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8098         fops->destroy_mtr_tbls(dev, fm);
8099 }
8100
8101 /**
8102  * Destroy the global meter drop table.
8103  *
8104  * @param[in] dev
8105  *   Pointer to Ethernet device.
8106  */
8107 void
8108 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
8109 {
8110         const struct mlx5_flow_driver_ops *fops;
8111
8112         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8113         fops->destroy_mtr_drop_tbls(dev);
8114 }
8115
8116 /**
8117  * Destroy the sub policy table with RX queue.
8118  *
8119  * @param[in] dev
8120  *   Pointer to Ethernet device.
8121  * @param[in] mtr_policy
8122  *   Pointer to meter policy table.
8123  */
8124 void
8125 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
8126                 struct mlx5_flow_meter_policy *mtr_policy)
8127 {
8128         const struct mlx5_flow_driver_ops *fops;
8129
8130         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8131         fops->destroy_sub_policy_with_rxq(dev, mtr_policy);
8132 }
8133
8134 /**
8135  * Allocate the needed aso flow meter id.
8136  *
8137  * @param[in] dev
8138  *   Pointer to Ethernet device.
8139  *
8140  * @return
8141  *   Index to aso flow meter on success, NULL otherwise.
8142  */
8143 uint32_t
8144 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev)
8145 {
8146         const struct mlx5_flow_driver_ops *fops;
8147
8148         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8149         return fops->create_meter(dev);
8150 }
8151
8152 /**
8153  * Free the aso flow meter id.
8154  *
8155  * @param[in] dev
8156  *   Pointer to Ethernet device.
8157  * @param[in] mtr_idx
8158  *  Index to aso flow meter to be free.
8159  *
8160  * @return
8161  *   0 on success.
8162  */
8163 void
8164 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx)
8165 {
8166         const struct mlx5_flow_driver_ops *fops;
8167
8168         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8169         fops->free_meter(dev, mtr_idx);
8170 }
8171
8172 /**
8173  * Allocate a counter.
8174  *
8175  * @param[in] dev
8176  *   Pointer to Ethernet device structure.
8177  *
8178  * @return
8179  *   Index to allocated counter  on success, 0 otherwise.
8180  */
8181 uint32_t
8182 mlx5_counter_alloc(struct rte_eth_dev *dev)
8183 {
8184         const struct mlx5_flow_driver_ops *fops;
8185         struct rte_flow_attr attr = { .transfer = 0 };
8186
8187         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
8188                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8189                 return fops->counter_alloc(dev);
8190         }
8191         DRV_LOG(ERR,
8192                 "port %u counter allocate is not supported.",
8193                  dev->data->port_id);
8194         return 0;
8195 }
8196
8197 /**
8198  * Free a counter.
8199  *
8200  * @param[in] dev
8201  *   Pointer to Ethernet device structure.
8202  * @param[in] cnt
8203  *   Index to counter to be free.
8204  */
8205 void
8206 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
8207 {
8208         const struct mlx5_flow_driver_ops *fops;
8209         struct rte_flow_attr attr = { .transfer = 0 };
8210
8211         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
8212                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8213                 fops->counter_free(dev, cnt);
8214                 return;
8215         }
8216         DRV_LOG(ERR,
8217                 "port %u counter free is not supported.",
8218                  dev->data->port_id);
8219 }
8220
8221 /**
8222  * Query counter statistics.
8223  *
8224  * @param[in] dev
8225  *   Pointer to Ethernet device structure.
8226  * @param[in] cnt
8227  *   Index to counter to query.
8228  * @param[in] clear
8229  *   Set to clear counter statistics.
8230  * @param[out] pkts
8231  *   The counter hits packets number to save.
8232  * @param[out] bytes
8233  *   The counter hits bytes number to save.
8234  *
8235  * @return
8236  *   0 on success, a negative errno value otherwise.
8237  */
8238 int
8239 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
8240                    bool clear, uint64_t *pkts, uint64_t *bytes, void **action)
8241 {
8242         const struct mlx5_flow_driver_ops *fops;
8243         struct rte_flow_attr attr = { .transfer = 0 };
8244
8245         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
8246                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8247                 return fops->counter_query(dev, cnt, clear, pkts,
8248                                         bytes, action);
8249         }
8250         DRV_LOG(ERR,
8251                 "port %u counter query is not supported.",
8252                  dev->data->port_id);
8253         return -ENOTSUP;
8254 }
8255
8256 /**
8257  * Get information about HWS pre-configurable resources.
8258  *
8259  * @param[in] dev
8260  *   Pointer to the rte_eth_dev structure.
8261  * @param[out] port_info
8262  *   Pointer to port information.
8263  * @param[out] queue_info
8264  *   Pointer to queue information.
8265  * @param[out] error
8266  *   Pointer to error structure.
8267  *
8268  * @return
8269  *   0 on success, a negative errno value otherwise and rte_errno is set.
8270  */
8271 static int
8272 mlx5_flow_info_get(struct rte_eth_dev *dev,
8273                    struct rte_flow_port_info *port_info,
8274                    struct rte_flow_queue_info *queue_info,
8275                    struct rte_flow_error *error)
8276 {
8277         const struct mlx5_flow_driver_ops *fops;
8278
8279         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8280                 return rte_flow_error_set(error, ENOTSUP,
8281                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8282                                 NULL,
8283                                 "info get with incorrect steering mode");
8284         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8285         return fops->info_get(dev, port_info, queue_info, error);
8286 }
8287
8288 /**
8289  * Configure port HWS resources.
8290  *
8291  * @param[in] dev
8292  *   Pointer to the rte_eth_dev structure.
8293  * @param[in] port_attr
8294  *   Port configuration attributes.
8295  * @param[in] nb_queue
8296  *   Number of queue.
8297  * @param[in] queue_attr
8298  *   Array that holds attributes for each flow queue.
8299  * @param[out] error
8300  *   Pointer to error structure.
8301  *
8302  * @return
8303  *   0 on success, a negative errno value otherwise and rte_errno is set.
8304  */
8305 static int
8306 mlx5_flow_port_configure(struct rte_eth_dev *dev,
8307                          const struct rte_flow_port_attr *port_attr,
8308                          uint16_t nb_queue,
8309                          const struct rte_flow_queue_attr *queue_attr[],
8310                          struct rte_flow_error *error)
8311 {
8312         const struct mlx5_flow_driver_ops *fops;
8313
8314         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8315                 return rte_flow_error_set(error, ENOTSUP,
8316                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8317                                 NULL,
8318                                 "port configure with incorrect steering mode");
8319         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8320         return fops->configure(dev, port_attr, nb_queue, queue_attr, error);
8321 }
8322
8323 /**
8324  * Create flow item template.
8325  *
8326  * @param[in] dev
8327  *   Pointer to the rte_eth_dev structure.
8328  * @param[in] attr
8329  *   Pointer to the item template attributes.
8330  * @param[in] items
8331  *   The template item pattern.
8332  * @param[out] error
8333  *   Pointer to error structure.
8334  *
8335  * @return
8336  *   0 on success, a negative errno value otherwise and rte_errno is set.
8337  */
8338 static struct rte_flow_pattern_template *
8339 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev,
8340                 const struct rte_flow_pattern_template_attr *attr,
8341                 const struct rte_flow_item items[],
8342                 struct rte_flow_error *error)
8343 {
8344         const struct mlx5_flow_driver_ops *fops;
8345
8346         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) {
8347                 rte_flow_error_set(error, ENOTSUP,
8348                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8349                                 NULL,
8350                                 "pattern create with incorrect steering mode");
8351                 return NULL;
8352         }
8353         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8354         return fops->pattern_template_create(dev, attr, items, error);
8355 }
8356
8357 /**
8358  * Destroy flow item template.
8359  *
8360  * @param[in] dev
8361  *   Pointer to the rte_eth_dev structure.
8362  * @param[in] template
8363  *   Pointer to the item template to be destroyed.
8364  * @param[out] error
8365  *   Pointer to error structure.
8366  *
8367  * @return
8368  *   0 on success, a negative errno value otherwise and rte_errno is set.
8369  */
8370 static int
8371 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev,
8372                                    struct rte_flow_pattern_template *template,
8373                                    struct rte_flow_error *error)
8374 {
8375         const struct mlx5_flow_driver_ops *fops;
8376
8377         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8378                 return rte_flow_error_set(error, ENOTSUP,
8379                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8380                                 NULL,
8381                                 "pattern destroy with incorrect steering mode");
8382         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8383         return fops->pattern_template_destroy(dev, template, error);
8384 }
8385
8386 /**
8387  * Create flow item template.
8388  *
8389  * @param[in] dev
8390  *   Pointer to the rte_eth_dev structure.
8391  * @param[in] attr
8392  *   Pointer to the action template attributes.
8393  * @param[in] actions
8394  *   Associated actions (list terminated by the END action).
8395  * @param[in] masks
8396  *   List of actions that marks which of the action's member is constant.
8397  * @param[out] error
8398  *   Pointer to error structure.
8399  *
8400  * @return
8401  *   0 on success, a negative errno value otherwise and rte_errno is set.
8402  */
8403 static struct rte_flow_actions_template *
8404 mlx5_flow_actions_template_create(struct rte_eth_dev *dev,
8405                         const struct rte_flow_actions_template_attr *attr,
8406                         const struct rte_flow_action actions[],
8407                         const struct rte_flow_action masks[],
8408                         struct rte_flow_error *error)
8409 {
8410         const struct mlx5_flow_driver_ops *fops;
8411
8412         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) {
8413                 rte_flow_error_set(error, ENOTSUP,
8414                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8415                                 NULL,
8416                                 "action create with incorrect steering mode");
8417                 return NULL;
8418         }
8419         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8420         return fops->actions_template_create(dev, attr, actions, masks, error);
8421 }
8422
8423 /**
8424  * Destroy flow action template.
8425  *
8426  * @param[in] dev
8427  *   Pointer to the rte_eth_dev structure.
8428  * @param[in] template
8429  *   Pointer to the action template to be destroyed.
8430  * @param[out] error
8431  *   Pointer to error structure.
8432  *
8433  * @return
8434  *   0 on success, a negative errno value otherwise and rte_errno is set.
8435  */
8436 static int
8437 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev,
8438                                    struct rte_flow_actions_template *template,
8439                                    struct rte_flow_error *error)
8440 {
8441         const struct mlx5_flow_driver_ops *fops;
8442
8443         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8444                 return rte_flow_error_set(error, ENOTSUP,
8445                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8446                                 NULL,
8447                                 "action destroy with incorrect steering mode");
8448         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8449         return fops->actions_template_destroy(dev, template, error);
8450 }
8451
8452 /**
8453  * Create flow table.
8454  *
8455  * @param[in] dev
8456  *   Pointer to the rte_eth_dev structure.
8457  * @param[in] attr
8458  *   Pointer to the table attributes.
8459  * @param[in] item_templates
8460  *   Item template array to be binded to the table.
8461  * @param[in] nb_item_templates
8462  *   Number of item template.
8463  * @param[in] action_templates
8464  *   Action template array to be binded to the table.
8465  * @param[in] nb_action_templates
8466  *   Number of action template.
8467  * @param[out] error
8468  *   Pointer to error structure.
8469  *
8470  * @return
8471  *    Table on success, NULL otherwise and rte_errno is set.
8472  */
8473 static struct rte_flow_template_table *
8474 mlx5_flow_table_create(struct rte_eth_dev *dev,
8475                        const struct rte_flow_template_table_attr *attr,
8476                        struct rte_flow_pattern_template *item_templates[],
8477                        uint8_t nb_item_templates,
8478                        struct rte_flow_actions_template *action_templates[],
8479                        uint8_t nb_action_templates,
8480                        struct rte_flow_error *error)
8481 {
8482         const struct mlx5_flow_driver_ops *fops;
8483
8484         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) {
8485                 rte_flow_error_set(error, ENOTSUP,
8486                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8487                                 NULL,
8488                                 "table create with incorrect steering mode");
8489                 return NULL;
8490         }
8491         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8492         return fops->template_table_create(dev,
8493                                            attr,
8494                                            item_templates,
8495                                            nb_item_templates,
8496                                            action_templates,
8497                                            nb_action_templates,
8498                                            error);
8499 }
8500
8501 /**
8502  * PMD destroy flow table.
8503  *
8504  * @param[in] dev
8505  *   Pointer to the rte_eth_dev structure.
8506  * @param[in] table
8507  *   Pointer to the table to be destroyed.
8508  * @param[out] error
8509  *   Pointer to error structure.
8510  *
8511  * @return
8512  *   0 on success, a negative errno value otherwise and rte_errno is set.
8513  */
8514 static int
8515 mlx5_flow_table_destroy(struct rte_eth_dev *dev,
8516                         struct rte_flow_template_table *table,
8517                         struct rte_flow_error *error)
8518 {
8519         const struct mlx5_flow_driver_ops *fops;
8520
8521         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8522                 return rte_flow_error_set(error, ENOTSUP,
8523                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8524                                 NULL,
8525                                 "table destroy with incorrect steering mode");
8526         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8527         return fops->template_table_destroy(dev, table, error);
8528 }
8529
8530 /**
8531  * Enqueue flow creation.
8532  *
8533  * @param[in] dev
8534  *   Pointer to the rte_eth_dev structure.
8535  * @param[in] queue_id
8536  *   The queue to create the flow.
8537  * @param[in] attr
8538  *   Pointer to the flow operation attributes.
8539  * @param[in] items
8540  *   Items with flow spec value.
8541  * @param[in] pattern_template_index
8542  *   The item pattern flow follows from the table.
8543  * @param[in] actions
8544  *   Action with flow spec value.
8545  * @param[in] action_template_index
8546  *   The action pattern flow follows from the table.
8547  * @param[in] user_data
8548  *   Pointer to the user_data.
8549  * @param[out] error
8550  *   Pointer to error structure.
8551  *
8552  * @return
8553  *    Flow pointer on success, NULL otherwise and rte_errno is set.
8554  */
8555 static struct rte_flow *
8556 mlx5_flow_async_flow_create(struct rte_eth_dev *dev,
8557                             uint32_t queue_id,
8558                             const struct rte_flow_op_attr *attr,
8559                             struct rte_flow_template_table *table,
8560                             const struct rte_flow_item items[],
8561                             uint8_t pattern_template_index,
8562                             const struct rte_flow_action actions[],
8563                             uint8_t action_template_index,
8564                             void *user_data,
8565                             struct rte_flow_error *error)
8566 {
8567         const struct mlx5_flow_driver_ops *fops;
8568
8569         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) {
8570                 rte_flow_error_set(error, ENOTSUP,
8571                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8572                                 NULL,
8573                                 "flow_q create with incorrect steering mode");
8574                 return NULL;
8575         }
8576         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8577         return fops->async_flow_create(dev, queue_id, attr, table,
8578                                        items, pattern_template_index,
8579                                        actions, action_template_index,
8580                                        user_data, error);
8581 }
8582
8583 /**
8584  * Enqueue flow destruction.
8585  *
8586  * @param[in] dev
8587  *   Pointer to the rte_eth_dev structure.
8588  * @param[in] queue
8589  *   The queue to destroy the flow.
8590  * @param[in] attr
8591  *   Pointer to the flow operation attributes.
8592  * @param[in] flow
8593  *   Pointer to the flow to be destroyed.
8594  * @param[in] user_data
8595  *   Pointer to the user_data.
8596  * @param[out] error
8597  *   Pointer to error structure.
8598  *
8599  * @return
8600  *    0 on success, negative value otherwise and rte_errno is set.
8601  */
8602 static int
8603 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev,
8604                              uint32_t queue,
8605                              const struct rte_flow_op_attr *attr,
8606                              struct rte_flow *flow,
8607                              void *user_data,
8608                              struct rte_flow_error *error)
8609 {
8610         const struct mlx5_flow_driver_ops *fops;
8611
8612         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8613                 return rte_flow_error_set(error, ENOTSUP,
8614                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8615                                 NULL,
8616                                 "flow_q destroy with incorrect steering mode");
8617         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8618         return fops->async_flow_destroy(dev, queue, attr, flow,
8619                                         user_data, error);
8620 }
8621
8622 /**
8623  * Pull the enqueued flows.
8624  *
8625  * @param[in] dev
8626  *   Pointer to the rte_eth_dev structure.
8627  * @param[in] queue
8628  *   The queue to pull the result.
8629  * @param[in/out] res
8630  *   Array to save the results.
8631  * @param[in] n_res
8632  *   Available result with the array.
8633  * @param[out] error
8634  *   Pointer to error structure.
8635  *
8636  * @return
8637  *    Result number on success, negative value otherwise and rte_errno is set.
8638  */
8639 static int
8640 mlx5_flow_pull(struct rte_eth_dev *dev,
8641                uint32_t queue,
8642                struct rte_flow_op_result res[],
8643                uint16_t n_res,
8644                struct rte_flow_error *error)
8645 {
8646         const struct mlx5_flow_driver_ops *fops;
8647
8648         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8649                 return rte_flow_error_set(error, ENOTSUP,
8650                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8651                                 NULL,
8652                                 "flow_q pull with incorrect steering mode");
8653         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8654         return fops->pull(dev, queue, res, n_res, error);
8655 }
8656
8657 /**
8658  * Push the enqueued flows.
8659  *
8660  * @param[in] dev
8661  *   Pointer to the rte_eth_dev structure.
8662  * @param[in] queue
8663  *   The queue to push the flows.
8664  * @param[out] error
8665  *   Pointer to error structure.
8666  *
8667  * @return
8668  *    0 on success, negative value otherwise and rte_errno is set.
8669  */
8670 static int
8671 mlx5_flow_push(struct rte_eth_dev *dev,
8672                uint32_t queue,
8673                struct rte_flow_error *error)
8674 {
8675         const struct mlx5_flow_driver_ops *fops;
8676
8677         if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW)
8678                 return rte_flow_error_set(error, ENOTSUP,
8679                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8680                                 NULL,
8681                                 "flow_q push with incorrect steering mode");
8682         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8683         return fops->push(dev, queue, error);
8684 }
8685
8686 /**
8687  * Create shared action.
8688  *
8689  * @param[in] dev
8690  *   Pointer to the rte_eth_dev structure.
8691  * @param[in] queue
8692  *   Which queue to be used..
8693  * @param[in] attr
8694  *   Operation attribute.
8695  * @param[in] conf
8696  *   Indirect action configuration.
8697  * @param[in] action
8698  *   rte_flow action detail.
8699  * @param[in] user_data
8700  *   Pointer to the user_data.
8701  * @param[out] error
8702  *   Pointer to error structure.
8703  *
8704  * @return
8705  *   Action handle on success, NULL otherwise and rte_errno is set.
8706  */
8707 static struct rte_flow_action_handle *
8708 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue,
8709                                  const struct rte_flow_op_attr *attr,
8710                                  const struct rte_flow_indir_action_conf *conf,
8711                                  const struct rte_flow_action *action,
8712                                  void *user_data,
8713                                  struct rte_flow_error *error)
8714 {
8715         const struct mlx5_flow_driver_ops *fops =
8716                         flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8717
8718         return fops->async_action_create(dev, queue, attr, conf, action,
8719                                          user_data, error);
8720 }
8721
8722 /**
8723  * Update shared action.
8724  *
8725  * @param[in] dev
8726  *   Pointer to the rte_eth_dev structure.
8727  * @param[in] queue
8728  *   Which queue to be used..
8729  * @param[in] attr
8730  *   Operation attribute.
8731  * @param[in] handle
8732  *   Action handle to be updated.
8733  * @param[in] update
8734  *   Update value.
8735  * @param[in] user_data
8736  *   Pointer to the user_data.
8737  * @param[out] error
8738  *   Pointer to error structure.
8739  *
8740  * @return
8741  *   0 on success, negative value otherwise and rte_errno is set.
8742  */
8743 static int
8744 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue,
8745                                      const struct rte_flow_op_attr *attr,
8746                                      struct rte_flow_action_handle *handle,
8747                                      const void *update,
8748                                      void *user_data,
8749                                      struct rte_flow_error *error)
8750 {
8751         const struct mlx5_flow_driver_ops *fops =
8752                         flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8753
8754         return fops->async_action_update(dev, queue, attr, handle,
8755                                          update, user_data, error);
8756 }
8757
8758 /**
8759  * Destroy shared action.
8760  *
8761  * @param[in] dev
8762  *   Pointer to the rte_eth_dev structure.
8763  * @param[in] queue
8764  *   Which queue to be used..
8765  * @param[in] attr
8766  *   Operation attribute.
8767  * @param[in] handle
8768  *   Action handle to be destroyed.
8769  * @param[in] user_data
8770  *   Pointer to the user_data.
8771  * @param[out] error
8772  *   Pointer to error structure.
8773  *
8774  * @return
8775  *   0 on success, negative value otherwise and rte_errno is set.
8776  */
8777 static int
8778 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue,
8779                                       const struct rte_flow_op_attr *attr,
8780                                       struct rte_flow_action_handle *handle,
8781                                       void *user_data,
8782                                       struct rte_flow_error *error)
8783 {
8784         const struct mlx5_flow_driver_ops *fops =
8785                         flow_get_drv_ops(MLX5_FLOW_TYPE_HW);
8786
8787         return fops->async_action_destroy(dev, queue, attr, handle,
8788                                           user_data, error);
8789 }
8790
8791 /**
8792  * Allocate a new memory for the counter values wrapped by all the needed
8793  * management.
8794  *
8795  * @param[in] sh
8796  *   Pointer to mlx5_dev_ctx_shared object.
8797  *
8798  * @return
8799  *   0 on success, a negative errno value otherwise.
8800  */
8801 static int
8802 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
8803 {
8804         struct mlx5_counter_stats_mem_mng *mem_mng;
8805         volatile struct flow_counter_stats *raw_data;
8806         int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
8807         int size = (sizeof(struct flow_counter_stats) *
8808                         MLX5_COUNTERS_PER_POOL +
8809                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
8810                         sizeof(struct mlx5_counter_stats_mem_mng);
8811         size_t pgsize = rte_mem_page_size();
8812         uint8_t *mem;
8813         int ret;
8814         int i;
8815
8816         if (pgsize == (size_t)-1) {
8817                 DRV_LOG(ERR, "Failed to get mem page size");
8818                 rte_errno = ENOMEM;
8819                 return -ENOMEM;
8820         }
8821         mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
8822         if (!mem) {
8823                 rte_errno = ENOMEM;
8824                 return -ENOMEM;
8825         }
8826         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
8827         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
8828         ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd,
8829                                           sh->cdev->pdn, mem, size,
8830                                           &mem_mng->wm);
8831         if (ret) {
8832                 rte_errno = errno;
8833                 mlx5_free(mem);
8834                 return -rte_errno;
8835         }
8836         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
8837         raw_data = (volatile struct flow_counter_stats *)mem;
8838         for (i = 0; i < raws_n; ++i) {
8839                 mem_mng->raws[i].mem_mng = mem_mng;
8840                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
8841         }
8842         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
8843                 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
8844                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
8845                                  next);
8846         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
8847         sh->cmng.mem_mng = mem_mng;
8848         return 0;
8849 }
8850
8851 /**
8852  * Set the statistic memory to the new counter pool.
8853  *
8854  * @param[in] sh
8855  *   Pointer to mlx5_dev_ctx_shared object.
8856  * @param[in] pool
8857  *   Pointer to the pool to set the statistic memory.
8858  *
8859  * @return
8860  *   0 on success, a negative errno value otherwise.
8861  */
8862 static int
8863 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
8864                                struct mlx5_flow_counter_pool *pool)
8865 {
8866         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
8867         /* Resize statistic memory once used out. */
8868         if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
8869             mlx5_flow_create_counter_stat_mem_mng(sh)) {
8870                 DRV_LOG(ERR, "Cannot resize counter stat mem.");
8871                 return -1;
8872         }
8873         rte_spinlock_lock(&pool->sl);
8874         pool->raw = cmng->mem_mng->raws + pool->index %
8875                     MLX5_CNT_CONTAINER_RESIZE;
8876         rte_spinlock_unlock(&pool->sl);
8877         pool->raw_hw = NULL;
8878         return 0;
8879 }
8880
8881 #define MLX5_POOL_QUERY_FREQ_US 1000000
8882
8883 /**
8884  * Set the periodic procedure for triggering asynchronous batch queries for all
8885  * the counter pools.
8886  *
8887  * @param[in] sh
8888  *   Pointer to mlx5_dev_ctx_shared object.
8889  */
8890 void
8891 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
8892 {
8893         uint32_t pools_n, us;
8894
8895         pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED);
8896         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
8897         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
8898         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
8899                 sh->cmng.query_thread_on = 0;
8900                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
8901         } else {
8902                 sh->cmng.query_thread_on = 1;
8903         }
8904 }
8905
8906 /**
8907  * The periodic procedure for triggering asynchronous batch queries for all the
8908  * counter pools. This function is probably called by the host thread.
8909  *
8910  * @param[in] arg
8911  *   The parameter for the alarm process.
8912  */
8913 void
8914 mlx5_flow_query_alarm(void *arg)
8915 {
8916         struct mlx5_dev_ctx_shared *sh = arg;
8917         int ret;
8918         uint16_t pool_index = sh->cmng.pool_index;
8919         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
8920         struct mlx5_flow_counter_pool *pool;
8921         uint16_t n_valid;
8922
8923         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
8924                 goto set_alarm;
8925         rte_spinlock_lock(&cmng->pool_update_sl);
8926         pool = cmng->pools[pool_index];
8927         n_valid = cmng->n_valid;
8928         rte_spinlock_unlock(&cmng->pool_update_sl);
8929         /* Set the statistic memory to the new created pool. */
8930         if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
8931                 goto set_alarm;
8932         if (pool->raw_hw)
8933                 /* There is a pool query in progress. */
8934                 goto set_alarm;
8935         pool->raw_hw =
8936                 LIST_FIRST(&sh->cmng.free_stat_raws);
8937         if (!pool->raw_hw)
8938                 /* No free counter statistics raw memory. */
8939                 goto set_alarm;
8940         /*
8941          * Identify the counters released between query trigger and query
8942          * handle more efficiently. The counter released in this gap period
8943          * should wait for a new round of query as the new arrived packets
8944          * will not be taken into account.
8945          */
8946         pool->query_gen++;
8947         ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
8948                                                MLX5_COUNTERS_PER_POOL,
8949                                                NULL, NULL,
8950                                                pool->raw_hw->mem_mng->wm.lkey,
8951                                                (void *)(uintptr_t)
8952                                                pool->raw_hw->data,
8953                                                sh->devx_comp,
8954                                                (uint64_t)(uintptr_t)pool);
8955         if (ret) {
8956                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
8957                         " %d", pool->min_dcs->id);
8958                 pool->raw_hw = NULL;
8959                 goto set_alarm;
8960         }
8961         LIST_REMOVE(pool->raw_hw, next);
8962         sh->cmng.pending_queries++;
8963         pool_index++;
8964         if (pool_index >= n_valid)
8965                 pool_index = 0;
8966 set_alarm:
8967         sh->cmng.pool_index = pool_index;
8968         mlx5_set_query_alarm(sh);
8969 }
8970
8971 /**
8972  * Check and callback event for new aged flow in the counter pool
8973  *
8974  * @param[in] sh
8975  *   Pointer to mlx5_dev_ctx_shared object.
8976  * @param[in] pool
8977  *   Pointer to Current counter pool.
8978  */
8979 static void
8980 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
8981                    struct mlx5_flow_counter_pool *pool)
8982 {
8983         struct mlx5_priv *priv;
8984         struct mlx5_flow_counter *cnt;
8985         struct mlx5_age_info *age_info;
8986         struct mlx5_age_param *age_param;
8987         struct mlx5_counter_stats_raw *cur = pool->raw_hw;
8988         struct mlx5_counter_stats_raw *prev = pool->raw;
8989         const uint64_t curr_time = MLX5_CURR_TIME_SEC;
8990         const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
8991         uint16_t expected = AGE_CANDIDATE;
8992         uint32_t i;
8993
8994         pool->time_of_last_age_check = curr_time;
8995         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
8996                 cnt = MLX5_POOL_GET_CNT(pool, i);
8997                 age_param = MLX5_CNT_TO_AGE(cnt);
8998                 if (__atomic_load_n(&age_param->state,
8999                                     __ATOMIC_RELAXED) != AGE_CANDIDATE)
9000                         continue;
9001                 if (cur->data[i].hits != prev->data[i].hits) {
9002                         __atomic_store_n(&age_param->sec_since_last_hit, 0,
9003                                          __ATOMIC_RELAXED);
9004                         continue;
9005                 }
9006                 if (__atomic_add_fetch(&age_param->sec_since_last_hit,
9007                                        time_delta,
9008                                        __ATOMIC_RELAXED) <= age_param->timeout)
9009                         continue;
9010                 /**
9011                  * Hold the lock first, or if between the
9012                  * state AGE_TMOUT and tailq operation the
9013                  * release happened, the release procedure
9014                  * may delete a non-existent tailq node.
9015                  */
9016                 priv = rte_eth_devices[age_param->port_id].data->dev_private;
9017                 age_info = GET_PORT_AGE_INFO(priv);
9018                 rte_spinlock_lock(&age_info->aged_sl);
9019                 if (__atomic_compare_exchange_n(&age_param->state, &expected,
9020                                                 AGE_TMOUT, false,
9021                                                 __ATOMIC_RELAXED,
9022                                                 __ATOMIC_RELAXED)) {
9023                         TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
9024                         MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
9025                 }
9026                 rte_spinlock_unlock(&age_info->aged_sl);
9027         }
9028         mlx5_age_event_prepare(sh);
9029 }
9030
9031 /**
9032  * Handler for the HW respond about ready values from an asynchronous batch
9033  * query. This function is probably called by the host thread.
9034  *
9035  * @param[in] sh
9036  *   The pointer to the shared device context.
9037  * @param[in] async_id
9038  *   The Devx async ID.
9039  * @param[in] status
9040  *   The status of the completion.
9041  */
9042 void
9043 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
9044                                   uint64_t async_id, int status)
9045 {
9046         struct mlx5_flow_counter_pool *pool =
9047                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
9048         struct mlx5_counter_stats_raw *raw_to_free;
9049         uint8_t query_gen = pool->query_gen ^ 1;
9050         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
9051         enum mlx5_counter_type cnt_type =
9052                 pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
9053                                 MLX5_COUNTER_TYPE_ORIGIN;
9054
9055         if (unlikely(status)) {
9056                 raw_to_free = pool->raw_hw;
9057         } else {
9058                 raw_to_free = pool->raw;
9059                 if (pool->is_aged)
9060                         mlx5_flow_aging_check(sh, pool);
9061                 rte_spinlock_lock(&pool->sl);
9062                 pool->raw = pool->raw_hw;
9063                 rte_spinlock_unlock(&pool->sl);
9064                 /* Be sure the new raw counters data is updated in memory. */
9065                 rte_io_wmb();
9066                 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
9067                         rte_spinlock_lock(&cmng->csl[cnt_type]);
9068                         TAILQ_CONCAT(&cmng->counters[cnt_type],
9069                                      &pool->counters[query_gen], next);
9070                         rte_spinlock_unlock(&cmng->csl[cnt_type]);
9071                 }
9072         }
9073         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
9074         pool->raw_hw = NULL;
9075         sh->cmng.pending_queries--;
9076 }
9077
9078 static int
9079 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
9080                     const struct flow_grp_info *grp_info,
9081                     struct rte_flow_error *error)
9082 {
9083         if (grp_info->transfer && grp_info->external &&
9084             grp_info->fdb_def_rule) {
9085                 if (group == UINT32_MAX)
9086                         return rte_flow_error_set
9087                                                 (error, EINVAL,
9088                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
9089                                                  NULL,
9090                                                  "group index not supported");
9091                 *table = group + 1;
9092         } else {
9093                 *table = group;
9094         }
9095         DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
9096         return 0;
9097 }
9098
9099 /**
9100  * Translate the rte_flow group index to HW table value.
9101  *
9102  * If tunnel offload is disabled, all group ids converted to flow table
9103  * id using the standard method.
9104  * If tunnel offload is enabled, group id can be converted using the
9105  * standard or tunnel conversion method. Group conversion method
9106  * selection depends on flags in `grp_info` parameter:
9107  * - Internal (grp_info.external == 0) groups conversion uses the
9108  *   standard method.
9109  * - Group ids in JUMP action converted with the tunnel conversion.
9110  * - Group id in rule attribute conversion depends on a rule type and
9111  *   group id value:
9112  *   ** non zero group attributes converted with the tunnel method
9113  *   ** zero group attribute in non-tunnel rule is converted using the
9114  *      standard method - there's only one root table
9115  *   ** zero group attribute in steer tunnel rule is converted with the
9116  *      standard method - single root table
9117  *   ** zero group attribute in match tunnel rule is a special OvS
9118  *      case: that value is used for portability reasons. That group
9119  *      id is converted with the tunnel conversion method.
9120  *
9121  * @param[in] dev
9122  *   Port device
9123  * @param[in] tunnel
9124  *   PMD tunnel offload object
9125  * @param[in] group
9126  *   rte_flow group index value.
9127  * @param[out] table
9128  *   HW table value.
9129  * @param[in] grp_info
9130  *   flags used for conversion
9131  * @param[out] error
9132  *   Pointer to error structure.
9133  *
9134  * @return
9135  *   0 on success, a negative errno value otherwise and rte_errno is set.
9136  */
9137 int
9138 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
9139                          const struct mlx5_flow_tunnel *tunnel,
9140                          uint32_t group, uint32_t *table,
9141                          const struct flow_grp_info *grp_info,
9142                          struct rte_flow_error *error)
9143 {
9144         int ret;
9145         bool standard_translation;
9146
9147         if (!grp_info->skip_scale && grp_info->external &&
9148             group < MLX5_MAX_TABLES_EXTERNAL)
9149                 group *= MLX5_FLOW_TABLE_FACTOR;
9150         if (is_tunnel_offload_active(dev)) {
9151                 standard_translation = !grp_info->external ||
9152                                         grp_info->std_tbl_fix;
9153         } else {
9154                 standard_translation = true;
9155         }
9156         DRV_LOG(DEBUG,
9157                 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s",
9158                 dev->data->port_id, group, grp_info->transfer,
9159                 grp_info->external, grp_info->fdb_def_rule,
9160                 standard_translation ? "STANDARD" : "TUNNEL");
9161         if (standard_translation)
9162                 ret = flow_group_to_table(dev->data->port_id, group, table,
9163                                           grp_info, error);
9164         else
9165                 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
9166                                                       table, error);
9167
9168         return ret;
9169 }
9170
9171 /**
9172  * Discover availability of metadata reg_c's.
9173  *
9174  * Iteratively use test flows to check availability.
9175  *
9176  * @param[in] dev
9177  *   Pointer to the Ethernet device structure.
9178  *
9179  * @return
9180  *   0 on success, a negative errno value otherwise and rte_errno is set.
9181  */
9182 int
9183 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
9184 {
9185         struct mlx5_priv *priv = dev->data->dev_private;
9186         enum modify_reg idx;
9187         int n = 0;
9188
9189         /* reg_c[0] and reg_c[1] are reserved. */
9190         priv->sh->flow_mreg_c[n++] = REG_C_0;
9191         priv->sh->flow_mreg_c[n++] = REG_C_1;
9192         /* Discover availability of other reg_c's. */
9193         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
9194                 struct rte_flow_attr attr = {
9195                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
9196                         .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
9197                         .ingress = 1,
9198                 };
9199                 struct rte_flow_item items[] = {
9200                         [0] = {
9201                                 .type = RTE_FLOW_ITEM_TYPE_END,
9202                         },
9203                 };
9204                 struct rte_flow_action actions[] = {
9205                         [0] = {
9206                                 .type = (enum rte_flow_action_type)
9207                                         MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
9208                                 .conf = &(struct mlx5_flow_action_copy_mreg){
9209                                         .src = REG_C_1,
9210                                         .dst = idx,
9211                                 },
9212                         },
9213                         [1] = {
9214                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
9215                                 .conf = &(struct rte_flow_action_jump){
9216                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
9217                                 },
9218                         },
9219                         [2] = {
9220                                 .type = RTE_FLOW_ACTION_TYPE_END,
9221                         },
9222                 };
9223                 uint32_t flow_idx;
9224                 struct rte_flow *flow;
9225                 struct rte_flow_error error;
9226
9227                 if (!priv->sh->config.dv_flow_en)
9228                         break;
9229                 /* Create internal flow, validation skips copy action. */
9230                 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
9231                                         items, actions, false, &error);
9232                 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
9233                                       flow_idx);
9234                 if (!flow)
9235                         continue;
9236                 priv->sh->flow_mreg_c[n++] = idx;
9237                 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
9238         }
9239         for (; n < MLX5_MREG_C_NUM; ++n)
9240                 priv->sh->flow_mreg_c[n] = REG_NON;
9241         priv->sh->metadata_regc_check_flag = 1;
9242         return 0;
9243 }
9244
9245 int
9246 save_dump_file(const uint8_t *data, uint32_t size,
9247         uint32_t type, uint64_t id, void *arg, FILE *file)
9248 {
9249         char line[BUF_SIZE];
9250         uint32_t out = 0;
9251         uint32_t k;
9252         uint32_t actions_num;
9253         struct rte_flow_query_count *count;
9254
9255         memset(line, 0, BUF_SIZE);
9256         switch (type) {
9257         case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR:
9258                 actions_num = *(uint32_t *)(arg);
9259                 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,",
9260                                 type, id, actions_num);
9261                 break;
9262         case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT:
9263                 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",",
9264                                 type, id);
9265                 break;
9266         case DR_DUMP_REC_TYPE_PMD_COUNTER:
9267                 count = (struct rte_flow_query_count *)arg;
9268                 fprintf(file,
9269                         "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n",
9270                         type, id, count->hits, count->bytes);
9271                 return 0;
9272         default:
9273                 return -1;
9274         }
9275
9276         for (k = 0; k < size; k++) {
9277                 /* Make sure we do not overrun the line buffer length. */
9278                 if (out >= BUF_SIZE - 4) {
9279                         line[out] = '\0';
9280                         break;
9281                 }
9282                 out += snprintf(line + out, BUF_SIZE - out, "%02x",
9283                                 (data[k]) & 0xff);
9284         }
9285         fprintf(file, "%s\n", line);
9286         return 0;
9287 }
9288
9289 int
9290 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow,
9291         struct rte_flow_query_count *count, struct rte_flow_error *error)
9292 {
9293         struct rte_flow_action action[2];
9294         enum mlx5_flow_drv_type ftype;
9295         const struct mlx5_flow_driver_ops *fops;
9296
9297         if (!flow) {
9298                 return rte_flow_error_set(error, ENOENT,
9299                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9300                                 NULL,
9301                                 "invalid flow handle");
9302         }
9303         action[0].type = RTE_FLOW_ACTION_TYPE_COUNT;
9304         action[1].type = RTE_FLOW_ACTION_TYPE_END;
9305         if (flow->counter) {
9306                 memset(count, 0, sizeof(struct rte_flow_query_count));
9307                 ftype = (enum mlx5_flow_drv_type)(flow->drv_type);
9308                 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN &&
9309                                                 ftype < MLX5_FLOW_TYPE_MAX);
9310                 fops = flow_get_drv_ops(ftype);
9311                 return fops->query(dev, flow, action, count, error);
9312         }
9313         return -1;
9314 }
9315
9316 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
9317 /**
9318  * Dump flow ipool data to file
9319  *
9320  * @param[in] dev
9321  *   The pointer to Ethernet device.
9322  * @param[in] file
9323  *   A pointer to a file for output.
9324  * @param[out] error
9325  *   Perform verbose error reporting if not NULL. PMDs initialize this
9326  *   structure in case of error only.
9327  * @return
9328  *   0 on success, a negative value otherwise.
9329  */
9330 int
9331 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev,
9332         struct rte_flow *flow, FILE *file,
9333         struct rte_flow_error *error)
9334 {
9335         struct mlx5_priv *priv = dev->data->dev_private;
9336         struct mlx5_flow_dv_modify_hdr_resource  *modify_hdr;
9337         struct mlx5_flow_dv_encap_decap_resource *encap_decap;
9338         uint32_t handle_idx;
9339         struct mlx5_flow_handle *dh;
9340         struct rte_flow_query_count count;
9341         uint32_t actions_num;
9342         const uint8_t *data;
9343         size_t size;
9344         uint64_t id;
9345         uint32_t type;
9346         void *action = NULL;
9347
9348         if (!flow) {
9349                 return rte_flow_error_set(error, ENOENT,
9350                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9351                                 NULL,
9352                                 "invalid flow handle");
9353         }
9354         handle_idx = flow->dev_handles;
9355         /* query counter */
9356         if (flow->counter &&
9357         (!mlx5_counter_query(dev, flow->counter, false,
9358         &count.hits, &count.bytes, &action)) && action) {
9359                 id = (uint64_t)(uintptr_t)action;
9360                 type = DR_DUMP_REC_TYPE_PMD_COUNTER;
9361                 save_dump_file(NULL, 0, type,
9362                         id, (void *)&count, file);
9363         }
9364
9365         while (handle_idx) {
9366                 dh = mlx5_ipool_get(priv->sh->ipool
9367                                 [MLX5_IPOOL_MLX5_FLOW], handle_idx);
9368                 if (!dh)
9369                         continue;
9370                 handle_idx = dh->next.next;
9371
9372                 /* Get modify_hdr and encap_decap buf from ipools. */
9373                 encap_decap = NULL;
9374                 modify_hdr = dh->dvh.modify_hdr;
9375
9376                 if (dh->dvh.rix_encap_decap) {
9377                         encap_decap = mlx5_ipool_get(priv->sh->ipool
9378                                                 [MLX5_IPOOL_DECAP_ENCAP],
9379                                                 dh->dvh.rix_encap_decap);
9380                 }
9381                 if (modify_hdr) {
9382                         data = (const uint8_t *)modify_hdr->actions;
9383                         size = (size_t)(modify_hdr->actions_num) * 8;
9384                         id = (uint64_t)(uintptr_t)modify_hdr->action;
9385                         actions_num = modify_hdr->actions_num;
9386                         type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR;
9387                         save_dump_file(data, size, type, id,
9388                                                 (void *)(&actions_num), file);
9389                 }
9390                 if (encap_decap) {
9391                         data = encap_decap->buf;
9392                         size = encap_decap->size;
9393                         id = (uint64_t)(uintptr_t)encap_decap->action;
9394                         type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT;
9395                         save_dump_file(data, size, type,
9396                                                 id, NULL, file);
9397                 }
9398         }
9399         return 0;
9400 }
9401
9402 /**
9403  * Dump all flow's encap_decap/modify_hdr/counter data to file
9404  *
9405  * @param[in] dev
9406  *   The pointer to Ethernet device.
9407  * @param[in] file
9408  *   A pointer to a file for output.
9409  * @param[out] error
9410  *   Perform verbose error reporting if not NULL. PMDs initialize this
9411  *   structure in case of error only.
9412  * @return
9413  *   0 on success, a negative value otherwise.
9414  */
9415 static int
9416 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev,
9417         FILE *file, struct rte_flow_error *error __rte_unused)
9418 {
9419         struct mlx5_priv *priv = dev->data->dev_private;
9420         struct mlx5_dev_ctx_shared *sh = priv->sh;
9421         struct mlx5_hlist *h;
9422         struct mlx5_flow_dv_modify_hdr_resource  *modify_hdr;
9423         struct mlx5_flow_dv_encap_decap_resource *encap_decap;
9424         struct rte_flow_query_count count;
9425         uint32_t actions_num;
9426         const uint8_t *data;
9427         size_t size;
9428         uint64_t id;
9429         uint32_t type;
9430         uint32_t i;
9431         uint32_t j;
9432         struct mlx5_list_inconst *l_inconst;
9433         struct mlx5_list_entry *e;
9434         int lcore_index;
9435         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
9436         uint32_t max;
9437         void *action;
9438
9439         /* encap_decap hlist is lcore_share, get global core cache. */
9440         i = MLX5_LIST_GLOBAL;
9441         h = sh->encaps_decaps;
9442         if (h) {
9443                 for (j = 0; j <= h->mask; j++) {
9444                         l_inconst = &h->buckets[j].l;
9445                         if (!l_inconst || !l_inconst->cache[i])
9446                                 continue;
9447
9448                         e = LIST_FIRST(&l_inconst->cache[i]->h);
9449                         while (e) {
9450                                 encap_decap =
9451                                 (struct mlx5_flow_dv_encap_decap_resource *)e;
9452                                 data = encap_decap->buf;
9453                                 size = encap_decap->size;
9454                                 id = (uint64_t)(uintptr_t)encap_decap->action;
9455                                 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT;
9456                                 save_dump_file(data, size, type,
9457                                         id, NULL, file);
9458                                 e = LIST_NEXT(e, next);
9459                         }
9460                 }
9461         }
9462
9463         /* get modify_hdr */
9464         h = sh->modify_cmds;
9465         if (h) {
9466                 lcore_index = rte_lcore_index(rte_lcore_id());
9467                 if (unlikely(lcore_index == -1)) {
9468                         lcore_index = MLX5_LIST_NLCORE;
9469                         rte_spinlock_lock(&h->l_const.lcore_lock);
9470                 }
9471                 i = lcore_index;
9472
9473                 for (j = 0; j <= h->mask; j++) {
9474                         l_inconst = &h->buckets[j].l;
9475                         if (!l_inconst || !l_inconst->cache[i])
9476                                 continue;
9477
9478                         e = LIST_FIRST(&l_inconst->cache[i]->h);
9479                         while (e) {
9480                                 modify_hdr =
9481                                 (struct mlx5_flow_dv_modify_hdr_resource *)e;
9482                                 data = (const uint8_t *)modify_hdr->actions;
9483                                 size = (size_t)(modify_hdr->actions_num) * 8;
9484                                 actions_num = modify_hdr->actions_num;
9485                                 id = (uint64_t)(uintptr_t)modify_hdr->action;
9486                                 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR;
9487                                 save_dump_file(data, size, type, id,
9488                                                 (void *)(&actions_num), file);
9489                                 e = LIST_NEXT(e, next);
9490                         }
9491                 }
9492
9493                 if (unlikely(lcore_index == MLX5_LIST_NLCORE))
9494                         rte_spinlock_unlock(&h->l_const.lcore_lock);
9495         }
9496
9497         /* get counter */
9498         MLX5_ASSERT(cmng->n_valid <= cmng->n);
9499         max = MLX5_COUNTERS_PER_POOL * cmng->n_valid;
9500         for (j = 1; j <= max; j++) {
9501                 action = NULL;
9502                 if ((!mlx5_counter_query(dev, j, false, &count.hits,
9503                 &count.bytes, &action)) && action) {
9504                         id = (uint64_t)(uintptr_t)action;
9505                         type = DR_DUMP_REC_TYPE_PMD_COUNTER;
9506                         save_dump_file(NULL, 0, type,
9507                                         id, (void *)&count, file);
9508                 }
9509         }
9510         return 0;
9511 }
9512 #endif
9513
9514 /**
9515  * Dump flow raw hw data to file
9516  *
9517  * @param[in] dev
9518  *    The pointer to Ethernet device.
9519  * @param[in] file
9520  *   A pointer to a file for output.
9521  * @param[out] error
9522  *   Perform verbose error reporting if not NULL. PMDs initialize this
9523  *   structure in case of error only.
9524  * @return
9525  *   0 on success, a negative value otherwise.
9526  */
9527 int
9528 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
9529                    FILE *file,
9530                    struct rte_flow_error *error __rte_unused)
9531 {
9532         struct mlx5_priv *priv = dev->data->dev_private;
9533         struct mlx5_dev_ctx_shared *sh = priv->sh;
9534         uint32_t handle_idx;
9535         int ret;
9536         struct mlx5_flow_handle *dh;
9537         struct rte_flow *flow;
9538
9539         if (!sh->config.dv_flow_en) {
9540                 if (fputs("device dv flow disabled\n", file) <= 0)
9541                         return -errno;
9542                 return -ENOTSUP;
9543         }
9544
9545         /* dump all */
9546         if (!flow_idx) {
9547 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
9548                 if (mlx5_flow_dev_dump_sh_all(dev, file, error))
9549                         return -EINVAL;
9550 #endif
9551                 return mlx5_devx_cmd_flow_dump(sh->fdb_domain,
9552                                         sh->rx_domain,
9553                                         sh->tx_domain, file);
9554         }
9555         /* dump one */
9556         flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
9557                         (uintptr_t)(void *)flow_idx);
9558         if (!flow)
9559                 return -EINVAL;
9560
9561 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
9562         mlx5_flow_dev_dump_ipool(dev, flow, file, error);
9563 #endif
9564         handle_idx = flow->dev_handles;
9565         while (handle_idx) {
9566                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
9567                                 handle_idx);
9568                 if (!dh)
9569                         return -ENOENT;
9570                 if (dh->drv_flow) {
9571                         ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow,
9572                                         file);
9573                         if (ret)
9574                                 return -ENOENT;
9575                 }
9576                 handle_idx = dh->next.next;
9577         }
9578         return 0;
9579 }
9580
9581 /**
9582  * Get aged-out flows.
9583  *
9584  * @param[in] dev
9585  *   Pointer to the Ethernet device structure.
9586  * @param[in] context
9587  *   The address of an array of pointers to the aged-out flows contexts.
9588  * @param[in] nb_countexts
9589  *   The length of context array pointers.
9590  * @param[out] error
9591  *   Perform verbose error reporting if not NULL. Initialized in case of
9592  *   error only.
9593  *
9594  * @return
9595  *   how many contexts get in success, otherwise negative errno value.
9596  *   if nb_contexts is 0, return the amount of all aged contexts.
9597  *   if nb_contexts is not 0 , return the amount of aged flows reported
9598  *   in the context array.
9599  */
9600 int
9601 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
9602                         uint32_t nb_contexts, struct rte_flow_error *error)
9603 {
9604         const struct mlx5_flow_driver_ops *fops;
9605         struct rte_flow_attr attr = { .transfer = 0 };
9606
9607         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
9608                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
9609                 return fops->get_aged_flows(dev, contexts, nb_contexts,
9610                                                     error);
9611         }
9612         DRV_LOG(ERR,
9613                 "port %u get aged flows is not supported.",
9614                  dev->data->port_id);
9615         return -ENOTSUP;
9616 }
9617
9618 /* Wrapper for driver action_validate op callback */
9619 static int
9620 flow_drv_action_validate(struct rte_eth_dev *dev,
9621                          const struct rte_flow_indir_action_conf *conf,
9622                          const struct rte_flow_action *action,
9623                          const struct mlx5_flow_driver_ops *fops,
9624                          struct rte_flow_error *error)
9625 {
9626         static const char err_msg[] = "indirect action validation unsupported";
9627
9628         if (!fops->action_validate) {
9629                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
9630                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
9631                                    NULL, err_msg);
9632                 return -rte_errno;
9633         }
9634         return fops->action_validate(dev, conf, action, error);
9635 }
9636
9637 /**
9638  * Destroys the shared action by handle.
9639  *
9640  * @param dev
9641  *   Pointer to Ethernet device structure.
9642  * @param[in] handle
9643  *   Handle for the indirect action object to be destroyed.
9644  * @param[out] error
9645  *   Perform verbose error reporting if not NULL. PMDs initialize this
9646  *   structure in case of error only.
9647  *
9648  * @return
9649  *   0 on success, a negative errno value otherwise and rte_errno is set.
9650  *
9651  * @note: wrapper for driver action_create op callback.
9652  */
9653 static int
9654 mlx5_action_handle_destroy(struct rte_eth_dev *dev,
9655                            struct rte_flow_action_handle *handle,
9656                            struct rte_flow_error *error)
9657 {
9658         static const char err_msg[] = "indirect action destruction unsupported";
9659         struct rte_flow_attr attr = { .transfer = 0 };
9660         const struct mlx5_flow_driver_ops *fops =
9661                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
9662
9663         if (!fops->action_destroy) {
9664                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
9665                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
9666                                    NULL, err_msg);
9667                 return -rte_errno;
9668         }
9669         return fops->action_destroy(dev, handle, error);
9670 }
9671
9672 /* Wrapper for driver action_destroy op callback */
9673 static int
9674 flow_drv_action_update(struct rte_eth_dev *dev,
9675                        struct rte_flow_action_handle *handle,
9676                        const void *update,
9677                        const struct mlx5_flow_driver_ops *fops,
9678                        struct rte_flow_error *error)
9679 {
9680         static const char err_msg[] = "indirect action update unsupported";
9681
9682         if (!fops->action_update) {
9683                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
9684                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
9685                                    NULL, err_msg);
9686                 return -rte_errno;
9687         }
9688         return fops->action_update(dev, handle, update, error);
9689 }
9690
9691 /* Wrapper for driver action_destroy op callback */
9692 static int
9693 flow_drv_action_query(struct rte_eth_dev *dev,
9694                       const struct rte_flow_action_handle *handle,
9695                       void *data,
9696                       const struct mlx5_flow_driver_ops *fops,
9697                       struct rte_flow_error *error)
9698 {
9699         static const char err_msg[] = "indirect action query unsupported";
9700
9701         if (!fops->action_query) {
9702                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
9703                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
9704                                    NULL, err_msg);
9705                 return -rte_errno;
9706         }
9707         return fops->action_query(dev, handle, data, error);
9708 }
9709
9710 /**
9711  * Create indirect action for reuse in multiple flow rules.
9712  *
9713  * @param dev
9714  *   Pointer to Ethernet device structure.
9715  * @param conf
9716  *   Pointer to indirect action object configuration.
9717  * @param[in] action
9718  *   Action configuration for indirect action object creation.
9719  * @param[out] error
9720  *   Perform verbose error reporting if not NULL. PMDs initialize this
9721  *   structure in case of error only.
9722  * @return
9723  *   A valid handle in case of success, NULL otherwise and rte_errno is set.
9724  */
9725 static struct rte_flow_action_handle *
9726 mlx5_action_handle_create(struct rte_eth_dev *dev,
9727                           const struct rte_flow_indir_action_conf *conf,
9728                           const struct rte_flow_action *action,
9729                           struct rte_flow_error *error)
9730 {
9731         static const char err_msg[] = "indirect action creation unsupported";
9732         struct rte_flow_attr attr = { .transfer = 0 };
9733         const struct mlx5_flow_driver_ops *fops =
9734                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
9735
9736         if (flow_drv_action_validate(dev, conf, action, fops, error))
9737                 return NULL;
9738         if (!fops->action_create) {
9739                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
9740                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
9741                                    NULL, err_msg);
9742                 return NULL;
9743         }
9744         return fops->action_create(dev, conf, action, error);
9745 }
9746
9747 /**
9748  * Updates inplace the indirect action configuration pointed by *handle*
9749  * with the configuration provided as *update* argument.
9750  * The update of the indirect action configuration effects all flow rules
9751  * reusing the action via handle.
9752  *
9753  * @param dev
9754  *   Pointer to Ethernet device structure.
9755  * @param[in] handle
9756  *   Handle for the indirect action to be updated.
9757  * @param[in] update
9758  *   Action specification used to modify the action pointed by handle.
9759  *   *update* could be of same type with the action pointed by the *handle*
9760  *   handle argument, or some other structures like a wrapper, depending on
9761  *   the indirect action type.
9762  * @param[out] error
9763  *   Perform verbose error reporting if not NULL. PMDs initialize this
9764  *   structure in case of error only.
9765  *
9766  * @return
9767  *   0 on success, a negative errno value otherwise and rte_errno is set.
9768  */
9769 static int
9770 mlx5_action_handle_update(struct rte_eth_dev *dev,
9771                 struct rte_flow_action_handle *handle,
9772                 const void *update,
9773                 struct rte_flow_error *error)
9774 {
9775         struct rte_flow_attr attr = { .transfer = 0 };
9776         const struct mlx5_flow_driver_ops *fops =
9777                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
9778         int ret;
9779
9780         ret = flow_drv_action_validate(dev, NULL,
9781                         (const struct rte_flow_action *)update, fops, error);
9782         if (ret)
9783                 return ret;
9784         return flow_drv_action_update(dev, handle, update, fops,
9785                                       error);
9786 }
9787
9788 /**
9789  * Query the indirect action by handle.
9790  *
9791  * This function allows retrieving action-specific data such as counters.
9792  * Data is gathered by special action which may be present/referenced in
9793  * more than one flow rule definition.
9794  *
9795  * see @RTE_FLOW_ACTION_TYPE_COUNT
9796  *
9797  * @param dev
9798  *   Pointer to Ethernet device structure.
9799  * @param[in] handle
9800  *   Handle for the indirect action to query.
9801  * @param[in, out] data
9802  *   Pointer to storage for the associated query data type.
9803  * @param[out] error
9804  *   Perform verbose error reporting if not NULL. PMDs initialize this
9805  *   structure in case of error only.
9806  *
9807  * @return
9808  *   0 on success, a negative errno value otherwise and rte_errno is set.
9809  */
9810 static int
9811 mlx5_action_handle_query(struct rte_eth_dev *dev,
9812                          const struct rte_flow_action_handle *handle,
9813                          void *data,
9814                          struct rte_flow_error *error)
9815 {
9816         struct rte_flow_attr attr = { .transfer = 0 };
9817         const struct mlx5_flow_driver_ops *fops =
9818                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
9819
9820         return flow_drv_action_query(dev, handle, data, fops, error);
9821 }
9822
9823 /**
9824  * Destroy all indirect actions (shared RSS).
9825  *
9826  * @param dev
9827  *   Pointer to Ethernet device.
9828  *
9829  * @return
9830  *   0 on success, a negative errno value otherwise and rte_errno is set.
9831  */
9832 int
9833 mlx5_action_handle_flush(struct rte_eth_dev *dev)
9834 {
9835         struct rte_flow_error error;
9836         struct mlx5_priv *priv = dev->data->dev_private;
9837         struct mlx5_shared_action_rss *shared_rss;
9838         int ret = 0;
9839         uint32_t idx;
9840
9841         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
9842                       priv->rss_shared_actions, idx, shared_rss, next) {
9843                 ret |= mlx5_action_handle_destroy(dev,
9844                        (struct rte_flow_action_handle *)(uintptr_t)idx, &error);
9845         }
9846         return ret;
9847 }
9848
9849 /**
9850  * Validate existing indirect actions against current device configuration
9851  * and attach them to device resources.
9852  *
9853  * @param dev
9854  *   Pointer to Ethernet device.
9855  *
9856  * @return
9857  *   0 on success, a negative errno value otherwise and rte_errno is set.
9858  */
9859 int
9860 mlx5_action_handle_attach(struct rte_eth_dev *dev)
9861 {
9862         struct mlx5_priv *priv = dev->data->dev_private;
9863         int ret = 0;
9864         struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last;
9865
9866         LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
9867                 const char *message;
9868                 uint32_t queue_idx;
9869
9870                 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues,
9871                                                ind_tbl->queues_n,
9872                                                &message, &queue_idx);
9873                 if (ret != 0) {
9874                         DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s",
9875                                 dev->data->port_id, ind_tbl->queues[queue_idx],
9876                                 message);
9877                         break;
9878                 }
9879         }
9880         if (ret != 0)
9881                 return ret;
9882         LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
9883                 ret = mlx5_ind_table_obj_attach(dev, ind_tbl);
9884                 if (ret != 0) {
9885                         DRV_LOG(ERR, "Port %u could not attach "
9886                                 "indirection table obj %p",
9887                                 dev->data->port_id, (void *)ind_tbl);
9888                         goto error;
9889                 }
9890         }
9891
9892         return 0;
9893 error:
9894         ind_tbl_last = ind_tbl;
9895         LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
9896                 if (ind_tbl == ind_tbl_last)
9897                         break;
9898                 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0)
9899                         DRV_LOG(CRIT, "Port %u could not detach "
9900                                 "indirection table obj %p on rollback",
9901                                 dev->data->port_id, (void *)ind_tbl);
9902         }
9903         return ret;
9904 }
9905
9906 /**
9907  * Detach indirect actions of the device from its resources.
9908  *
9909  * @param dev
9910  *   Pointer to Ethernet device.
9911  *
9912  * @return
9913  *   0 on success, a negative errno value otherwise and rte_errno is set.
9914  */
9915 int
9916 mlx5_action_handle_detach(struct rte_eth_dev *dev)
9917 {
9918         struct mlx5_priv *priv = dev->data->dev_private;
9919         int ret = 0;
9920         struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last;
9921
9922         LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
9923                 ret = mlx5_ind_table_obj_detach(dev, ind_tbl);
9924                 if (ret != 0) {
9925                         DRV_LOG(ERR, "Port %u could not detach "
9926                                 "indirection table obj %p",
9927                                 dev->data->port_id, (void *)ind_tbl);
9928                         goto error;
9929                 }
9930         }
9931         return 0;
9932 error:
9933         ind_tbl_last = ind_tbl;
9934         LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) {
9935                 if (ind_tbl == ind_tbl_last)
9936                         break;
9937                 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0)
9938                         DRV_LOG(CRIT, "Port %u could not attach "
9939                                 "indirection table obj %p on rollback",
9940                                 dev->data->port_id, (void *)ind_tbl);
9941         }
9942         return ret;
9943 }
9944
9945 #ifndef HAVE_MLX5DV_DR
9946 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1))
9947 #else
9948 #define MLX5_DOMAIN_SYNC_FLOW \
9949         (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW)
9950 #endif
9951
9952 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
9953 {
9954         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
9955         const struct mlx5_flow_driver_ops *fops;
9956         int ret;
9957         struct rte_flow_attr attr = { .transfer = 0 };
9958
9959         fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr));
9960         ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW);
9961         if (ret > 0)
9962                 ret = -ret;
9963         return ret;
9964 }
9965
9966 const struct mlx5_flow_tunnel *
9967 mlx5_get_tof(const struct rte_flow_item *item,
9968              const struct rte_flow_action *action,
9969              enum mlx5_tof_rule_type *rule_type)
9970 {
9971         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
9972                 if (item->type == (typeof(item->type))
9973                                   MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) {
9974                         *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE;
9975                         return flow_items_to_tunnel(item);
9976                 }
9977         }
9978         for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) {
9979                 if (action->type == (typeof(action->type))
9980                                     MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) {
9981                         *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE;
9982                         return flow_actions_to_tunnel(action);
9983                 }
9984         }
9985         return NULL;
9986 }
9987
9988 /**
9989  * tunnel offload functionality is defined for DV environment only
9990  */
9991 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
9992 __extension__
9993 union tunnel_offload_mark {
9994         uint32_t val;
9995         struct {
9996                 uint32_t app_reserve:8;
9997                 uint32_t table_id:15;
9998                 uint32_t transfer:1;
9999                 uint32_t _unused_:8;
10000         };
10001 };
10002
10003 static bool
10004 mlx5_access_tunnel_offload_db
10005         (struct rte_eth_dev *dev,
10006          bool (*match)(struct rte_eth_dev *,
10007                        struct mlx5_flow_tunnel *, const void *),
10008          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
10009          void (*miss)(struct rte_eth_dev *, void *),
10010          void *ctx, bool lock_op);
10011
10012 static int
10013 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
10014                              struct rte_flow *flow,
10015                              const struct rte_flow_attr *attr,
10016                              const struct rte_flow_action *app_actions,
10017                              uint32_t flow_idx,
10018                              const struct mlx5_flow_tunnel *tunnel,
10019                              struct tunnel_default_miss_ctx *ctx,
10020                              struct rte_flow_error *error)
10021 {
10022         struct mlx5_priv *priv = dev->data->dev_private;
10023         struct mlx5_flow *dev_flow;
10024         struct rte_flow_attr miss_attr = *attr;
10025         const struct rte_flow_item miss_items[2] = {
10026                 {
10027                         .type = RTE_FLOW_ITEM_TYPE_ETH,
10028                         .spec = NULL,
10029                         .last = NULL,
10030                         .mask = NULL
10031                 },
10032                 {
10033                         .type = RTE_FLOW_ITEM_TYPE_END,
10034                         .spec = NULL,
10035                         .last = NULL,
10036                         .mask = NULL
10037                 }
10038         };
10039         union tunnel_offload_mark mark_id;
10040         struct rte_flow_action_mark miss_mark;
10041         struct rte_flow_action miss_actions[3] = {
10042                 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
10043                 [2] = { .type = RTE_FLOW_ACTION_TYPE_END,  .conf = NULL }
10044         };
10045         const struct rte_flow_action_jump *jump_data;
10046         uint32_t i, flow_table = 0; /* prevent compilation warning */
10047         struct flow_grp_info grp_info = {
10048                 .external = 1,
10049                 .transfer = attr->transfer,
10050                 .fdb_def_rule = !!priv->fdb_def_rule,
10051                 .std_tbl_fix = 0,
10052         };
10053         int ret;
10054
10055         if (!attr->transfer) {
10056                 uint32_t q_size;
10057
10058                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
10059                 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
10060                 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
10061                                          0, SOCKET_ID_ANY);
10062                 if (!ctx->queue)
10063                         return rte_flow_error_set
10064                                 (error, ENOMEM,
10065                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
10066                                 NULL, "invalid default miss RSS");
10067                 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
10068                 ctx->action_rss.level = 0,
10069                 ctx->action_rss.types = priv->rss_conf.rss_hf,
10070                 ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
10071                 ctx->action_rss.queue_num = priv->reta_idx_n,
10072                 ctx->action_rss.key = priv->rss_conf.rss_key,
10073                 ctx->action_rss.queue = ctx->queue;
10074                 if (!priv->reta_idx_n || !priv->rxqs_n)
10075                         return rte_flow_error_set
10076                                 (error, EINVAL,
10077                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
10078                                 NULL, "invalid port configuration");
10079                 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG))
10080                         ctx->action_rss.types = 0;
10081                 for (i = 0; i != priv->reta_idx_n; ++i)
10082                         ctx->queue[i] = (*priv->reta_idx)[i];
10083         } else {
10084                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
10085                 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
10086         }
10087         miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
10088         for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
10089         jump_data = app_actions->conf;
10090         miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
10091         miss_attr.group = jump_data->group;
10092         ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
10093                                        &flow_table, &grp_info, error);
10094         if (ret)
10095                 return rte_flow_error_set(error, EINVAL,
10096                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
10097                                           NULL, "invalid tunnel id");
10098         mark_id.app_reserve = 0;
10099         mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
10100         mark_id.transfer = !!attr->transfer;
10101         mark_id._unused_ = 0;
10102         miss_mark.id = mark_id.val;
10103         dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
10104                                     miss_items, miss_actions, flow_idx, error);
10105         if (!dev_flow)
10106                 return -rte_errno;
10107         dev_flow->flow = flow;
10108         dev_flow->external = true;
10109         dev_flow->tunnel = tunnel;
10110         dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE;
10111         /* Subflow object was created, we must include one in the list. */
10112         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
10113                       dev_flow->handle, next);
10114         DRV_LOG(DEBUG,
10115                 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
10116                 dev->data->port_id, tunnel->app_tunnel.type,
10117                 tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
10118         ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
10119                                   miss_actions, error);
10120         if (!ret)
10121                 ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
10122                                                   error);
10123
10124         return ret;
10125 }
10126
10127 static const struct mlx5_flow_tbl_data_entry  *
10128 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
10129 {
10130         struct mlx5_priv *priv = dev->data->dev_private;
10131         struct mlx5_dev_ctx_shared *sh = priv->sh;
10132         struct mlx5_list_entry *he;
10133         union tunnel_offload_mark mbits = { .val = mark };
10134         union mlx5_flow_tbl_key table_key = {
10135                 {
10136                         .level = tunnel_id_to_flow_tbl(mbits.table_id),
10137                         .id = 0,
10138                         .reserved = 0,
10139                         .dummy = 0,
10140                         .is_fdb = !!mbits.transfer,
10141                         .is_egress = 0,
10142                 }
10143         };
10144         struct mlx5_flow_cb_ctx ctx = {
10145                 .data = &table_key.v64,
10146         };
10147
10148         he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx);
10149         return he ?
10150                container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
10151 }
10152
10153 static void
10154 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx,
10155                                    struct mlx5_list_entry *entry)
10156 {
10157         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10158         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
10159
10160         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
10161                         tunnel_flow_tbl_to_id(tte->flow_table));
10162         mlx5_free(tte);
10163 }
10164
10165 static int
10166 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused,
10167                                   struct mlx5_list_entry *entry, void *cb_ctx)
10168 {
10169         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10170         union tunnel_tbl_key tbl = {
10171                 .val = *(uint64_t *)(ctx->data),
10172         };
10173         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
10174
10175         return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
10176 }
10177
10178 static struct mlx5_list_entry *
10179 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx)
10180 {
10181         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10182         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10183         struct tunnel_tbl_entry *tte;
10184         union tunnel_tbl_key tbl = {
10185                 .val = *(uint64_t *)(ctx->data),
10186         };
10187
10188         tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
10189                           sizeof(*tte), 0,
10190                           SOCKET_ID_ANY);
10191         if (!tte)
10192                 goto err;
10193         mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
10194                           &tte->flow_table);
10195         if (tte->flow_table >= MLX5_MAX_TABLES) {
10196                 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
10197                         tte->flow_table);
10198                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
10199                                 tte->flow_table);
10200                 goto err;
10201         } else if (!tte->flow_table) {
10202                 goto err;
10203         }
10204         tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
10205         tte->tunnel_id = tbl.tunnel_id;
10206         tte->group = tbl.group;
10207         return &tte->hash;
10208 err:
10209         if (tte)
10210                 mlx5_free(tte);
10211         return NULL;
10212 }
10213
10214 static struct mlx5_list_entry *
10215 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused,
10216                                   struct mlx5_list_entry *oentry,
10217                                   void *cb_ctx __rte_unused)
10218 {
10219         struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte),
10220                                                    0, SOCKET_ID_ANY);
10221
10222         if (!tte)
10223                 return NULL;
10224         memcpy(tte, oentry, sizeof(*tte));
10225         return &tte->hash;
10226 }
10227
10228 static void
10229 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused,
10230                                        struct mlx5_list_entry *entry)
10231 {
10232         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
10233
10234         mlx5_free(tte);
10235 }
10236
10237 static uint32_t
10238 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
10239                                 const struct mlx5_flow_tunnel *tunnel,
10240                                 uint32_t group, uint32_t *table,
10241                                 struct rte_flow_error *error)
10242 {
10243         struct mlx5_list_entry *he;
10244         struct tunnel_tbl_entry *tte;
10245         union tunnel_tbl_key key = {
10246                 .tunnel_id = tunnel ? tunnel->tunnel_id : 0,
10247                 .group = group
10248         };
10249         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
10250         struct mlx5_hlist *group_hash;
10251         struct mlx5_flow_cb_ctx ctx = {
10252                 .data = &key.val,
10253         };
10254
10255         group_hash = tunnel ? tunnel->groups : thub->groups;
10256         he = mlx5_hlist_register(group_hash, key.val, &ctx);
10257         if (!he)
10258                 return rte_flow_error_set(error, EINVAL,
10259                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
10260                                           NULL,
10261                                           "tunnel group index not supported");
10262         tte = container_of(he, typeof(*tte), hash);
10263         *table = tte->flow_table;
10264         DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
10265                 dev->data->port_id, key.tunnel_id, group, *table);
10266         return 0;
10267 }
10268
10269 static void
10270 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
10271                       struct mlx5_flow_tunnel *tunnel)
10272 {
10273         struct mlx5_priv *priv = dev->data->dev_private;
10274         struct mlx5_indexed_pool *ipool;
10275
10276         DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
10277                 dev->data->port_id, tunnel->tunnel_id);
10278         LIST_REMOVE(tunnel, chain);
10279         mlx5_hlist_destroy(tunnel->groups);
10280         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
10281         mlx5_ipool_free(ipool, tunnel->tunnel_id);
10282 }
10283
10284 static bool
10285 mlx5_access_tunnel_offload_db
10286         (struct rte_eth_dev *dev,
10287          bool (*match)(struct rte_eth_dev *,
10288                        struct mlx5_flow_tunnel *, const void *),
10289          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
10290          void (*miss)(struct rte_eth_dev *, void *),
10291          void *ctx, bool lock_op)
10292 {
10293         bool verdict = false;
10294         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
10295         struct mlx5_flow_tunnel *tunnel;
10296
10297         rte_spinlock_lock(&thub->sl);
10298         LIST_FOREACH(tunnel, &thub->tunnels, chain) {
10299                 verdict = match(dev, tunnel, (const void *)ctx);
10300                 if (verdict)
10301                         break;
10302         }
10303         if (!lock_op)
10304                 rte_spinlock_unlock(&thub->sl);
10305         if (verdict && hit)
10306                 hit(dev, tunnel, ctx);
10307         if (!verdict && miss)
10308                 miss(dev, ctx);
10309         if (lock_op)
10310                 rte_spinlock_unlock(&thub->sl);
10311
10312         return verdict;
10313 }
10314
10315 struct tunnel_db_find_tunnel_id_ctx {
10316         uint32_t tunnel_id;
10317         struct mlx5_flow_tunnel *tunnel;
10318 };
10319
10320 static bool
10321 find_tunnel_id_match(struct rte_eth_dev *dev,
10322                      struct mlx5_flow_tunnel *tunnel, const void *x)
10323 {
10324         const struct tunnel_db_find_tunnel_id_ctx *ctx = x;
10325
10326         RTE_SET_USED(dev);
10327         return tunnel->tunnel_id == ctx->tunnel_id;
10328 }
10329
10330 static void
10331 find_tunnel_id_hit(struct rte_eth_dev *dev,
10332                    struct mlx5_flow_tunnel *tunnel, void *x)
10333 {
10334         struct tunnel_db_find_tunnel_id_ctx *ctx = x;
10335         RTE_SET_USED(dev);
10336         ctx->tunnel = tunnel;
10337 }
10338
10339 static struct mlx5_flow_tunnel *
10340 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
10341 {
10342         struct tunnel_db_find_tunnel_id_ctx ctx = {
10343                 .tunnel_id = id,
10344         };
10345
10346         mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match,
10347                                       find_tunnel_id_hit, NULL, &ctx, true);
10348
10349         return ctx.tunnel;
10350 }
10351
10352 static struct mlx5_flow_tunnel *
10353 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
10354                           const struct rte_flow_tunnel *app_tunnel)
10355 {
10356         struct mlx5_priv *priv = dev->data->dev_private;
10357         struct mlx5_indexed_pool *ipool;
10358         struct mlx5_flow_tunnel *tunnel;
10359         uint32_t id;
10360
10361         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
10362         tunnel = mlx5_ipool_zmalloc(ipool, &id);
10363         if (!tunnel)
10364                 return NULL;
10365         if (id >= MLX5_MAX_TUNNELS) {
10366                 mlx5_ipool_free(ipool, id);
10367                 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
10368                 return NULL;
10369         }
10370         tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true,
10371                                            priv->sh,
10372                                            mlx5_flow_tunnel_grp2tbl_create_cb,
10373                                            mlx5_flow_tunnel_grp2tbl_match_cb,
10374                                            mlx5_flow_tunnel_grp2tbl_remove_cb,
10375                                            mlx5_flow_tunnel_grp2tbl_clone_cb,
10376                                         mlx5_flow_tunnel_grp2tbl_clone_free_cb);
10377         if (!tunnel->groups) {
10378                 mlx5_ipool_free(ipool, id);
10379                 return NULL;
10380         }
10381         /* initiate new PMD tunnel */
10382         memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
10383         tunnel->tunnel_id = id;
10384         tunnel->action.type = (typeof(tunnel->action.type))
10385                               MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
10386         tunnel->action.conf = tunnel;
10387         tunnel->item.type = (typeof(tunnel->item.type))
10388                             MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
10389         tunnel->item.spec = tunnel;
10390         tunnel->item.last = NULL;
10391         tunnel->item.mask = NULL;
10392
10393         DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
10394                 dev->data->port_id, tunnel->tunnel_id);
10395
10396         return tunnel;
10397 }
10398
10399 struct tunnel_db_get_tunnel_ctx {
10400         const struct rte_flow_tunnel *app_tunnel;
10401         struct mlx5_flow_tunnel *tunnel;
10402 };
10403
10404 static bool get_tunnel_match(struct rte_eth_dev *dev,
10405                              struct mlx5_flow_tunnel *tunnel, const void *x)
10406 {
10407         const struct tunnel_db_get_tunnel_ctx *ctx = x;
10408
10409         RTE_SET_USED(dev);
10410         return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel,
10411                        sizeof(*ctx->app_tunnel));
10412 }
10413
10414 static void get_tunnel_hit(struct rte_eth_dev *dev,
10415                            struct mlx5_flow_tunnel *tunnel, void *x)
10416 {
10417         /* called under tunnel spinlock protection */
10418         struct tunnel_db_get_tunnel_ctx *ctx = x;
10419
10420         RTE_SET_USED(dev);
10421         tunnel->refctn++;
10422         ctx->tunnel = tunnel;
10423 }
10424
10425 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x)
10426 {
10427         /* called under tunnel spinlock protection */
10428         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
10429         struct tunnel_db_get_tunnel_ctx *ctx = x;
10430
10431         rte_spinlock_unlock(&thub->sl);
10432         ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel);
10433         rte_spinlock_lock(&thub->sl);
10434         if (ctx->tunnel) {
10435                 ctx->tunnel->refctn = 1;
10436                 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain);
10437         }
10438 }
10439
10440
10441 static int
10442 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
10443                      const struct rte_flow_tunnel *app_tunnel,
10444                      struct mlx5_flow_tunnel **tunnel)
10445 {
10446         struct tunnel_db_get_tunnel_ctx ctx = {
10447                 .app_tunnel = app_tunnel,
10448         };
10449
10450         mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit,
10451                                       get_tunnel_miss, &ctx, true);
10452         *tunnel = ctx.tunnel;
10453         return ctx.tunnel ? 0 : -ENOMEM;
10454 }
10455
10456 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
10457 {
10458         struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
10459
10460         if (!thub)
10461                 return;
10462         if (!LIST_EMPTY(&thub->tunnels))
10463                 DRV_LOG(WARNING, "port %u tunnels present", port_id);
10464         mlx5_hlist_destroy(thub->groups);
10465         mlx5_free(thub);
10466 }
10467
10468 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
10469 {
10470         int err;
10471         struct mlx5_flow_tunnel_hub *thub;
10472
10473         thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
10474                            0, SOCKET_ID_ANY);
10475         if (!thub)
10476                 return -ENOMEM;
10477         LIST_INIT(&thub->tunnels);
10478         rte_spinlock_init(&thub->sl);
10479         thub->groups = mlx5_hlist_create("flow groups", 64,
10480                                          false, true, sh,
10481                                          mlx5_flow_tunnel_grp2tbl_create_cb,
10482                                          mlx5_flow_tunnel_grp2tbl_match_cb,
10483                                          mlx5_flow_tunnel_grp2tbl_remove_cb,
10484                                          mlx5_flow_tunnel_grp2tbl_clone_cb,
10485                                         mlx5_flow_tunnel_grp2tbl_clone_free_cb);
10486         if (!thub->groups) {
10487                 err = -rte_errno;
10488                 goto err;
10489         }
10490         sh->tunnel_hub = thub;
10491
10492         return 0;
10493
10494 err:
10495         if (thub->groups)
10496                 mlx5_hlist_destroy(thub->groups);
10497         if (thub)
10498                 mlx5_free(thub);
10499         return err;
10500 }
10501
10502 static inline int
10503 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
10504                           struct rte_flow_tunnel *tunnel,
10505                           struct rte_flow_error *error)
10506 {
10507         struct mlx5_priv *priv = dev->data->dev_private;
10508
10509         if (!priv->sh->config.dv_flow_en)
10510                 return rte_flow_error_set(error, ENOTSUP,
10511                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
10512                                           "flow DV interface is off");
10513         if (!is_tunnel_offload_active(dev))
10514                 return rte_flow_error_set(error, ENOTSUP,
10515                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
10516                                           "tunnel offload was not activated");
10517         if (!tunnel)
10518                 return rte_flow_error_set(error, EINVAL,
10519                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
10520                                           "no application tunnel");
10521         switch (tunnel->type) {
10522         default:
10523                 return rte_flow_error_set(error, EINVAL,
10524                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
10525                                           "unsupported tunnel type");
10526         case RTE_FLOW_ITEM_TYPE_VXLAN:
10527         case RTE_FLOW_ITEM_TYPE_GRE:
10528         case RTE_FLOW_ITEM_TYPE_NVGRE:
10529         case RTE_FLOW_ITEM_TYPE_GENEVE:
10530                 break;
10531         }
10532         return 0;
10533 }
10534
10535 static int
10536 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
10537                     struct rte_flow_tunnel *app_tunnel,
10538                     struct rte_flow_action **actions,
10539                     uint32_t *num_of_actions,
10540                     struct rte_flow_error *error)
10541 {
10542         struct mlx5_flow_tunnel *tunnel;
10543         int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error);
10544
10545         if (ret)
10546                 return ret;
10547         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
10548         if (ret < 0) {
10549                 return rte_flow_error_set(error, ret,
10550                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
10551                                           "failed to initialize pmd tunnel");
10552         }
10553         *actions = &tunnel->action;
10554         *num_of_actions = 1;
10555         return 0;
10556 }
10557
10558 static int
10559 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
10560                        struct rte_flow_tunnel *app_tunnel,
10561                        struct rte_flow_item **items,
10562                        uint32_t *num_of_items,
10563                        struct rte_flow_error *error)
10564 {
10565         struct mlx5_flow_tunnel *tunnel;
10566         int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error);
10567
10568         if (ret)
10569                 return ret;
10570         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
10571         if (ret < 0) {
10572                 return rte_flow_error_set(error, ret,
10573                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
10574                                           "failed to initialize pmd tunnel");
10575         }
10576         *items = &tunnel->item;
10577         *num_of_items = 1;
10578         return 0;
10579 }
10580
10581 struct tunnel_db_element_release_ctx {
10582         struct rte_flow_item *items;
10583         struct rte_flow_action *actions;
10584         uint32_t num_elements;
10585         struct rte_flow_error *error;
10586         int ret;
10587 };
10588
10589 static bool
10590 tunnel_element_release_match(struct rte_eth_dev *dev,
10591                              struct mlx5_flow_tunnel *tunnel, const void *x)
10592 {
10593         const struct tunnel_db_element_release_ctx *ctx = x;
10594
10595         RTE_SET_USED(dev);
10596         if (ctx->num_elements != 1)
10597                 return false;
10598         else if (ctx->items)
10599                 return ctx->items == &tunnel->item;
10600         else if (ctx->actions)
10601                 return ctx->actions == &tunnel->action;
10602
10603         return false;
10604 }
10605
10606 static void
10607 tunnel_element_release_hit(struct rte_eth_dev *dev,
10608                            struct mlx5_flow_tunnel *tunnel, void *x)
10609 {
10610         struct tunnel_db_element_release_ctx *ctx = x;
10611         ctx->ret = 0;
10612         if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
10613                 mlx5_flow_tunnel_free(dev, tunnel);
10614 }
10615
10616 static void
10617 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x)
10618 {
10619         struct tunnel_db_element_release_ctx *ctx = x;
10620         RTE_SET_USED(dev);
10621         ctx->ret = rte_flow_error_set(ctx->error, EINVAL,
10622                                       RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
10623                                       "invalid argument");
10624 }
10625
10626 static int
10627 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
10628                        struct rte_flow_item *pmd_items,
10629                        uint32_t num_items, struct rte_flow_error *err)
10630 {
10631         struct tunnel_db_element_release_ctx ctx = {
10632                 .items = pmd_items,
10633                 .actions = NULL,
10634                 .num_elements = num_items,
10635                 .error = err,
10636         };
10637
10638         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
10639                                       tunnel_element_release_hit,
10640                                       tunnel_element_release_miss, &ctx, false);
10641
10642         return ctx.ret;
10643 }
10644
10645 static int
10646 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
10647                          struct rte_flow_action *pmd_actions,
10648                          uint32_t num_actions, struct rte_flow_error *err)
10649 {
10650         struct tunnel_db_element_release_ctx ctx = {
10651                 .items = NULL,
10652                 .actions = pmd_actions,
10653                 .num_elements = num_actions,
10654                 .error = err,
10655         };
10656
10657         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
10658                                       tunnel_element_release_hit,
10659                                       tunnel_element_release_miss, &ctx, false);
10660
10661         return ctx.ret;
10662 }
10663
10664 static int
10665 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
10666                                   struct rte_mbuf *m,
10667                                   struct rte_flow_restore_info *info,
10668                                   struct rte_flow_error *err)
10669 {
10670         uint64_t ol_flags = m->ol_flags;
10671         const struct mlx5_flow_tbl_data_entry *tble;
10672         const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID;
10673
10674         if (!is_tunnel_offload_active(dev)) {
10675                 info->flags = 0;
10676                 return 0;
10677         }
10678
10679         if ((ol_flags & mask) != mask)
10680                 goto err;
10681         tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
10682         if (!tble) {
10683                 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
10684                         dev->data->port_id, m->hash.fdir.hi);
10685                 goto err;
10686         }
10687         MLX5_ASSERT(tble->tunnel);
10688         memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
10689         info->group_id = tble->group_id;
10690         info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
10691                       RTE_FLOW_RESTORE_INFO_GROUP_ID |
10692                       RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
10693
10694         return 0;
10695
10696 err:
10697         return rte_flow_error_set(err, EINVAL,
10698                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10699                                   "failed to get restore info");
10700 }
10701
10702 #else /* HAVE_IBV_FLOW_DV_SUPPORT */
10703 static int
10704 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev,
10705                            __rte_unused struct rte_flow_tunnel *app_tunnel,
10706                            __rte_unused struct rte_flow_action **actions,
10707                            __rte_unused uint32_t *num_of_actions,
10708                            __rte_unused struct rte_flow_error *error)
10709 {
10710         return -ENOTSUP;
10711 }
10712
10713 static int
10714 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev,
10715                        __rte_unused struct rte_flow_tunnel *app_tunnel,
10716                        __rte_unused struct rte_flow_item **items,
10717                        __rte_unused uint32_t *num_of_items,
10718                        __rte_unused struct rte_flow_error *error)
10719 {
10720         return -ENOTSUP;
10721 }
10722
10723 static int
10724 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev,
10725                               __rte_unused struct rte_flow_item *pmd_items,
10726                               __rte_unused uint32_t num_items,
10727                               __rte_unused struct rte_flow_error *err)
10728 {
10729         return -ENOTSUP;
10730 }
10731
10732 static int
10733 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev,
10734                                 __rte_unused struct rte_flow_action *pmd_action,
10735                                 __rte_unused uint32_t num_actions,
10736                                 __rte_unused struct rte_flow_error *err)
10737 {
10738         return -ENOTSUP;
10739 }
10740
10741 static int
10742 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev,
10743                                   __rte_unused struct rte_mbuf *m,
10744                                   __rte_unused struct rte_flow_restore_info *i,
10745                                   __rte_unused struct rte_flow_error *err)
10746 {
10747         return -ENOTSUP;
10748 }
10749
10750 static int
10751 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev,
10752                              __rte_unused struct rte_flow *flow,
10753                              __rte_unused const struct rte_flow_attr *attr,
10754                              __rte_unused const struct rte_flow_action *actions,
10755                              __rte_unused uint32_t flow_idx,
10756                              __rte_unused const struct mlx5_flow_tunnel *tunnel,
10757                              __rte_unused struct tunnel_default_miss_ctx *ctx,
10758                              __rte_unused struct rte_flow_error *error)
10759 {
10760         return -ENOTSUP;
10761 }
10762
10763 static struct mlx5_flow_tunnel *
10764 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev,
10765                     __rte_unused uint32_t id)
10766 {
10767         return NULL;
10768 }
10769
10770 static void
10771 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev,
10772                       __rte_unused struct mlx5_flow_tunnel *tunnel)
10773 {
10774 }
10775
10776 static uint32_t
10777 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev,
10778                                 __rte_unused const struct mlx5_flow_tunnel *t,
10779                                 __rte_unused uint32_t group,
10780                                 __rte_unused uint32_t *table,
10781                                 struct rte_flow_error *error)
10782 {
10783         return rte_flow_error_set(error, ENOTSUP,
10784                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10785                                   "tunnel offload requires DV support");
10786 }
10787
10788 void
10789 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh,
10790                         __rte_unused  uint16_t port_id)
10791 {
10792 }
10793 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
10794
10795 /* Flex flow item API */
10796 static struct rte_flow_item_flex_handle *
10797 mlx5_flow_flex_item_create(struct rte_eth_dev *dev,
10798                            const struct rte_flow_item_flex_conf *conf,
10799                            struct rte_flow_error *error)
10800 {
10801         static const char err_msg[] = "flex item creation unsupported";
10802         struct mlx5_priv *priv = dev->data->dev_private;
10803         struct rte_flow_attr attr = { .transfer = 0 };
10804         const struct mlx5_flow_driver_ops *fops =
10805                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
10806
10807         if (!priv->pci_dev) {
10808                 rte_flow_error_set(error, ENOTSUP,
10809                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10810                                    "create flex item on PF only");
10811                 return NULL;
10812         }
10813         switch (priv->pci_dev->id.device_id) {
10814         case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF:
10815         case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF:
10816                 break;
10817         default:
10818                 rte_flow_error_set(error, ENOTSUP,
10819                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10820                                    "flex item available on BlueField ports only");
10821                 return NULL;
10822         }
10823         if (!fops->item_create) {
10824                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
10825                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
10826                                    NULL, err_msg);
10827                 return NULL;
10828         }
10829         return fops->item_create(dev, conf, error);
10830 }
10831
10832 static int
10833 mlx5_flow_flex_item_release(struct rte_eth_dev *dev,
10834                             const struct rte_flow_item_flex_handle *handle,
10835                             struct rte_flow_error *error)
10836 {
10837         static const char err_msg[] = "flex item release unsupported";
10838         struct rte_flow_attr attr = { .transfer = 0 };
10839         const struct mlx5_flow_driver_ops *fops =
10840                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
10841
10842         if (!fops->item_release) {
10843                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
10844                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
10845                                    NULL, err_msg);
10846                 return -rte_errno;
10847         }
10848         return fops->item_release(dev, handle, error);
10849 }
10850
10851 static void
10852 mlx5_dbg__print_pattern(const struct rte_flow_item *item)
10853 {
10854         int ret;
10855         struct rte_flow_error error;
10856
10857         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
10858                 char *item_name;
10859                 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name,
10860                                     sizeof(item_name),
10861                                     (void *)(uintptr_t)item->type, &error);
10862                 if (ret > 0)
10863                         printf("%s ", item_name);
10864                 else
10865                         printf("%d\n", (int)item->type);
10866         }
10867         printf("END\n");
10868 }
10869
10870 static int
10871 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item)
10872 {
10873         const struct rte_flow_item_udp *spec = udp_item->spec;
10874         const struct rte_flow_item_udp *mask = udp_item->mask;
10875         uint16_t udp_dport = 0;
10876
10877         if (spec != NULL) {
10878                 if (!mask)
10879                         mask = &rte_flow_item_udp_mask;
10880                 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port &
10881                                 mask->hdr.dst_port);
10882         }
10883         return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN);
10884 }
10885
10886 static const struct mlx5_flow_expand_node *
10887 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern,
10888                 unsigned int item_idx,
10889                 const struct mlx5_flow_expand_node graph[],
10890                 const struct mlx5_flow_expand_node *node)
10891 {
10892         const struct rte_flow_item *item = pattern + item_idx, *prev_item;
10893
10894         if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN &&
10895                         node != NULL &&
10896                         node->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
10897                 /*
10898                  * The expansion node is VXLAN and it is also the last
10899                  * expandable item in the pattern, so need to continue
10900                  * expansion of the inner tunnel.
10901                  */
10902                 MLX5_ASSERT(item_idx > 0);
10903                 prev_item = pattern + item_idx - 1;
10904                 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP);
10905                 if (mlx5_flow_is_std_vxlan_port(prev_item))
10906                         return &graph[MLX5_EXPANSION_STD_VXLAN];
10907                 return &graph[MLX5_EXPANSION_L3_VXLAN];
10908         }
10909         return node;
10910 }
10911
10912 /* Map of Verbs to Flow priority with 8 Verbs priorities. */
10913 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
10914         { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
10915 };
10916
10917 /* Map of Verbs to Flow priority with 16 Verbs priorities. */
10918 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
10919         { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
10920         { 9, 10, 11 }, { 12, 13, 14 },
10921 };
10922
10923 /**
10924  * Discover the number of available flow priorities.
10925  *
10926  * @param dev
10927  *   Ethernet device.
10928  *
10929  * @return
10930  *   On success, number of available flow priorities.
10931  *   On failure, a negative errno-style code and rte_errno is set.
10932  */
10933 int
10934 mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
10935 {
10936         static const uint16_t vprio[] = {8, 16};
10937         const struct mlx5_priv *priv = dev->data->dev_private;
10938         const struct mlx5_flow_driver_ops *fops;
10939         enum mlx5_flow_drv_type type;
10940         int ret;
10941
10942         type = mlx5_flow_os_get_type();
10943         if (type == MLX5_FLOW_TYPE_MAX) {
10944                 type = MLX5_FLOW_TYPE_VERBS;
10945                 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en)
10946                         type = MLX5_FLOW_TYPE_DV;
10947         }
10948         fops = flow_get_drv_ops(type);
10949         if (fops->discover_priorities == NULL) {
10950                 DRV_LOG(ERR, "Priority discovery not supported");
10951                 rte_errno = ENOTSUP;
10952                 return -rte_errno;
10953         }
10954         ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio));
10955         if (ret < 0)
10956                 return ret;
10957         switch (ret) {
10958         case 8:
10959                 ret = RTE_DIM(priority_map_3);
10960                 break;
10961         case 16:
10962                 ret = RTE_DIM(priority_map_5);
10963                 break;
10964         default:
10965                 rte_errno = ENOTSUP;
10966                 DRV_LOG(ERR,
10967                         "port %u maximum priority: %d expected 8/16",
10968                         dev->data->port_id, ret);
10969                 return -rte_errno;
10970         }
10971         DRV_LOG(INFO, "port %u supported flow priorities:"
10972                 " 0-%d for ingress or egress root table,"
10973                 " 0-%d for non-root table or transfer root table.",
10974                 dev->data->port_id, ret - 2,
10975                 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1);
10976         return ret;
10977 }
10978
10979 /**
10980  * Adjust flow priority based on the highest layer and the request priority.
10981  *
10982  * @param[in] dev
10983  *   Pointer to the Ethernet device structure.
10984  * @param[in] priority
10985  *   The rule base priority.
10986  * @param[in] subpriority
10987  *   The priority based on the items.
10988  *
10989  * @return
10990  *   The new priority.
10991  */
10992 uint32_t
10993 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
10994                           uint32_t subpriority)
10995 {
10996         uint32_t res = 0;
10997         struct mlx5_priv *priv = dev->data->dev_private;
10998
10999         switch (priv->sh->flow_max_priority) {
11000         case RTE_DIM(priority_map_3):
11001                 res = priority_map_3[priority][subpriority];
11002                 break;
11003         case RTE_DIM(priority_map_5):
11004                 res = priority_map_5[priority][subpriority];
11005                 break;
11006         }
11007         return  res;
11008 }