drivers/net/mlx5/mlx5_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2016 6WIND S.A.
   3  * Copyright 2016 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <stdbool.h>
  10 #include <sys/queue.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_ether.h>
  14 #include <ethdev_driver.h>
  15 #include <rte_eal_paging.h>
  16 #include <rte_flow.h>
  17 #include <rte_cycles.h>
  18 #include <rte_flow_driver.h>
  19 #include <rte_malloc.h>
  20 #include <rte_ip.h>
  21
  22 #include <mlx5_glue.h>
  23 #include <mlx5_devx_cmds.h>
  24 #include <mlx5_prm.h>
  25 #include <mlx5_malloc.h>
  26
  27 #include "mlx5_defs.h"
  28 #include "mlx5.h"
  29 #include "mlx5_flow.h"
  30 #include "mlx5_flow_os.h"
  31 #include "mlx5_rx.h"
  32 #include "mlx5_tx.h"
  33 #include "mlx5_common_os.h"
  34 #include "rte_pmd_mlx5.h"
  35
  36 struct tunnel_default_miss_ctx {
  37         uint16_t *queue;
  38         __extension__
  39         union {
  40                 struct rte_flow_action_rss action_rss;
  41                 struct rte_flow_action_queue miss_queue;
  42                 struct rte_flow_action_jump miss_jump;
  43                 uint8_t raw[0];
  44         };
  45 };
  46
  47 static int
  48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
  49                              struct rte_flow *flow,
  50                              const struct rte_flow_attr *attr,
  51                              const struct rte_flow_action *app_actions,
  52                              uint32_t flow_idx,
  53                              const struct mlx5_flow_tunnel *tunnel,
  54                              struct tunnel_default_miss_ctx *ctx,
  55                              struct rte_flow_error *error);
  56 static struct mlx5_flow_tunnel *
  57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id);
  58 static void
  59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel);
  60 static uint32_t
  61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
  62                                 const struct mlx5_flow_tunnel *tunnel,
  63                                 uint32_t group, uint32_t *table,
  64                                 struct rte_flow_error *error);
  65
  66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void);
  67 static void mlx5_flow_pop_thread_workspace(void);
  68
  69
  70 /** Device flow drivers. */
  71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
  72
  73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
  74
  75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
  76         [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
  77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  78         [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
  79 #endif
  80         [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
  81         [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
  82 };
  83
  84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */
  85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \
  86         (const int []){ \
  87                 __VA_ARGS__, 0, \
  88         }
  89
  90 /** Node object of input graph for mlx5_flow_expand_rss(). */
  91 struct mlx5_flow_expand_node {
  92         const int *const next;
  93         /**<
  94          * List of next node indexes. Index 0 is interpreted as a terminator.
  95          */
  96         const enum rte_flow_item_type type;
  97         /**< Pattern item type of current node. */
  98         uint64_t rss_types;
  99         /**<
 100          * RSS types bit-field associated with this node
 101          * (see ETH_RSS_* definitions).
 102          */
 103         uint64_t node_flags;
 104         /**<
 105          *  Bit-fields that define how the node is used in the expansion.
 106          * (see MLX5_EXPANSION_NODE_* definitions).
 107          */
 108 };
 109
 110 /* Optional expand field. The expansion alg will not go deeper. */
 111 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0)
 112
 113 /* The node is not added implicitly as expansion to the flow pattern.
 114  * If the node type does not match the flow pattern item type, the
 115  * expansion alg will go deeper to its next items.
 116  * In the current implementation, the list of next nodes indexes can
 117  * have up to one node with this flag set and it has to be the last
 118  * node index (before the list terminator).
 119  */
 120 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1)
 121
 122 /** Object returned by mlx5_flow_expand_rss(). */
 123 struct mlx5_flow_expand_rss {
 124         uint32_t entries;
 125         /**< Number of entries @p patterns and @p priorities. */
 126         struct {
 127                 struct rte_flow_item *pattern; /**< Expanded pattern array. */
 128                 uint32_t priority; /**< Priority offset for each expansion. */
 129         } entry[];
 130 };
 131
 132 static void
 133 mlx5_dbg__print_pattern(const struct rte_flow_item *item);
 134
 135 static bool
 136 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item)
 137 {
 138         switch (item->type) {
 139         case RTE_FLOW_ITEM_TYPE_ETH:
 140         case RTE_FLOW_ITEM_TYPE_VLAN:
 141         case RTE_FLOW_ITEM_TYPE_IPV4:
 142         case RTE_FLOW_ITEM_TYPE_IPV6:
 143         case RTE_FLOW_ITEM_TYPE_UDP:
 144         case RTE_FLOW_ITEM_TYPE_TCP:
 145         case RTE_FLOW_ITEM_TYPE_VXLAN:
 146         case RTE_FLOW_ITEM_TYPE_NVGRE:
 147         case RTE_FLOW_ITEM_TYPE_GRE:
 148         case RTE_FLOW_ITEM_TYPE_GENEVE:
 149         case RTE_FLOW_ITEM_TYPE_MPLS:
 150         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 151         case RTE_FLOW_ITEM_TYPE_GRE_KEY:
 152         case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
 153         case RTE_FLOW_ITEM_TYPE_GTP:
 154                 return true;
 155         default:
 156                 break;
 157         }
 158         return false;
 159 }
 160
 161 static enum rte_flow_item_type
 162 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
 163 {
 164         enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID;
 165         uint16_t ether_type = 0;
 166         uint16_t ether_type_m;
 167         uint8_t ip_next_proto = 0;
 168         uint8_t ip_next_proto_m;
 169
 170         if (item == NULL || item->spec == NULL)
 171                 return ret;
 172         switch (item->type) {
 173         case RTE_FLOW_ITEM_TYPE_ETH:
 174                 if (item->mask)
 175                         ether_type_m = ((const struct rte_flow_item_eth *)
 176                                                 (item->mask))->type;
 177                 else
 178                         ether_type_m = rte_flow_item_eth_mask.type;
 179                 if (ether_type_m != RTE_BE16(0xFFFF))
 180                         break;
 181                 ether_type = ((const struct rte_flow_item_eth *)
 182                                 (item->spec))->type;
 183                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 184                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 185                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 186                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 187                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 188                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 189                 else
 190                         ret = RTE_FLOW_ITEM_TYPE_END;
 191                 break;
 192         case RTE_FLOW_ITEM_TYPE_VLAN:
 193                 if (item->mask)
 194                         ether_type_m = ((const struct rte_flow_item_vlan *)
 195                                                 (item->mask))->inner_type;
 196                 else
 197                         ether_type_m = rte_flow_item_vlan_mask.inner_type;
 198                 if (ether_type_m != RTE_BE16(0xFFFF))
 199                         break;
 200                 ether_type = ((const struct rte_flow_item_vlan *)
 201                                 (item->spec))->inner_type;
 202                 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4)
 203                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 204                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6)
 205                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 206                 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN)
 207                         ret = RTE_FLOW_ITEM_TYPE_VLAN;
 208                 else
 209                         ret = RTE_FLOW_ITEM_TYPE_END;
 210                 break;
 211         case RTE_FLOW_ITEM_TYPE_IPV4:
 212                 if (item->mask)
 213                         ip_next_proto_m = ((const struct rte_flow_item_ipv4 *)
 214                                         (item->mask))->hdr.next_proto_id;
 215                 else
 216                         ip_next_proto_m =
 217                                 rte_flow_item_ipv4_mask.hdr.next_proto_id;
 218                 if (ip_next_proto_m != 0xFF)
 219                         break;
 220                 ip_next_proto = ((const struct rte_flow_item_ipv4 *)
 221                                 (item->spec))->hdr.next_proto_id;
 222                 if (ip_next_proto == IPPROTO_UDP)
 223                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 224                 else if (ip_next_proto == IPPROTO_TCP)
 225                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 226                 else if (ip_next_proto == IPPROTO_IP)
 227                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 228                 else if (ip_next_proto == IPPROTO_IPV6)
 229                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 230                 else
 231                         ret = RTE_FLOW_ITEM_TYPE_END;
 232                 break;
 233         case RTE_FLOW_ITEM_TYPE_IPV6:
 234                 if (item->mask)
 235                         ip_next_proto_m = ((const struct rte_flow_item_ipv6 *)
 236                                                 (item->mask))->hdr.proto;
 237                 else
 238                         ip_next_proto_m =
 239                                 rte_flow_item_ipv6_mask.hdr.proto;
 240                 if (ip_next_proto_m != 0xFF)
 241                         break;
 242                 ip_next_proto = ((const struct rte_flow_item_ipv6 *)
 243                                 (item->spec))->hdr.proto;
 244                 if (ip_next_proto == IPPROTO_UDP)
 245                         ret = RTE_FLOW_ITEM_TYPE_UDP;
 246                 else if (ip_next_proto == IPPROTO_TCP)
 247                         ret = RTE_FLOW_ITEM_TYPE_TCP;
 248                 else if (ip_next_proto == IPPROTO_IP)
 249                         ret = RTE_FLOW_ITEM_TYPE_IPV4;
 250                 else if (ip_next_proto == IPPROTO_IPV6)
 251                         ret = RTE_FLOW_ITEM_TYPE_IPV6;
 252                 else
 253                         ret = RTE_FLOW_ITEM_TYPE_END;
 254                 break;
 255         default:
 256                 ret = RTE_FLOW_ITEM_TYPE_VOID;
 257                 break;
 258         }
 259         return ret;
 260 }
 261
 262 #define MLX5_RSS_EXP_ELT_N 16
 263
 264 /**
 265  * Expand RSS flows into several possible flows according to the RSS hash
 266  * fields requested and the driver capabilities.
 267  *
 268  * @param[out] buf
 269  *   Buffer to store the result expansion.
 270  * @param[in] size
 271  *   Buffer size in bytes. If 0, @p buf can be NULL.
 272  * @param[in] pattern
 273  *   User flow pattern.
 274  * @param[in] types
 275  *   RSS types to expand (see ETH_RSS_* definitions).
 276  * @param[in] graph
 277  *   Input graph to expand @p pattern according to @p types.
 278  * @param[in] graph_root_index
 279  *   Index of root node in @p graph, typically 0.
 280  *
 281  * @return
 282  *   A positive value representing the size of @p buf in bytes regardless of
 283  *   @p size on success, a negative errno value otherwise and rte_errno is
 284  *   set, the following errors are defined:
 285  *
 286  *   -E2BIG: graph-depth @p graph is too deep.
 287  *   -EINVAL: @p size has not enough space for expanded pattern.
 288  */
 289 static int
 290 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 291                      const struct rte_flow_item *pattern, uint64_t types,
 292                      const struct mlx5_flow_expand_node graph[],
 293                      int graph_root_index)
 294 {
 295         const struct rte_flow_item *item;
 296         const struct mlx5_flow_expand_node *node = &graph[graph_root_index];
 297         const int *next_node;
 298         const int *stack[MLX5_RSS_EXP_ELT_N];
 299         int stack_pos = 0;
 300         struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N];
 301         unsigned int i;
 302         size_t lsize;
 303         size_t user_pattern_size = 0;
 304         void *addr = NULL;
 305         const struct mlx5_flow_expand_node *next = NULL;
 306         struct rte_flow_item missed_item;
 307         int missed = 0;
 308         int elt = 0;
 309         const struct rte_flow_item *last_item = NULL;
 310
 311         memset(&missed_item, 0, sizeof(missed_item));
 312         lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
 313                 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]);
 314         if (lsize > size)
 315                 return -EINVAL;
 316         buf->entry[0].priority = 0;
 317         buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N];
 318         buf->entries = 0;
 319         addr = buf->entry[0].pattern;
 320         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 321                 if (!mlx5_flow_is_rss_expandable_item(item)) {
 322                         user_pattern_size += sizeof(*item);
 323                         continue;
 324                 }
 325                 last_item = item;
 326                 i = 0;
 327                 while (node->next && node->next[i]) {
 328                         next = &graph[node->next[i]];
 329                         if (next->type == item->type)
 330                                 break;
 331                         if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) {
 332                                 node = next;
 333                                 i = 0;
 334                         } else {
 335                                 ++i;
 336                         }
 337                 }
 338                 if (next)
 339                         node = next;
 340                 user_pattern_size += sizeof(*item);
 341         }
 342         user_pattern_size += sizeof(*item); /* Handle END item. */
 343         lsize += user_pattern_size;
 344         if (lsize > size)
 345                 return -EINVAL;
 346         /* Copy the user pattern in the first entry of the buffer. */
 347         rte_memcpy(addr, pattern, user_pattern_size);
 348         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 349         buf->entries = 1;
 350         /* Start expanding. */
 351         memset(flow_items, 0, sizeof(flow_items));
 352         user_pattern_size -= sizeof(*item);
 353         /*
 354          * Check if the last valid item has spec set, need complete pattern,
 355          * and the pattern can be used for expansion.
 356          */
 357         missed_item.type = mlx5_flow_expand_rss_item_complete(last_item);
 358         if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 359                 /* Item type END indicates expansion is not required. */
 360                 return lsize;
 361         }
 362         if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 363                 next = NULL;
 364                 missed = 1;
 365                 for (i = 0; node->next && node->next[i]; ++i) {
 366                         next = &graph[node->next[i]];
 367                         if (next->type == missed_item.type) {
 368                                 flow_items[0].type = missed_item.type;
 369                                 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 370                                 break;
 371                         }
 372                         next = NULL;
 373                 }
 374         }
 375         if (next && missed) {
 376                 elt = 2; /* missed item + item end. */
 377                 node = next;
 378                 lsize += elt * sizeof(*item) + user_pattern_size;
 379                 if (lsize > size)
 380                         return -EINVAL;
 381                 if (node->rss_types & types) {
 382                         buf->entry[buf->entries].priority = 1;
 383                         buf->entry[buf->entries].pattern = addr;
 384                         buf->entries++;
 385                         rte_memcpy(addr, buf->entry[0].pattern,
 386                                    user_pattern_size);
 387                         addr = (void *)(((uintptr_t)addr) + user_pattern_size);
 388                         rte_memcpy(addr, flow_items, elt * sizeof(*item));
 389                         addr = (void *)(((uintptr_t)addr) +
 390                                         elt * sizeof(*item));
 391                 }
 392         }
 393         memset(flow_items, 0, sizeof(flow_items));
 394         next_node = node->next;
 395         while (next_node) {
 396                 /*
 397                  * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT
 398                  * flag set, because they were not found in the flow pattern.
 399                  */
 400                 node = &graph[*next_node];
 401                 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT))
 402                         break;
 403                 next_node = node->next;
 404         }
 405         stack[stack_pos] = next_node;
 406         node = next_node ? &graph[*next_node] : NULL;
 407         while (node) {
 408                 flow_items[stack_pos].type = node->type;
 409                 if (node->rss_types & types) {
 410                         size_t n;
 411                         /*
 412                          * compute the number of items to copy from the
 413                          * expansion and copy it.
 414                          * When the stack_pos is 0, there are 1 element in it,
 415                          * plus the addition END item.
 416                          */
 417                         elt = stack_pos + 2;
 418                         flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
 419                         lsize += elt * sizeof(*item) + user_pattern_size;
 420                         if (lsize > size)
 421                                 return -EINVAL;
 422                         n = elt * sizeof(*item);
 423                         buf->entry[buf->entries].priority =
 424                                 stack_pos + 1 + missed;
 425                         buf->entry[buf->entries].pattern = addr;
 426                         buf->entries++;
 427                         rte_memcpy(addr, buf->entry[0].pattern,
 428                                    user_pattern_size);
 429                         addr = (void *)(((uintptr_t)addr) +
 430                                         user_pattern_size);
 431                         rte_memcpy(addr, &missed_item,
 432                                    missed * sizeof(*item));
 433                         addr = (void *)(((uintptr_t)addr) +
 434                                 missed * sizeof(*item));
 435                         rte_memcpy(addr, flow_items, n);
 436                         addr = (void *)(((uintptr_t)addr) + n);
 437                 }
 438                 /* Go deeper. */
 439                 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) &&
 440                                 node->next) {
 441                         next_node = node->next;
 442                         if (stack_pos++ == MLX5_RSS_EXP_ELT_N) {
 443                                 rte_errno = E2BIG;
 444                                 return -rte_errno;
 445                         }
 446                         stack[stack_pos] = next_node;
 447                 } else if (*(next_node + 1)) {
 448                         /* Follow up with the next possibility. */
 449                         ++next_node;
 450                 } else {
 451                         /* Move to the next path. */
 452                         if (stack_pos)
 453                                 next_node = stack[--stack_pos];
 454                         next_node++;
 455                         stack[stack_pos] = next_node;
 456                 }
 457                 node = *next_node ? &graph[*next_node] : NULL;
 458         };
 459         return lsize;
 460 }
 461
 462 enum mlx5_expansion {
 463         MLX5_EXPANSION_ROOT,
 464         MLX5_EXPANSION_ROOT_OUTER,
 465         MLX5_EXPANSION_OUTER_ETH,
 466         MLX5_EXPANSION_OUTER_VLAN,
 467         MLX5_EXPANSION_OUTER_IPV4,
 468         MLX5_EXPANSION_OUTER_IPV4_UDP,
 469         MLX5_EXPANSION_OUTER_IPV4_TCP,
 470         MLX5_EXPANSION_OUTER_IPV6,
 471         MLX5_EXPANSION_OUTER_IPV6_UDP,
 472         MLX5_EXPANSION_OUTER_IPV6_TCP,
 473         MLX5_EXPANSION_VXLAN,
 474         MLX5_EXPANSION_VXLAN_GPE,
 475         MLX5_EXPANSION_GRE,
 476         MLX5_EXPANSION_NVGRE,
 477         MLX5_EXPANSION_GRE_KEY,
 478         MLX5_EXPANSION_MPLS,
 479         MLX5_EXPANSION_ETH,
 480         MLX5_EXPANSION_VLAN,
 481         MLX5_EXPANSION_IPV4,
 482         MLX5_EXPANSION_IPV4_UDP,
 483         MLX5_EXPANSION_IPV4_TCP,
 484         MLX5_EXPANSION_IPV6,
 485         MLX5_EXPANSION_IPV6_UDP,
 486         MLX5_EXPANSION_IPV6_TCP,
 487         MLX5_EXPANSION_IPV6_FRAG_EXT,
 488         MLX5_EXPANSION_GTP
 489 };
 490
 491 /** Supported expansion of items. */
 492 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 493         [MLX5_EXPANSION_ROOT] = {
 494                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 495                                                   MLX5_EXPANSION_IPV4,
 496                                                   MLX5_EXPANSION_IPV6),
 497                 .type = RTE_FLOW_ITEM_TYPE_END,
 498         },
 499         [MLX5_EXPANSION_ROOT_OUTER] = {
 500                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
 501                                                   MLX5_EXPANSION_OUTER_IPV4,
 502                                                   MLX5_EXPANSION_OUTER_IPV6),
 503                 .type = RTE_FLOW_ITEM_TYPE_END,
 504         },
 505         [MLX5_EXPANSION_OUTER_ETH] = {
 506                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 507                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 508                 .rss_types = 0,
 509         },
 510         [MLX5_EXPANSION_OUTER_VLAN] = {
 511                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 512                                                   MLX5_EXPANSION_OUTER_IPV6),
 513                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 514                 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
 515         },
 516         [MLX5_EXPANSION_OUTER_IPV4] = {
 517                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 518                         (MLX5_EXPANSION_OUTER_IPV4_UDP,
 519                          MLX5_EXPANSION_OUTER_IPV4_TCP,
 520                          MLX5_EXPANSION_GRE,
 521                          MLX5_EXPANSION_NVGRE,
 522                          MLX5_EXPANSION_IPV4,
 523                          MLX5_EXPANSION_IPV6),
 524                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 525                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 526                         ETH_RSS_NONFRAG_IPV4_OTHER,
 527         },
 528         [MLX5_EXPANSION_OUTER_IPV4_UDP] = {
 529                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 530                                                   MLX5_EXPANSION_VXLAN_GPE,
 531                                                   MLX5_EXPANSION_MPLS,
 532                                                   MLX5_EXPANSION_GTP),
 533                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 534                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 535         },
 536         [MLX5_EXPANSION_OUTER_IPV4_TCP] = {
 537                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 538                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 539         },
 540         [MLX5_EXPANSION_OUTER_IPV6] = {
 541                 .next = MLX5_FLOW_EXPAND_RSS_NEXT
 542                         (MLX5_EXPANSION_OUTER_IPV6_UDP,
 543                          MLX5_EXPANSION_OUTER_IPV6_TCP,
 544                          MLX5_EXPANSION_IPV4,
 545                          MLX5_EXPANSION_IPV6,
 546                          MLX5_EXPANSION_GRE,
 547                          MLX5_EXPANSION_NVGRE),
 548                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 549                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 550                         ETH_RSS_NONFRAG_IPV6_OTHER,
 551         },
 552         [MLX5_EXPANSION_OUTER_IPV6_UDP] = {
 553                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
 554                                                   MLX5_EXPANSION_VXLAN_GPE,
 555                                                   MLX5_EXPANSION_MPLS,
 556                                                   MLX5_EXPANSION_GTP),
 557                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 558                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 559         },
 560         [MLX5_EXPANSION_OUTER_IPV6_TCP] = {
 561                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 562                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 563         },
 564         [MLX5_EXPANSION_VXLAN] = {
 565                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 566                                                   MLX5_EXPANSION_IPV4,
 567                                                   MLX5_EXPANSION_IPV6),
 568                 .type = RTE_FLOW_ITEM_TYPE_VXLAN,
 569         },
 570         [MLX5_EXPANSION_VXLAN_GPE] = {
 571                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 572                                                   MLX5_EXPANSION_IPV4,
 573                                                   MLX5_EXPANSION_IPV6),
 574                 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 575         },
 576         [MLX5_EXPANSION_GRE] = {
 577                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 578                                                   MLX5_EXPANSION_IPV6,
 579                                                   MLX5_EXPANSION_GRE_KEY,
 580                                                   MLX5_EXPANSION_MPLS),
 581                 .type = RTE_FLOW_ITEM_TYPE_GRE,
 582         },
 583         [MLX5_EXPANSION_GRE_KEY] = {
 584                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 585                                                   MLX5_EXPANSION_IPV6,
 586                                                   MLX5_EXPANSION_MPLS),
 587                 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY,
 588                 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
 589         },
 590         [MLX5_EXPANSION_NVGRE] = {
 591                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
 592                 .type = RTE_FLOW_ITEM_TYPE_NVGRE,
 593         },
 594         [MLX5_EXPANSION_MPLS] = {
 595                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 596                                                   MLX5_EXPANSION_IPV6,
 597                                                   MLX5_EXPANSION_ETH),
 598                 .type = RTE_FLOW_ITEM_TYPE_MPLS,
 599                 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
 600         },
 601         [MLX5_EXPANSION_ETH] = {
 602                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 603                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 604         },
 605         [MLX5_EXPANSION_VLAN] = {
 606                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 607                                                   MLX5_EXPANSION_IPV6),
 608                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 609                 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
 610         },
 611         [MLX5_EXPANSION_IPV4] = {
 612                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
 613                                                   MLX5_EXPANSION_IPV4_TCP),
 614                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 615                 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 616                         ETH_RSS_NONFRAG_IPV4_OTHER,
 617         },
 618         [MLX5_EXPANSION_IPV4_UDP] = {
 619                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 620                 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 621         },
 622         [MLX5_EXPANSION_IPV4_TCP] = {
 623                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 624                 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 625         },
 626         [MLX5_EXPANSION_IPV6] = {
 627                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
 628                                                   MLX5_EXPANSION_IPV6_TCP,
 629                                                   MLX5_EXPANSION_IPV6_FRAG_EXT),
 630                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 631                 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 632                         ETH_RSS_NONFRAG_IPV6_OTHER,
 633         },
 634         [MLX5_EXPANSION_IPV6_UDP] = {
 635                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 636                 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 637         },
 638         [MLX5_EXPANSION_IPV6_TCP] = {
 639                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 640                 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 641         },
 642         [MLX5_EXPANSION_IPV6_FRAG_EXT] = {
 643                 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT,
 644         },
 645         [MLX5_EXPANSION_GTP] = {
 646                 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 647                                                   MLX5_EXPANSION_IPV6),
 648                 .type = RTE_FLOW_ITEM_TYPE_GTP,
 649         },
 650 };
 651
 652 static struct rte_flow_action_handle *
 653 mlx5_action_handle_create(struct rte_eth_dev *dev,
 654                           const struct rte_flow_indir_action_conf *conf,
 655                           const struct rte_flow_action *action,
 656                           struct rte_flow_error *error);
 657 static int mlx5_action_handle_destroy
 658                                 (struct rte_eth_dev *dev,
 659                                  struct rte_flow_action_handle *handle,
 660                                  struct rte_flow_error *error);
 661 static int mlx5_action_handle_update
 662                                 (struct rte_eth_dev *dev,
 663                                  struct rte_flow_action_handle *handle,
 664                                  const void *update,
 665                                  struct rte_flow_error *error);
 666 static int mlx5_action_handle_query
 667                                 (struct rte_eth_dev *dev,
 668                                  const struct rte_flow_action_handle *handle,
 669                                  void *data,
 670                                  struct rte_flow_error *error);
 671 static int
 672 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
 673                     struct rte_flow_tunnel *app_tunnel,
 674                     struct rte_flow_action **actions,
 675                     uint32_t *num_of_actions,
 676                     struct rte_flow_error *error);
 677 static int
 678 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
 679                        struct rte_flow_tunnel *app_tunnel,
 680                        struct rte_flow_item **items,
 681                        uint32_t *num_of_items,
 682                        struct rte_flow_error *error);
 683 static int
 684 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
 685                               struct rte_flow_item *pmd_items,
 686                               uint32_t num_items, struct rte_flow_error *err);
 687 static int
 688 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
 689                                 struct rte_flow_action *pmd_actions,
 690                                 uint32_t num_actions,
 691                                 struct rte_flow_error *err);
 692 static int
 693 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
 694                                   struct rte_mbuf *m,
 695                                   struct rte_flow_restore_info *info,
 696                                   struct rte_flow_error *err);
 697
 698 static const struct rte_flow_ops mlx5_flow_ops = {
 699         .validate = mlx5_flow_validate,
 700         .create = mlx5_flow_create,
 701         .destroy = mlx5_flow_destroy,
 702         .flush = mlx5_flow_flush,
 703         .isolate = mlx5_flow_isolate,
 704         .query = mlx5_flow_query,
 705         .dev_dump = mlx5_flow_dev_dump,
 706         .get_aged_flows = mlx5_flow_get_aged_flows,
 707         .action_handle_create = mlx5_action_handle_create,
 708         .action_handle_destroy = mlx5_action_handle_destroy,
 709         .action_handle_update = mlx5_action_handle_update,
 710         .action_handle_query = mlx5_action_handle_query,
 711         .tunnel_decap_set = mlx5_flow_tunnel_decap_set,
 712         .tunnel_match = mlx5_flow_tunnel_match,
 713         .tunnel_action_decap_release = mlx5_flow_tunnel_action_release,
 714         .tunnel_item_release = mlx5_flow_tunnel_item_release,
 715         .get_restore_info = mlx5_flow_tunnel_get_restore_info,
 716 };
 717
 718 /* Tunnel information. */
 719 struct mlx5_flow_tunnel_info {
 720         uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
 721         uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
 722 };
 723
 724 static struct mlx5_flow_tunnel_info tunnels_info[] = {
 725         {
 726                 .tunnel = MLX5_FLOW_LAYER_VXLAN,
 727                 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
 728         },
 729         {
 730                 .tunnel = MLX5_FLOW_LAYER_GENEVE,
 731                 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP,
 732         },
 733         {
 734                 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 735                 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 736         },
 737         {
 738                 .tunnel = MLX5_FLOW_LAYER_GRE,
 739                 .ptype = RTE_PTYPE_TUNNEL_GRE,
 740         },
 741         {
 742                 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
 743                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP,
 744         },
 745         {
 746                 .tunnel = MLX5_FLOW_LAYER_MPLS,
 747                 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
 748         },
 749         {
 750                 .tunnel = MLX5_FLOW_LAYER_NVGRE,
 751                 .ptype = RTE_PTYPE_TUNNEL_NVGRE,
 752         },
 753         {
 754                 .tunnel = MLX5_FLOW_LAYER_IPIP,
 755                 .ptype = RTE_PTYPE_TUNNEL_IP,
 756         },
 757         {
 758                 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP,
 759                 .ptype = RTE_PTYPE_TUNNEL_IP,
 760         },
 761         {
 762                 .tunnel = MLX5_FLOW_LAYER_GTP,
 763                 .ptype = RTE_PTYPE_TUNNEL_GTPU,
 764         },
 765 };
 766
 767
 768
 769 /**
 770  * Translate tag ID to register.
 771  *
 772  * @param[in] dev
 773  *   Pointer to the Ethernet device structure.
 774  * @param[in] feature
 775  *   The feature that request the register.
 776  * @param[in] id
 777  *   The request register ID.
 778  * @param[out] error
 779  *   Error description in case of any.
 780  *
 781  * @return
 782  *   The request register on success, a negative errno
 783  *   value otherwise and rte_errno is set.
 784  */
 785 int
 786 mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 787                      enum mlx5_feature_name feature,
 788                      uint32_t id,
 789                      struct rte_flow_error *error)
 790 {
 791         struct mlx5_priv *priv = dev->data->dev_private;
 792         struct mlx5_dev_config *config = &priv->config;
 793         enum modify_reg start_reg;
 794         bool skip_mtr_reg = false;
 795
 796         switch (feature) {
 797         case MLX5_HAIRPIN_RX:
 798                 return REG_B;
 799         case MLX5_HAIRPIN_TX:
 800                 return REG_A;
 801         case MLX5_METADATA_RX:
 802                 switch (config->dv_xmeta_en) {
 803                 case MLX5_XMETA_MODE_LEGACY:
 804                         return REG_B;
 805                 case MLX5_XMETA_MODE_META16:
 806                         return REG_C_0;
 807                 case MLX5_XMETA_MODE_META32:
 808                         return REG_C_1;
 809                 }
 810                 break;
 811         case MLX5_METADATA_TX:
 812                 return REG_A;
 813         case MLX5_METADATA_FDB:
 814                 switch (config->dv_xmeta_en) {
 815                 case MLX5_XMETA_MODE_LEGACY:
 816                         return REG_NON;
 817                 case MLX5_XMETA_MODE_META16:
 818                         return REG_C_0;
 819                 case MLX5_XMETA_MODE_META32:
 820                         return REG_C_1;
 821                 }
 822                 break;
 823         case MLX5_FLOW_MARK:
 824                 switch (config->dv_xmeta_en) {
 825                 case MLX5_XMETA_MODE_LEGACY:
 826                         return REG_NON;
 827                 case MLX5_XMETA_MODE_META16:
 828                         return REG_C_1;
 829                 case MLX5_XMETA_MODE_META32:
 830                         return REG_C_0;
 831                 }
 832                 break;
 833         case MLX5_MTR_ID:
 834                 /*
 835                  * If meter color and meter id share one register, flow match
 836                  * should use the meter color register for match.
 837                  */
 838                 if (priv->mtr_reg_share)
 839                         return priv->mtr_color_reg;
 840                 else
 841                         return priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 842                                REG_C_3;
 843         case MLX5_MTR_COLOR:
 844         case MLX5_ASO_FLOW_HIT:
 845         case MLX5_ASO_CONNTRACK:
 846                 /* All features use the same REG_C. */
 847                 MLX5_ASSERT(priv->mtr_color_reg != REG_NON);
 848                 return priv->mtr_color_reg;
 849         case MLX5_COPY_MARK:
 850                 /*
 851                  * Metadata COPY_MARK register using is in meter suffix sub
 852                  * flow while with meter. It's safe to share the same register.
 853                  */
 854                 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3;
 855         case MLX5_APP_TAG:
 856                 /*
 857                  * If meter is enable, it will engage the register for color
 858                  * match and flow match. If meter color match is not using the
 859                  * REG_C_2, need to skip the REG_C_x be used by meter color
 860                  * match.
 861                  * If meter is disable, free to use all available registers.
 862                  */
 863                 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 :
 864                             (priv->mtr_reg_share ? REG_C_3 : REG_C_4);
 865                 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2);
 866                 if (id > (uint32_t)(REG_C_7 - start_reg))
 867                         return rte_flow_error_set(error, EINVAL,
 868                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 869                                                   NULL, "invalid tag id");
 870                 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 871                         return rte_flow_error_set(error, ENOTSUP,
 872                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 873                                                   NULL, "unsupported tag id");
 874                 /*
 875                  * This case means meter is using the REG_C_x great than 2.
 876                  * Take care not to conflict with meter color REG_C_x.
 877                  * If the available index REG_C_y >= REG_C_x, skip the
 878                  * color register.
 879                  */
 880                 if (skip_mtr_reg && config->flow_mreg_c
 881                     [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 882                         if (id >= (uint32_t)(REG_C_7 - start_reg))
 883                                 return rte_flow_error_set(error, EINVAL,
 884                                                        RTE_FLOW_ERROR_TYPE_ITEM,
 885                                                         NULL, "invalid tag id");
 886                         if (config->flow_mreg_c
 887                             [id + 1 + start_reg - REG_C_0] != REG_NON)
 888                                 return config->flow_mreg_c
 889                                                [id + 1 + start_reg - REG_C_0];
 890                         return rte_flow_error_set(error, ENOTSUP,
 891                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 892                                                   NULL, "unsupported tag id");
 893                 }
 894                 return config->flow_mreg_c[id + start_reg - REG_C_0];
 895         }
 896         MLX5_ASSERT(false);
 897         return rte_flow_error_set(error, EINVAL,
 898                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 899                                   NULL, "invalid feature name");
 900 }
 901
 902 /**
 903  * Check extensive flow metadata register support.
 904  *
 905  * @param dev
 906  *   Pointer to rte_eth_dev structure.
 907  *
 908  * @return
 909  *   True if device supports extensive flow metadata register, otherwise false.
 910  */
 911 bool
 912 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 913 {
 914         struct mlx5_priv *priv = dev->data->dev_private;
 915         struct mlx5_dev_config *config = &priv->config;
 916
 917         /*
 918          * Having available reg_c can be regarded inclusively as supporting
 919          * extensive flow metadata register, which could mean,
 920          * - metadata register copy action by modify header.
 921          * - 16 modify header actions is supported.
 922          * - reg_c's are preserved across different domain (FDB and NIC) on
 923          *   packet loopback by flow lookup miss.
 924          */
 925         return config->flow_mreg_c[2] != REG_NON;
 926 }
 927
 928 /**
 929  * Get the lowest priority.
 930  *
 931  * @param[in] dev
 932  *   Pointer to the Ethernet device structure.
 933  * @param[in] attributes
 934  *   Pointer to device flow rule attributes.
 935  *
 936  * @return
 937  *   The value of lowest priority of flow.
 938  */
 939 uint32_t
 940 mlx5_get_lowest_priority(struct rte_eth_dev *dev,
 941                           const struct rte_flow_attr *attr)
 942 {
 943         struct mlx5_priv *priv = dev->data->dev_private;
 944
 945         if (!attr->group && !attr->transfer)
 946                 return priv->config.flow_prio - 2;
 947         return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1;
 948 }
 949
 950 /**
 951  * Calculate matcher priority of the flow.
 952  *
 953  * @param[in] dev
 954  *   Pointer to the Ethernet device structure.
 955  * @param[in] attr
 956  *   Pointer to device flow rule attributes.
 957  * @param[in] subpriority
 958  *   The priority based on the items.
 959  * @return
 960  *   The matcher priority of the flow.
 961  */
 962 uint16_t
 963 mlx5_get_matcher_priority(struct rte_eth_dev *dev,
 964                           const struct rte_flow_attr *attr,
 965                           uint32_t subpriority)
 966 {
 967         uint16_t priority = (uint16_t)attr->priority;
 968         struct mlx5_priv *priv = dev->data->dev_private;
 969
 970         if (!attr->group && !attr->transfer) {
 971                 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
 972                         priority = priv->config.flow_prio - 1;
 973                 return mlx5_os_flow_adjust_priority(dev, priority, subpriority);
 974         }
 975         if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
 976                 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO;
 977         return priority * 3 + subpriority;
 978 }
 979
 980 /**
 981  * Verify the @p item specifications (spec, last, mask) are compatible with the
 982  * NIC capabilities.
 983  *
 984  * @param[in] item
 985  *   Item specification.
 986  * @param[in] mask
 987  *   @p item->mask or flow default bit-masks.
 988  * @param[in] nic_mask
 989  *   Bit-masks covering supported fields by the NIC to compare with user mask.
 990  * @param[in] size
 991  *   Bit-masks size in bytes.
 992  * @param[in] range_accepted
 993  *   True if range of values is accepted for specific fields, false otherwise.
 994  * @param[out] error
 995  *   Pointer to error structure.
 996  *
 997  * @return
 998  *   0 on success, a negative errno value otherwise and rte_errno is set.
 999  */
1000 int
1001 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
1002                           const uint8_t *mask,
1003                           const uint8_t *nic_mask,
1004                           unsigned int size,
1005                           bool range_accepted,
1006                           struct rte_flow_error *error)
1007 {
1008         unsigned int i;
1009
1010         MLX5_ASSERT(nic_mask);
1011         for (i = 0; i < size; ++i)
1012                 if ((nic_mask[i] | mask[i]) != nic_mask[i])
1013                         return rte_flow_error_set(error, ENOTSUP,
1014                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1015                                                   item,
1016                                                   "mask enables non supported"
1017                                                   " bits");
1018         if (!item->spec && (item->mask || item->last))
1019                 return rte_flow_error_set(error, EINVAL,
1020                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1021                                           "mask/last without a spec is not"
1022                                           " supported");
1023         if (item->spec && item->last && !range_accepted) {
1024                 uint8_t spec[size];
1025                 uint8_t last[size];
1026                 unsigned int i;
1027                 int ret;
1028
1029                 for (i = 0; i < size; ++i) {
1030                         spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
1031                         last[i] = ((const uint8_t *)item->last)[i] & mask[i];
1032                 }
1033                 ret = memcmp(spec, last, size);
1034                 if (ret != 0)
1035                         return rte_flow_error_set(error, EINVAL,
1036                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1037                                                   item,
1038                                                   "range is not valid");
1039         }
1040         return 0;
1041 }
1042
1043 /**
1044  * Adjust the hash fields according to the @p flow information.
1045  *
1046  * @param[in] dev_flow.
1047  *   Pointer to the mlx5_flow.
1048  * @param[in] tunnel
1049  *   1 when the hash field is for a tunnel item.
1050  * @param[in] layer_types
1051  *   ETH_RSS_* types.
1052  * @param[in] hash_fields
1053  *   Item hash fields.
1054  *
1055  * @return
1056  *   The hash fields that should be used.
1057  */
1058 uint64_t
1059 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc,
1060                             int tunnel __rte_unused, uint64_t layer_types,
1061                             uint64_t hash_fields)
1062 {
1063 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1064         int rss_request_inner = rss_desc->level >= 2;
1065
1066         /* Check RSS hash level for tunnel. */
1067         if (tunnel && rss_request_inner)
1068                 hash_fields |= IBV_RX_HASH_INNER;
1069         else if (tunnel || rss_request_inner)
1070                 return 0;
1071 #endif
1072         /* Check if requested layer matches RSS hash fields. */
1073         if (!(rss_desc->types & layer_types))
1074                 return 0;
1075         return hash_fields;
1076 }
1077
1078 /**
1079  * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
1080  * if several tunnel rules are used on this queue, the tunnel ptype will be
1081  * cleared.
1082  *
1083  * @param rxq_ctrl
1084  *   Rx queue to update.
1085  */
1086 static void
1087 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
1088 {
1089         unsigned int i;
1090         uint32_t tunnel_ptype = 0;
1091
1092         /* Look up for the ptype to use. */
1093         for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
1094                 if (!rxq_ctrl->flow_tunnels_n[i])
1095                         continue;
1096                 if (!tunnel_ptype) {
1097                         tunnel_ptype = tunnels_info[i].ptype;
1098                 } else {
1099                         tunnel_ptype = 0;
1100                         break;
1101                 }
1102         }
1103         rxq_ctrl->rxq.tunnel = tunnel_ptype;
1104 }
1105
1106 /**
1107  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
1108  * flow.
1109  *
1110  * @param[in] dev
1111  *   Pointer to the Ethernet device structure.
1112  * @param[in] dev_handle
1113  *   Pointer to device flow handle structure.
1114  */
1115 void
1116 flow_drv_rxq_flags_set(struct rte_eth_dev *dev,
1117                        struct mlx5_flow_handle *dev_handle)
1118 {
1119         struct mlx5_priv *priv = dev->data->dev_private;
1120         const int mark = dev_handle->mark;
1121         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1122         struct mlx5_ind_table_obj *ind_tbl = NULL;
1123         unsigned int i;
1124
1125         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1126                 struct mlx5_hrxq *hrxq;
1127
1128                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1129                               dev_handle->rix_hrxq);
1130                 if (hrxq)
1131                         ind_tbl = hrxq->ind_table;
1132         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1133                 struct mlx5_shared_action_rss *shared_rss;
1134
1135                 shared_rss = mlx5_ipool_get
1136                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1137                          dev_handle->rix_srss);
1138                 if (shared_rss)
1139                         ind_tbl = shared_rss->ind_tbl;
1140         }
1141         if (!ind_tbl)
1142                 return;
1143         for (i = 0; i != ind_tbl->queues_n; ++i) {
1144                 int idx = ind_tbl->queues[i];
1145                 struct mlx5_rxq_ctrl *rxq_ctrl =
1146                         container_of((*priv->rxqs)[idx],
1147                                      struct mlx5_rxq_ctrl, rxq);
1148
1149                 /*
1150                  * To support metadata register copy on Tx loopback,
1151                  * this must be always enabled (metadata may arive
1152                  * from other port - not from local flows only.
1153                  */
1154                 if (priv->config.dv_flow_en &&
1155                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1156                     mlx5_flow_ext_mreg_supported(dev)) {
1157                         rxq_ctrl->rxq.mark = 1;
1158                         rxq_ctrl->flow_mark_n = 1;
1159                 } else if (mark) {
1160                         rxq_ctrl->rxq.mark = 1;
1161                         rxq_ctrl->flow_mark_n++;
1162                 }
1163                 if (tunnel) {
1164                         unsigned int j;
1165
1166                         /* Increase the counter matching the flow. */
1167                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1168                                 if ((tunnels_info[j].tunnel &
1169                                      dev_handle->layers) ==
1170                                     tunnels_info[j].tunnel) {
1171                                         rxq_ctrl->flow_tunnels_n[j]++;
1172                                         break;
1173                                 }
1174                         }
1175                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1176                 }
1177         }
1178 }
1179
1180 /**
1181  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
1182  *
1183  * @param[in] dev
1184  *   Pointer to the Ethernet device structure.
1185  * @param[in] flow
1186  *   Pointer to flow structure.
1187  */
1188 static void
1189 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
1190 {
1191         struct mlx5_priv *priv = dev->data->dev_private;
1192         uint32_t handle_idx;
1193         struct mlx5_flow_handle *dev_handle;
1194
1195         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1196                        handle_idx, dev_handle, next)
1197                 flow_drv_rxq_flags_set(dev, dev_handle);
1198 }
1199
1200 /**
1201  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1202  * device flow if no other flow uses it with the same kind of request.
1203  *
1204  * @param dev
1205  *   Pointer to Ethernet device.
1206  * @param[in] dev_handle
1207  *   Pointer to the device flow handle structure.
1208  */
1209 static void
1210 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev,
1211                         struct mlx5_flow_handle *dev_handle)
1212 {
1213         struct mlx5_priv *priv = dev->data->dev_private;
1214         const int mark = dev_handle->mark;
1215         const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
1216         struct mlx5_ind_table_obj *ind_tbl = NULL;
1217         unsigned int i;
1218
1219         if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1220                 struct mlx5_hrxq *hrxq;
1221
1222                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1223                               dev_handle->rix_hrxq);
1224                 if (hrxq)
1225                         ind_tbl = hrxq->ind_table;
1226         } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
1227                 struct mlx5_shared_action_rss *shared_rss;
1228
1229                 shared_rss = mlx5_ipool_get
1230                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
1231                          dev_handle->rix_srss);
1232                 if (shared_rss)
1233                         ind_tbl = shared_rss->ind_tbl;
1234         }
1235         if (!ind_tbl)
1236                 return;
1237         MLX5_ASSERT(dev->data->dev_started);
1238         for (i = 0; i != ind_tbl->queues_n; ++i) {
1239                 int idx = ind_tbl->queues[i];
1240                 struct mlx5_rxq_ctrl *rxq_ctrl =
1241                         container_of((*priv->rxqs)[idx],
1242                                      struct mlx5_rxq_ctrl, rxq);
1243
1244                 if (priv->config.dv_flow_en &&
1245                     priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
1246                     mlx5_flow_ext_mreg_supported(dev)) {
1247                         rxq_ctrl->rxq.mark = 1;
1248                         rxq_ctrl->flow_mark_n = 1;
1249                 } else if (mark) {
1250                         rxq_ctrl->flow_mark_n--;
1251                         rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
1252                 }
1253                 if (tunnel) {
1254                         unsigned int j;
1255
1256                         /* Decrease the counter matching the flow. */
1257                         for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
1258                                 if ((tunnels_info[j].tunnel &
1259                                      dev_handle->layers) ==
1260                                     tunnels_info[j].tunnel) {
1261                                         rxq_ctrl->flow_tunnels_n[j]--;
1262                                         break;
1263                                 }
1264                         }
1265                         flow_rxq_tunnel_ptype_update(rxq_ctrl);
1266                 }
1267         }
1268 }
1269
1270 /**
1271  * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
1272  * @p flow if no other flow uses it with the same kind of request.
1273  *
1274  * @param dev
1275  *   Pointer to Ethernet device.
1276  * @param[in] flow
1277  *   Pointer to the flow.
1278  */
1279 static void
1280 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
1281 {
1282         struct mlx5_priv *priv = dev->data->dev_private;
1283         uint32_t handle_idx;
1284         struct mlx5_flow_handle *dev_handle;
1285
1286         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1287                        handle_idx, dev_handle, next)
1288                 flow_drv_rxq_flags_trim(dev, dev_handle);
1289 }
1290
1291 /**
1292  * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
1293  *
1294  * @param dev
1295  *   Pointer to Ethernet device.
1296  */
1297 static void
1298 flow_rxq_flags_clear(struct rte_eth_dev *dev)
1299 {
1300         struct mlx5_priv *priv = dev->data->dev_private;
1301         unsigned int i;
1302
1303         for (i = 0; i != priv->rxqs_n; ++i) {
1304                 struct mlx5_rxq_ctrl *rxq_ctrl;
1305                 unsigned int j;
1306
1307                 if (!(*priv->rxqs)[i])
1308                         continue;
1309                 rxq_ctrl = container_of((*priv->rxqs)[i],
1310                                         struct mlx5_rxq_ctrl, rxq);
1311                 rxq_ctrl->flow_mark_n = 0;
1312                 rxq_ctrl->rxq.mark = 0;
1313                 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
1314                         rxq_ctrl->flow_tunnels_n[j] = 0;
1315                 rxq_ctrl->rxq.tunnel = 0;
1316         }
1317 }
1318
1319 /**
1320  * Set the Rx queue dynamic metadata (mask and offset) for a flow
1321  *
1322  * @param[in] dev
1323  *   Pointer to the Ethernet device structure.
1324  */
1325 void
1326 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev)
1327 {
1328         struct mlx5_priv *priv = dev->data->dev_private;
1329         struct mlx5_rxq_data *data;
1330         unsigned int i;
1331
1332         for (i = 0; i != priv->rxqs_n; ++i) {
1333                 if (!(*priv->rxqs)[i])
1334                         continue;
1335                 data = (*priv->rxqs)[i];
1336                 if (!rte_flow_dynf_metadata_avail()) {
1337                         data->dynf_meta = 0;
1338                         data->flow_meta_mask = 0;
1339                         data->flow_meta_offset = -1;
1340                         data->flow_meta_port_mask = 0;
1341                 } else {
1342                         data->dynf_meta = 1;
1343                         data->flow_meta_mask = rte_flow_dynf_metadata_mask;
1344                         data->flow_meta_offset = rte_flow_dynf_metadata_offs;
1345                         data->flow_meta_port_mask = priv->sh->dv_meta_mask;
1346                 }
1347         }
1348 }
1349
1350 /*
1351  * return a pointer to the desired action in the list of actions.
1352  *
1353  * @param[in] actions
1354  *   The list of actions to search the action in.
1355  * @param[in] action
1356  *   The action to find.
1357  *
1358  * @return
1359  *   Pointer to the action in the list, if found. NULL otherwise.
1360  */
1361 const struct rte_flow_action *
1362 mlx5_flow_find_action(const struct rte_flow_action *actions,
1363                       enum rte_flow_action_type action)
1364 {
1365         if (actions == NULL)
1366                 return NULL;
1367         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1368                 if (actions->type == action)
1369                         return actions;
1370         return NULL;
1371 }
1372
1373 /*
1374  * Validate the flag action.
1375  *
1376  * @param[in] action_flags
1377  *   Bit-fields that holds the actions detected until now.
1378  * @param[in] attr
1379  *   Attributes of flow that includes this action.
1380  * @param[out] error
1381  *   Pointer to error structure.
1382  *
1383  * @return
1384  *   0 on success, a negative errno value otherwise and rte_errno is set.
1385  */
1386 int
1387 mlx5_flow_validate_action_flag(uint64_t action_flags,
1388                                const struct rte_flow_attr *attr,
1389                                struct rte_flow_error *error)
1390 {
1391         if (action_flags & MLX5_FLOW_ACTION_MARK)
1392                 return rte_flow_error_set(error, EINVAL,
1393                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1394                                           "can't mark and flag in same flow");
1395         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1396                 return rte_flow_error_set(error, EINVAL,
1397                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1398                                           "can't have 2 flag"
1399                                           " actions in same flow");
1400         if (attr->egress)
1401                 return rte_flow_error_set(error, ENOTSUP,
1402                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1403                                           "flag action not supported for "
1404                                           "egress");
1405         return 0;
1406 }
1407
1408 /*
1409  * Validate the mark action.
1410  *
1411  * @param[in] action
1412  *   Pointer to the queue action.
1413  * @param[in] action_flags
1414  *   Bit-fields that holds the actions detected until now.
1415  * @param[in] attr
1416  *   Attributes of flow that includes this action.
1417  * @param[out] error
1418  *   Pointer to error structure.
1419  *
1420  * @return
1421  *   0 on success, a negative errno value otherwise and rte_errno is set.
1422  */
1423 int
1424 mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
1425                                uint64_t action_flags,
1426                                const struct rte_flow_attr *attr,
1427                                struct rte_flow_error *error)
1428 {
1429         const struct rte_flow_action_mark *mark = action->conf;
1430
1431         if (!mark)
1432                 return rte_flow_error_set(error, EINVAL,
1433                                           RTE_FLOW_ERROR_TYPE_ACTION,
1434                                           action,
1435                                           "configuration cannot be null");
1436         if (mark->id >= MLX5_FLOW_MARK_MAX)
1437                 return rte_flow_error_set(error, EINVAL,
1438                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1439                                           &mark->id,
1440                                           "mark id must in 0 <= id < "
1441                                           RTE_STR(MLX5_FLOW_MARK_MAX));
1442         if (action_flags & MLX5_FLOW_ACTION_FLAG)
1443                 return rte_flow_error_set(error, EINVAL,
1444                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1445                                           "can't flag and mark in same flow");
1446         if (action_flags & MLX5_FLOW_ACTION_MARK)
1447                 return rte_flow_error_set(error, EINVAL,
1448                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1449                                           "can't have 2 mark actions in same"
1450                                           " flow");
1451         if (attr->egress)
1452                 return rte_flow_error_set(error, ENOTSUP,
1453                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1454                                           "mark action not supported for "
1455                                           "egress");
1456         return 0;
1457 }
1458
1459 /*
1460  * Validate the drop action.
1461  *
1462  * @param[in] action_flags
1463  *   Bit-fields that holds the actions detected until now.
1464  * @param[in] attr
1465  *   Attributes of flow that includes this action.
1466  * @param[out] error
1467  *   Pointer to error structure.
1468  *
1469  * @return
1470  *   0 on success, a negative errno value otherwise and rte_errno is set.
1471  */
1472 int
1473 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused,
1474                                const struct rte_flow_attr *attr,
1475                                struct rte_flow_error *error)
1476 {
1477         if (attr->egress)
1478                 return rte_flow_error_set(error, ENOTSUP,
1479                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1480                                           "drop action not supported for "
1481                                           "egress");
1482         return 0;
1483 }
1484
1485 /*
1486  * Validate the queue action.
1487  *
1488  * @param[in] action
1489  *   Pointer to the queue action.
1490  * @param[in] action_flags
1491  *   Bit-fields that holds the actions detected until now.
1492  * @param[in] dev
1493  *   Pointer to the Ethernet device structure.
1494  * @param[in] attr
1495  *   Attributes of flow that includes this action.
1496  * @param[out] error
1497  *   Pointer to error structure.
1498  *
1499  * @return
1500  *   0 on success, a negative errno value otherwise and rte_errno is set.
1501  */
1502 int
1503 mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
1504                                 uint64_t action_flags,
1505                                 struct rte_eth_dev *dev,
1506                                 const struct rte_flow_attr *attr,
1507                                 struct rte_flow_error *error)
1508 {
1509         struct mlx5_priv *priv = dev->data->dev_private;
1510         const struct rte_flow_action_queue *queue = action->conf;
1511
1512         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1513                 return rte_flow_error_set(error, EINVAL,
1514                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1515                                           "can't have 2 fate actions in"
1516                                           " same flow");
1517         if (!priv->rxqs_n)
1518                 return rte_flow_error_set(error, EINVAL,
1519                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1520                                           NULL, "No Rx queues configured");
1521         if (queue->index >= priv->rxqs_n)
1522                 return rte_flow_error_set(error, EINVAL,
1523                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1524                                           &queue->index,
1525                                           "queue index out of range");
1526         if (!(*priv->rxqs)[queue->index])
1527                 return rte_flow_error_set(error, EINVAL,
1528                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1529                                           &queue->index,
1530                                           "queue is not configured");
1531         if (attr->egress)
1532                 return rte_flow_error_set(error, ENOTSUP,
1533                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1534                                           "queue action not supported for "
1535                                           "egress");
1536         return 0;
1537 }
1538
1539 /*
1540  * Validate the rss action.
1541  *
1542  * @param[in] dev
1543  *   Pointer to the Ethernet device structure.
1544  * @param[in] action
1545  *   Pointer to the queue action.
1546  * @param[out] error
1547  *   Pointer to error structure.
1548  *
1549  * @return
1550  *   0 on success, a negative errno value otherwise and rte_errno is set.
1551  */
1552 int
1553 mlx5_validate_action_rss(struct rte_eth_dev *dev,
1554                          const struct rte_flow_action *action,
1555                          struct rte_flow_error *error)
1556 {
1557         struct mlx5_priv *priv = dev->data->dev_private;
1558         const struct rte_flow_action_rss *rss = action->conf;
1559         enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED;
1560         unsigned int i;
1561
1562         if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
1563             rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
1564                 return rte_flow_error_set(error, ENOTSUP,
1565                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1566                                           &rss->func,
1567                                           "RSS hash function not supported");
1568 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
1569         if (rss->level > 2)
1570 #else
1571         if (rss->level > 1)
1572 #endif
1573                 return rte_flow_error_set(error, ENOTSUP,
1574                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1575                                           &rss->level,
1576                                           "tunnel RSS is not supported");
1577         /* allow RSS key_len 0 in case of NULL (default) RSS key. */
1578         if (rss->key_len == 0 && rss->key != NULL)
1579                 return rte_flow_error_set(error, ENOTSUP,
1580                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1581                                           &rss->key_len,
1582                                           "RSS hash key length 0");
1583         if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN)
1584                 return rte_flow_error_set(error, ENOTSUP,
1585                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1586                                           &rss->key_len,
1587                                           "RSS hash key too small");
1588         if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
1589                 return rte_flow_error_set(error, ENOTSUP,
1590                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1591                                           &rss->key_len,
1592                                           "RSS hash key too large");
1593         if (rss->queue_num > priv->config.ind_table_max_size)
1594                 return rte_flow_error_set(error, ENOTSUP,
1595                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1596                                           &rss->queue_num,
1597                                           "number of queues too large");
1598         if (rss->types & MLX5_RSS_HF_MASK)
1599                 return rte_flow_error_set(error, ENOTSUP,
1600                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1601                                           &rss->types,
1602                                           "some RSS protocols are not"
1603                                           " supported");
1604         if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) &&
1605             !(rss->types & ETH_RSS_IP))
1606                 return rte_flow_error_set(error, EINVAL,
1607                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1608                                           "L3 partial RSS requested but L3 RSS"
1609                                           " type not specified");
1610         if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) &&
1611             !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP)))
1612                 return rte_flow_error_set(error, EINVAL,
1613                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1614                                           "L4 partial RSS requested but L4 RSS"
1615                                           " type not specified");
1616         if (!priv->rxqs_n)
1617                 return rte_flow_error_set(error, EINVAL,
1618                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1619                                           NULL, "No Rx queues configured");
1620         if (!rss->queue_num)
1621                 return rte_flow_error_set(error, EINVAL,
1622                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1623                                           NULL, "No queues configured");
1624         for (i = 0; i != rss->queue_num; ++i) {
1625                 struct mlx5_rxq_ctrl *rxq_ctrl;
1626
1627                 if (rss->queue[i] >= priv->rxqs_n)
1628                         return rte_flow_error_set
1629                                 (error, EINVAL,
1630                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1631                                  &rss->queue[i], "queue index out of range");
1632                 if (!(*priv->rxqs)[rss->queue[i]])
1633                         return rte_flow_error_set
1634                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1635                                  &rss->queue[i], "queue is not configured");
1636                 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]],
1637                                         struct mlx5_rxq_ctrl, rxq);
1638                 if (i == 0)
1639                         rxq_type = rxq_ctrl->type;
1640                 if (rxq_type != rxq_ctrl->type)
1641                         return rte_flow_error_set
1642                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1643                                  &rss->queue[i],
1644                                  "combining hairpin and regular RSS queues is not supported");
1645         }
1646         return 0;
1647 }
1648
1649 /*
1650  * Validate the rss action.
1651  *
1652  * @param[in] action
1653  *   Pointer to the queue action.
1654  * @param[in] action_flags
1655  *   Bit-fields that holds the actions detected until now.
1656  * @param[in] dev
1657  *   Pointer to the Ethernet device structure.
1658  * @param[in] attr
1659  *   Attributes of flow that includes this action.
1660  * @param[in] item_flags
1661  *   Items that were detected.
1662  * @param[out] error
1663  *   Pointer to error structure.
1664  *
1665  * @return
1666  *   0 on success, a negative errno value otherwise and rte_errno is set.
1667  */
1668 int
1669 mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
1670                               uint64_t action_flags,
1671                               struct rte_eth_dev *dev,
1672                               const struct rte_flow_attr *attr,
1673                               uint64_t item_flags,
1674                               struct rte_flow_error *error)
1675 {
1676         const struct rte_flow_action_rss *rss = action->conf;
1677         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1678         int ret;
1679
1680         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1681                 return rte_flow_error_set(error, EINVAL,
1682                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1683                                           "can't have 2 fate actions"
1684                                           " in same flow");
1685         ret = mlx5_validate_action_rss(dev, action, error);
1686         if (ret)
1687                 return ret;
1688         if (attr->egress)
1689                 return rte_flow_error_set(error, ENOTSUP,
1690                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1691                                           "rss action not supported for "
1692                                           "egress");
1693         if (rss->level > 1 && !tunnel)
1694                 return rte_flow_error_set(error, EINVAL,
1695                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1696                                           "inner RSS is not supported for "
1697                                           "non-tunnel flows");
1698         if ((item_flags & MLX5_FLOW_LAYER_ECPRI) &&
1699             !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) {
1700                 return rte_flow_error_set(error, EINVAL,
1701                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
1702                                           "RSS on eCPRI is not supported now");
1703         }
1704         if ((item_flags & MLX5_FLOW_LAYER_MPLS) &&
1705             !(item_flags &
1706               (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) &&
1707             rss->level > 1)
1708                 return rte_flow_error_set(error, EINVAL,
1709                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1710                                           "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern");
1711         return 0;
1712 }
1713
1714 /*
1715  * Validate the default miss action.
1716  *
1717  * @param[in] action_flags
1718  *   Bit-fields that holds the actions detected until now.
1719  * @param[out] error
1720  *   Pointer to error structure.
1721  *
1722  * @return
1723  *   0 on success, a negative errno value otherwise and rte_errno is set.
1724  */
1725 int
1726 mlx5_flow_validate_action_default_miss(uint64_t action_flags,
1727                                 const struct rte_flow_attr *attr,
1728                                 struct rte_flow_error *error)
1729 {
1730         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
1731                 return rte_flow_error_set(error, EINVAL,
1732                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1733                                           "can't have 2 fate actions in"
1734                                           " same flow");
1735         if (attr->egress)
1736                 return rte_flow_error_set(error, ENOTSUP,
1737                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1738                                           "default miss action not supported "
1739                                           "for egress");
1740         if (attr->group)
1741                 return rte_flow_error_set(error, ENOTSUP,
1742                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL,
1743                                           "only group 0 is supported");
1744         if (attr->transfer)
1745                 return rte_flow_error_set(error, ENOTSUP,
1746                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1747                                           NULL, "transfer is not supported");
1748         return 0;
1749 }
1750
1751 /*
1752  * Validate the count action.
1753  *
1754  * @param[in] dev
1755  *   Pointer to the Ethernet device structure.
1756  * @param[in] attr
1757  *   Attributes of flow that includes this action.
1758  * @param[out] error
1759  *   Pointer to error structure.
1760  *
1761  * @return
1762  *   0 on success, a negative errno value otherwise and rte_errno is set.
1763  */
1764 int
1765 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
1766                                 const struct rte_flow_attr *attr,
1767                                 struct rte_flow_error *error)
1768 {
1769         if (attr->egress)
1770                 return rte_flow_error_set(error, ENOTSUP,
1771                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1772                                           "count action not supported for "
1773                                           "egress");
1774         return 0;
1775 }
1776
1777 /*
1778  * Validate the ASO CT action.
1779  *
1780  * @param[in] dev
1781  *   Pointer to the Ethernet device structure.
1782  * @param[in] conntrack
1783  *   Pointer to the CT action profile.
1784  * @param[out] error
1785  *   Pointer to error structure.
1786  *
1787  * @return
1788  *   0 on success, a negative errno value otherwise and rte_errno is set.
1789  */
1790 int
1791 mlx5_validate_action_ct(struct rte_eth_dev *dev,
1792                         const struct rte_flow_action_conntrack *conntrack,
1793                         struct rte_flow_error *error)
1794 {
1795         RTE_SET_USED(dev);
1796
1797         if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT)
1798                 return rte_flow_error_set(error, EINVAL,
1799                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1800                                           "Invalid CT state");
1801         if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST)
1802                 return rte_flow_error_set(error, EINVAL,
1803                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1804                                           "Invalid last TCP packet flag");
1805         return 0;
1806 }
1807
1808 /**
1809  * Verify the @p attributes will be correctly understood by the NIC and store
1810  * them in the @p flow if everything is correct.
1811  *
1812  * @param[in] dev
1813  *   Pointer to the Ethernet device structure.
1814  * @param[in] attributes
1815  *   Pointer to flow attributes
1816  * @param[out] error
1817  *   Pointer to error structure.
1818  *
1819  * @return
1820  *   0 on success, a negative errno value otherwise and rte_errno is set.
1821  */
1822 int
1823 mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
1824                               const struct rte_flow_attr *attributes,
1825                               struct rte_flow_error *error)
1826 {
1827         struct mlx5_priv *priv = dev->data->dev_private;
1828         uint32_t priority_max = priv->config.flow_prio - 1;
1829
1830         if (attributes->group)
1831                 return rte_flow_error_set(error, ENOTSUP,
1832                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1833                                           NULL, "groups is not supported");
1834         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
1835             attributes->priority >= priority_max)
1836                 return rte_flow_error_set(error, ENOTSUP,
1837                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1838                                           NULL, "priority out of range");
1839         if (attributes->egress)
1840                 return rte_flow_error_set(error, ENOTSUP,
1841                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
1842                                           "egress is not supported");
1843         if (attributes->transfer && !priv->config.dv_esw_en)
1844                 return rte_flow_error_set(error, ENOTSUP,
1845                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1846                                           NULL, "transfer is not supported");
1847         if (!attributes->ingress)
1848                 return rte_flow_error_set(error, EINVAL,
1849                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1850                                           NULL,
1851                                           "ingress attribute is mandatory");
1852         return 0;
1853 }
1854
1855 /**
1856  * Validate ICMP6 item.
1857  *
1858  * @param[in] item
1859  *   Item specification.
1860  * @param[in] item_flags
1861  *   Bit-fields that holds the items detected until now.
1862  * @param[in] ext_vlan_sup
1863  *   Whether extended VLAN features are supported or not.
1864  * @param[out] error
1865  *   Pointer to error structure.
1866  *
1867  * @return
1868  *   0 on success, a negative errno value otherwise and rte_errno is set.
1869  */
1870 int
1871 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item,
1872                                uint64_t item_flags,
1873                                uint8_t target_protocol,
1874                                struct rte_flow_error *error)
1875 {
1876         const struct rte_flow_item_icmp6 *mask = item->mask;
1877         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1878         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1879                                       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1880         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1881                                       MLX5_FLOW_LAYER_OUTER_L4;
1882         int ret;
1883
1884         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6)
1885                 return rte_flow_error_set(error, EINVAL,
1886                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1887                                           "protocol filtering not compatible"
1888                                           " with ICMP6 layer");
1889         if (!(item_flags & l3m))
1890                 return rte_flow_error_set(error, EINVAL,
1891                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1892                                           "IPv6 is mandatory to filter on"
1893                                           " ICMP6");
1894         if (item_flags & l4m)
1895                 return rte_flow_error_set(error, EINVAL,
1896                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1897                                           "multiple L4 layers not supported");
1898         if (!mask)
1899                 mask = &rte_flow_item_icmp6_mask;
1900         ret = mlx5_flow_item_acceptable
1901                 (item, (const uint8_t *)mask,
1902                  (const uint8_t *)&rte_flow_item_icmp6_mask,
1903                  sizeof(struct rte_flow_item_icmp6),
1904                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1905         if (ret < 0)
1906                 return ret;
1907         return 0;
1908 }
1909
1910 /**
1911  * Validate ICMP item.
1912  *
1913  * @param[in] item
1914  *   Item specification.
1915  * @param[in] item_flags
1916  *   Bit-fields that holds the items detected until now.
1917  * @param[out] error
1918  *   Pointer to error structure.
1919  *
1920  * @return
1921  *   0 on success, a negative errno value otherwise and rte_errno is set.
1922  */
1923 int
1924 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item,
1925                              uint64_t item_flags,
1926                              uint8_t target_protocol,
1927                              struct rte_flow_error *error)
1928 {
1929         const struct rte_flow_item_icmp *mask = item->mask;
1930         const struct rte_flow_item_icmp nic_mask = {
1931                 .hdr.icmp_type = 0xff,
1932                 .hdr.icmp_code = 0xff,
1933                 .hdr.icmp_ident = RTE_BE16(0xffff),
1934                 .hdr.icmp_seq_nb = RTE_BE16(0xffff),
1935         };
1936         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1937         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1938                                       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1939         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1940                                       MLX5_FLOW_LAYER_OUTER_L4;
1941         int ret;
1942
1943         if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP)
1944                 return rte_flow_error_set(error, EINVAL,
1945                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1946                                           "protocol filtering not compatible"
1947                                           " with ICMP layer");
1948         if (!(item_flags & l3m))
1949                 return rte_flow_error_set(error, EINVAL,
1950                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1951                                           "IPv4 is mandatory to filter"
1952                                           " on ICMP");
1953         if (item_flags & l4m)
1954                 return rte_flow_error_set(error, EINVAL,
1955                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1956                                           "multiple L4 layers not supported");
1957         if (!mask)
1958                 mask = &nic_mask;
1959         ret = mlx5_flow_item_acceptable
1960                 (item, (const uint8_t *)mask,
1961                  (const uint8_t *)&nic_mask,
1962                  sizeof(struct rte_flow_item_icmp),
1963                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1964         if (ret < 0)
1965                 return ret;
1966         return 0;
1967 }
1968
1969 /**
1970  * Validate Ethernet item.
1971  *
1972  * @param[in] item
1973  *   Item specification.
1974  * @param[in] item_flags
1975  *   Bit-fields that holds the items detected until now.
1976  * @param[out] error
1977  *   Pointer to error structure.
1978  *
1979  * @return
1980  *   0 on success, a negative errno value otherwise and rte_errno is set.
1981  */
1982 int
1983 mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
1984                             uint64_t item_flags, bool ext_vlan_sup,
1985                             struct rte_flow_error *error)
1986 {
1987         const struct rte_flow_item_eth *mask = item->mask;
1988         const struct rte_flow_item_eth nic_mask = {
1989                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1990                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1991                 .type = RTE_BE16(0xffff),
1992                 .has_vlan = ext_vlan_sup ? 1 : 0,
1993         };
1994         int ret;
1995         int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1996         const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1997                                        MLX5_FLOW_LAYER_OUTER_L2;
1998
1999         if (item_flags & ethm)
2000                 return rte_flow_error_set(error, ENOTSUP,
2001                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2002                                           "multiple L2 layers not supported");
2003         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) ||
2004             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3)))
2005                 return rte_flow_error_set(error, EINVAL,
2006                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2007                                           "L2 layer should not follow "
2008                                           "L3 layers");
2009         if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) ||
2010             (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN)))
2011                 return rte_flow_error_set(error, EINVAL,
2012                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2013                                           "L2 layer should not follow VLAN");
2014         if (item_flags & MLX5_FLOW_LAYER_GTP)
2015                 return rte_flow_error_set(error, EINVAL,
2016                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2017                                           "L2 layer should not follow GTP");
2018         if (!mask)
2019                 mask = &rte_flow_item_eth_mask;
2020         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2021                                         (const uint8_t *)&nic_mask,
2022                                         sizeof(struct rte_flow_item_eth),
2023                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2024         return ret;
2025 }
2026
2027 /**
2028  * Validate VLAN item.
2029  *
2030  * @param[in] item
2031  *   Item specification.
2032  * @param[in] item_flags
2033  *   Bit-fields that holds the items detected until now.
2034  * @param[in] dev
2035  *   Ethernet device flow is being created on.
2036  * @param[out] error
2037  *   Pointer to error structure.
2038  *
2039  * @return
2040  *   0 on success, a negative errno value otherwise and rte_errno is set.
2041  */
2042 int
2043 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
2044                              uint64_t item_flags,
2045                              struct rte_eth_dev *dev,
2046                              struct rte_flow_error *error)
2047 {
2048         const struct rte_flow_item_vlan *spec = item->spec;
2049         const struct rte_flow_item_vlan *mask = item->mask;
2050         const struct rte_flow_item_vlan nic_mask = {
2051                 .tci = RTE_BE16(UINT16_MAX),
2052                 .inner_type = RTE_BE16(UINT16_MAX),
2053         };
2054         uint16_t vlan_tag = 0;
2055         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2056         int ret;
2057         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2058                                         MLX5_FLOW_LAYER_INNER_L4) :
2059                                        (MLX5_FLOW_LAYER_OUTER_L3 |
2060                                         MLX5_FLOW_LAYER_OUTER_L4);
2061         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2062                                         MLX5_FLOW_LAYER_OUTER_VLAN;
2063
2064         if (item_flags & vlanm)
2065                 return rte_flow_error_set(error, EINVAL,
2066                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2067                                           "multiple VLAN layers not supported");
2068         else if ((item_flags & l34m) != 0)
2069                 return rte_flow_error_set(error, EINVAL,
2070                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2071                                           "VLAN cannot follow L3/L4 layer");
2072         if (!mask)
2073                 mask = &rte_flow_item_vlan_mask;
2074         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2075                                         (const uint8_t *)&nic_mask,
2076                                         sizeof(struct rte_flow_item_vlan),
2077                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2078         if (ret)
2079                 return ret;
2080         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2081                 struct mlx5_priv *priv = dev->data->dev_private;
2082
2083                 if (priv->vmwa_context) {
2084                         /*
2085                          * Non-NULL context means we have a virtual machine
2086                          * and SR-IOV enabled, we have to create VLAN interface
2087                          * to make hypervisor to setup E-Switch vport
2088                          * context correctly. We avoid creating the multiple
2089                          * VLAN interfaces, so we cannot support VLAN tag mask.
2090                          */
2091                         return rte_flow_error_set(error, EINVAL,
2092                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2093                                                   item,
2094                                                   "VLAN tag mask is not"
2095                                                   " supported in virtual"
2096                                                   " environment");
2097                 }
2098         }
2099         if (spec) {
2100                 vlan_tag = spec->tci;
2101                 vlan_tag &= mask->tci;
2102         }
2103         /*
2104          * From verbs perspective an empty VLAN is equivalent
2105          * to a packet without VLAN layer.
2106          */
2107         if (!vlan_tag)
2108                 return rte_flow_error_set(error, EINVAL,
2109                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2110                                           item->spec,
2111                                           "VLAN cannot be empty");
2112         return 0;
2113 }
2114
2115 /**
2116  * Validate IPV4 item.
2117  *
2118  * @param[in] item
2119  *   Item specification.
2120  * @param[in] item_flags
2121  *   Bit-fields that holds the items detected until now.
2122  * @param[in] last_item
2123  *   Previous validated item in the pattern items.
2124  * @param[in] ether_type
2125  *   Type in the ethernet layer header (including dot1q).
2126  * @param[in] acc_mask
2127  *   Acceptable mask, if NULL default internal default mask
2128  *   will be used to check whether item fields are supported.
2129  * @param[in] range_accepted
2130  *   True if range of values is accepted for specific fields, false otherwise.
2131  * @param[out] error
2132  *   Pointer to error structure.
2133  *
2134  * @return
2135  *   0 on success, a negative errno value otherwise and rte_errno is set.
2136  */
2137 int
2138 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
2139                              uint64_t item_flags,
2140                              uint64_t last_item,
2141                              uint16_t ether_type,
2142                              const struct rte_flow_item_ipv4 *acc_mask,
2143                              bool range_accepted,
2144                              struct rte_flow_error *error)
2145 {
2146         const struct rte_flow_item_ipv4 *mask = item->mask;
2147         const struct rte_flow_item_ipv4 *spec = item->spec;
2148         const struct rte_flow_item_ipv4 nic_mask = {
2149                 .hdr = {
2150                         .src_addr = RTE_BE32(0xffffffff),
2151                         .dst_addr = RTE_BE32(0xffffffff),
2152                         .type_of_service = 0xff,
2153                         .next_proto_id = 0xff,
2154                 },
2155         };
2156         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2157         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2158                                       MLX5_FLOW_LAYER_OUTER_L3;
2159         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2160                                       MLX5_FLOW_LAYER_OUTER_L4;
2161         int ret;
2162         uint8_t next_proto = 0xFF;
2163         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2164                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2165                                   MLX5_FLOW_LAYER_INNER_VLAN);
2166
2167         if ((last_item & l2_vlan) && ether_type &&
2168             ether_type != RTE_ETHER_TYPE_IPV4)
2169                 return rte_flow_error_set(error, EINVAL,
2170                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2171                                           "IPv4 cannot follow L2/VLAN layer "
2172                                           "which ether type is not IPv4");
2173         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2174                 if (mask && spec)
2175                         next_proto = mask->hdr.next_proto_id &
2176                                      spec->hdr.next_proto_id;
2177                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2178                         return rte_flow_error_set(error, EINVAL,
2179                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2180                                                   item,
2181                                                   "multiple tunnel "
2182                                                   "not supported");
2183         }
2184         if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP)
2185                 return rte_flow_error_set(error, EINVAL,
2186                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2187                                           "wrong tunnel type - IPv6 specified "
2188                                           "but IPv4 item provided");
2189         if (item_flags & l3m)
2190                 return rte_flow_error_set(error, ENOTSUP,
2191                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2192                                           "multiple L3 layers not supported");
2193         else if (item_flags & l4m)
2194                 return rte_flow_error_set(error, EINVAL,
2195                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2196                                           "L3 cannot follow an L4 layer.");
2197         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2198                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2199                 return rte_flow_error_set(error, EINVAL,
2200                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2201                                           "L3 cannot follow an NVGRE layer.");
2202         if (!mask)
2203                 mask = &rte_flow_item_ipv4_mask;
2204         else if (mask->hdr.next_proto_id != 0 &&
2205                  mask->hdr.next_proto_id != 0xff)
2206                 return rte_flow_error_set(error, EINVAL,
2207                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
2208                                           "partial mask is not supported"
2209                                           " for protocol");
2210         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2211                                         acc_mask ? (const uint8_t *)acc_mask
2212                                                  : (const uint8_t *)&nic_mask,
2213                                         sizeof(struct rte_flow_item_ipv4),
2214                                         range_accepted, error);
2215         if (ret < 0)
2216                 return ret;
2217         return 0;
2218 }
2219
2220 /**
2221  * Validate IPV6 item.
2222  *
2223  * @param[in] item
2224  *   Item specification.
2225  * @param[in] item_flags
2226  *   Bit-fields that holds the items detected until now.
2227  * @param[in] last_item
2228  *   Previous validated item in the pattern items.
2229  * @param[in] ether_type
2230  *   Type in the ethernet layer header (including dot1q).
2231  * @param[in] acc_mask
2232  *   Acceptable mask, if NULL default internal default mask
2233  *   will be used to check whether item fields are supported.
2234  * @param[out] error
2235  *   Pointer to error structure.
2236  *
2237  * @return
2238  *   0 on success, a negative errno value otherwise and rte_errno is set.
2239  */
2240 int
2241 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
2242                              uint64_t item_flags,
2243                              uint64_t last_item,
2244                              uint16_t ether_type,
2245                              const struct rte_flow_item_ipv6 *acc_mask,
2246                              struct rte_flow_error *error)
2247 {
2248         const struct rte_flow_item_ipv6 *mask = item->mask;
2249         const struct rte_flow_item_ipv6 *spec = item->spec;
2250         const struct rte_flow_item_ipv6 nic_mask = {
2251                 .hdr = {
2252                         .src_addr =
2253                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2254                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2255                         .dst_addr =
2256                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
2257                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
2258                         .vtc_flow = RTE_BE32(0xffffffff),
2259                         .proto = 0xff,
2260                 },
2261         };
2262         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2263         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2264                                       MLX5_FLOW_LAYER_OUTER_L3;
2265         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2266                                       MLX5_FLOW_LAYER_OUTER_L4;
2267         int ret;
2268         uint8_t next_proto = 0xFF;
2269         const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 |
2270                                   MLX5_FLOW_LAYER_OUTER_VLAN |
2271                                   MLX5_FLOW_LAYER_INNER_VLAN);
2272
2273         if ((last_item & l2_vlan) && ether_type &&
2274             ether_type != RTE_ETHER_TYPE_IPV6)
2275                 return rte_flow_error_set(error, EINVAL,
2276                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2277                                           "IPv6 cannot follow L2/VLAN layer "
2278                                           "which ether type is not IPv6");
2279         if (mask && mask->hdr.proto == UINT8_MAX && spec)
2280                 next_proto = spec->hdr.proto;
2281         if (item_flags & MLX5_FLOW_LAYER_TUNNEL) {
2282                 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6)
2283                         return rte_flow_error_set(error, EINVAL,
2284                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2285                                                   item,
2286                                                   "multiple tunnel "
2287                                                   "not supported");
2288         }
2289         if (next_proto == IPPROTO_HOPOPTS  ||
2290             next_proto == IPPROTO_ROUTING  ||
2291             next_proto == IPPROTO_FRAGMENT ||
2292             next_proto == IPPROTO_ESP      ||
2293             next_proto == IPPROTO_AH       ||
2294             next_proto == IPPROTO_DSTOPTS)
2295                 return rte_flow_error_set(error, EINVAL,
2296                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2297                                           "IPv6 proto (next header) should "
2298                                           "not be set as extension header");
2299         if (item_flags & MLX5_FLOW_LAYER_IPIP)
2300                 return rte_flow_error_set(error, EINVAL,
2301                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2302                                           "wrong tunnel type - IPv4 specified "
2303                                           "but IPv6 item provided");
2304         if (item_flags & l3m)
2305                 return rte_flow_error_set(error, ENOTSUP,
2306                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2307                                           "multiple L3 layers not supported");
2308         else if (item_flags & l4m)
2309                 return rte_flow_error_set(error, EINVAL,
2310                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2311                                           "L3 cannot follow an L4 layer.");
2312         else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) &&
2313                   !(item_flags & MLX5_FLOW_LAYER_INNER_L2))
2314                 return rte_flow_error_set(error, EINVAL,
2315                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2316                                           "L3 cannot follow an NVGRE layer.");
2317         if (!mask)
2318                 mask = &rte_flow_item_ipv6_mask;
2319         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2320                                         acc_mask ? (const uint8_t *)acc_mask
2321                                                  : (const uint8_t *)&nic_mask,
2322                                         sizeof(struct rte_flow_item_ipv6),
2323                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2324         if (ret < 0)
2325                 return ret;
2326         return 0;
2327 }
2328
2329 /**
2330  * Validate UDP item.
2331  *
2332  * @param[in] item
2333  *   Item specification.
2334  * @param[in] item_flags
2335  *   Bit-fields that holds the items detected until now.
2336  * @param[in] target_protocol
2337  *   The next protocol in the previous item.
2338  * @param[in] flow_mask
2339  *   mlx5 flow-specific (DV, verbs, etc.) supported header fields mask.
2340  * @param[out] error
2341  *   Pointer to error structure.
2342  *
2343  * @return
2344  *   0 on success, a negative errno value otherwise and rte_errno is set.
2345  */
2346 int
2347 mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
2348                             uint64_t item_flags,
2349                             uint8_t target_protocol,
2350                             struct rte_flow_error *error)
2351 {
2352         const struct rte_flow_item_udp *mask = item->mask;
2353         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2354         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2355                                       MLX5_FLOW_LAYER_OUTER_L3;
2356         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2357                                       MLX5_FLOW_LAYER_OUTER_L4;
2358         int ret;
2359
2360         if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
2361                 return rte_flow_error_set(error, EINVAL,
2362                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2363                                           "protocol filtering not compatible"
2364                                           " with UDP layer");
2365         if (!(item_flags & l3m))
2366                 return rte_flow_error_set(error, EINVAL,
2367                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2368                                           "L3 is mandatory to filter on L4");
2369         if (item_flags & l4m)
2370                 return rte_flow_error_set(error, EINVAL,
2371                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2372                                           "multiple L4 layers not supported");
2373         if (!mask)
2374                 mask = &rte_flow_item_udp_mask;
2375         ret = mlx5_flow_item_acceptable
2376                 (item, (const uint8_t *)mask,
2377                  (const uint8_t *)&rte_flow_item_udp_mask,
2378                  sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2379                  error);
2380         if (ret < 0)
2381                 return ret;
2382         return 0;
2383 }
2384
2385 /**
2386  * Validate TCP item.
2387  *
2388  * @param[in] item
2389  *   Item specification.
2390  * @param[in] item_flags
2391  *   Bit-fields that holds the items detected until now.
2392  * @param[in] target_protocol
2393  *   The next protocol in the previous item.
2394  * @param[out] error
2395  *   Pointer to error structure.
2396  *
2397  * @return
2398  *   0 on success, a negative errno value otherwise and rte_errno is set.
2399  */
2400 int
2401 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
2402                             uint64_t item_flags,
2403                             uint8_t target_protocol,
2404                             const struct rte_flow_item_tcp *flow_mask,
2405                             struct rte_flow_error *error)
2406 {
2407         const struct rte_flow_item_tcp *mask = item->mask;
2408         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2409         const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
2410                                       MLX5_FLOW_LAYER_OUTER_L3;
2411         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2412                                       MLX5_FLOW_LAYER_OUTER_L4;
2413         int ret;
2414
2415         MLX5_ASSERT(flow_mask);
2416         if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
2417                 return rte_flow_error_set(error, EINVAL,
2418                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2419                                           "protocol filtering not compatible"
2420                                           " with TCP layer");
2421         if (!(item_flags & l3m))
2422                 return rte_flow_error_set(error, EINVAL,
2423                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2424                                           "L3 is mandatory to filter on L4");
2425         if (item_flags & l4m)
2426                 return rte_flow_error_set(error, EINVAL,
2427                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2428                                           "multiple L4 layers not supported");
2429         if (!mask)
2430                 mask = &rte_flow_item_tcp_mask;
2431         ret = mlx5_flow_item_acceptable
2432                 (item, (const uint8_t *)mask,
2433                  (const uint8_t *)flow_mask,
2434                  sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2435                  error);
2436         if (ret < 0)
2437                 return ret;
2438         return 0;
2439 }
2440
2441 /**
2442  * Validate VXLAN item.
2443  *
2444  * @param[in] dev
2445  *   Pointer to the Ethernet device structure.
2446  * @param[in] udp_dport
2447  *   UDP destination port
2448  * @param[in] item
2449  *   Item specification.
2450  * @param[in] item_flags
2451  *   Bit-fields that holds the items detected until now.
2452  * @param[in] attr
2453  *   Flow rule attributes.
2454  * @param[out] error
2455  *   Pointer to error structure.
2456  *
2457  * @return
2458  *   0 on success, a negative errno value otherwise and rte_errno is set.
2459  */
2460 int
2461 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev,
2462                               uint16_t udp_dport,
2463                               const struct rte_flow_item *item,
2464                               uint64_t item_flags,
2465                               const struct rte_flow_attr *attr,
2466                               struct rte_flow_error *error)
2467 {
2468         const struct rte_flow_item_vxlan *spec = item->spec;
2469         const struct rte_flow_item_vxlan *mask = item->mask;
2470         int ret;
2471         struct mlx5_priv *priv = dev->data->dev_private;
2472         union vni {
2473                 uint32_t vlan_id;
2474                 uint8_t vni[4];
2475         } id = { .vlan_id = 0, };
2476         const struct rte_flow_item_vxlan nic_mask = {
2477                 .vni = "\xff\xff\xff",
2478                 .rsvd1 = 0xff,
2479         };
2480         const struct rte_flow_item_vxlan *valid_mask;
2481
2482         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2483                 return rte_flow_error_set(error, ENOTSUP,
2484                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2485                                           "multiple tunnel layers not"
2486                                           " supported");
2487         valid_mask = &rte_flow_item_vxlan_mask;
2488         /*
2489          * Verify only UDPv4 is present as defined in
2490          * https://tools.ietf.org/html/rfc7348
2491          */
2492         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2493                 return rte_flow_error_set(error, EINVAL,
2494                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2495                                           "no outer UDP layer found");
2496         if (!mask)
2497                 mask = &rte_flow_item_vxlan_mask;
2498
2499         if (priv->sh->steering_format_version !=
2500             MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 ||
2501             !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) {
2502                 /* FDB domain & NIC domain non-zero group */
2503                 if ((attr->transfer || attr->group) && priv->sh->misc5_cap)
2504                         valid_mask = &nic_mask;
2505                 /* Group zero in NIC domain */
2506                 if (!attr->group && !attr->transfer &&
2507                     priv->sh->tunnel_header_0_1)
2508                         valid_mask = &nic_mask;
2509         }
2510         ret = mlx5_flow_item_acceptable
2511                 (item, (const uint8_t *)mask,
2512                  (const uint8_t *)valid_mask,
2513                  sizeof(struct rte_flow_item_vxlan),
2514                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2515         if (ret < 0)
2516                 return ret;
2517         if (spec) {
2518                 memcpy(&id.vni[1], spec->vni, 3);
2519                 memcpy(&id.vni[1], mask->vni, 3);
2520         }
2521         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2522                 return rte_flow_error_set(error, ENOTSUP,
2523                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2524                                           "VXLAN tunnel must be fully defined");
2525         return 0;
2526 }
2527
2528 /**
2529  * Validate VXLAN_GPE item.
2530  *
2531  * @param[in] item
2532  *   Item specification.
2533  * @param[in] item_flags
2534  *   Bit-fields that holds the items detected until now.
2535  * @param[in] priv
2536  *   Pointer to the private data structure.
2537  * @param[in] target_protocol
2538  *   The next protocol in the previous item.
2539  * @param[out] error
2540  *   Pointer to error structure.
2541  *
2542  * @return
2543  *   0 on success, a negative errno value otherwise and rte_errno is set.
2544  */
2545 int
2546 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
2547                                   uint64_t item_flags,
2548                                   struct rte_eth_dev *dev,
2549                                   struct rte_flow_error *error)
2550 {
2551         struct mlx5_priv *priv = dev->data->dev_private;
2552         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
2553         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
2554         int ret;
2555         union vni {
2556                 uint32_t vlan_id;
2557                 uint8_t vni[4];
2558         } id = { .vlan_id = 0, };
2559
2560         if (!priv->config.l3_vxlan_en)
2561                 return rte_flow_error_set(error, ENOTSUP,
2562                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2563                                           "L3 VXLAN is not enabled by device"
2564                                           " parameter and/or not configured in"
2565                                           " firmware");
2566         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2567                 return rte_flow_error_set(error, ENOTSUP,
2568                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2569                                           "multiple tunnel layers not"
2570                                           " supported");
2571         /*
2572          * Verify only UDPv4 is present as defined in
2573          * https://tools.ietf.org/html/rfc7348
2574          */
2575         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2576                 return rte_flow_error_set(error, EINVAL,
2577                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2578                                           "no outer UDP layer found");
2579         if (!mask)
2580                 mask = &rte_flow_item_vxlan_gpe_mask;
2581         ret = mlx5_flow_item_acceptable
2582                 (item, (const uint8_t *)mask,
2583                  (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
2584                  sizeof(struct rte_flow_item_vxlan_gpe),
2585                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2586         if (ret < 0)
2587                 return ret;
2588         if (spec) {
2589                 if (spec->protocol)
2590                         return rte_flow_error_set(error, ENOTSUP,
2591                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2592                                                   item,
2593                                                   "VxLAN-GPE protocol"
2594                                                   " not supported");
2595                 memcpy(&id.vni[1], spec->vni, 3);
2596                 memcpy(&id.vni[1], mask->vni, 3);
2597         }
2598         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2599                 return rte_flow_error_set(error, ENOTSUP,
2600                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2601                                           "VXLAN-GPE tunnel must be fully"
2602                                           " defined");
2603         return 0;
2604 }
2605 /**
2606  * Validate GRE Key item.
2607  *
2608  * @param[in] item
2609  *   Item specification.
2610  * @param[in] item_flags
2611  *   Bit flags to mark detected items.
2612  * @param[in] gre_item
2613  *   Pointer to gre_item
2614  * @param[out] error
2615  *   Pointer to error structure.
2616  *
2617  * @return
2618  *   0 on success, a negative errno value otherwise and rte_errno is set.
2619  */
2620 int
2621 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item,
2622                                 uint64_t item_flags,
2623                                 const struct rte_flow_item *gre_item,
2624                                 struct rte_flow_error *error)
2625 {
2626         const rte_be32_t *mask = item->mask;
2627         int ret = 0;
2628         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
2629         const struct rte_flow_item_gre *gre_spec;
2630         const struct rte_flow_item_gre *gre_mask;
2631
2632         if (item_flags & MLX5_FLOW_LAYER_GRE_KEY)
2633                 return rte_flow_error_set(error, ENOTSUP,
2634                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2635                                           "Multiple GRE key not support");
2636         if (!(item_flags & MLX5_FLOW_LAYER_GRE))
2637                 return rte_flow_error_set(error, ENOTSUP,
2638                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2639                                           "No preceding GRE header");
2640         if (item_flags & MLX5_FLOW_LAYER_INNER)
2641                 return rte_flow_error_set(error, ENOTSUP,
2642                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2643                                           "GRE key following a wrong item");
2644         gre_mask = gre_item->mask;
2645         if (!gre_mask)
2646                 gre_mask = &rte_flow_item_gre_mask;
2647         gre_spec = gre_item->spec;
2648         if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) &&
2649                          !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000)))
2650                 return rte_flow_error_set(error, EINVAL,
2651                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2652                                           "Key bit must be on");
2653
2654         if (!mask)
2655                 mask = &gre_key_default_mask;
2656         ret = mlx5_flow_item_acceptable
2657                 (item, (const uint8_t *)mask,
2658                  (const uint8_t *)&gre_key_default_mask,
2659                  sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2660         return ret;
2661 }
2662
2663 /**
2664  * Validate GRE item.
2665  *
2666  * @param[in] item
2667  *   Item specification.
2668  * @param[in] item_flags
2669  *   Bit flags to mark detected items.
2670  * @param[in] target_protocol
2671  *   The next protocol in the previous item.
2672  * @param[out] error
2673  *   Pointer to error structure.
2674  *
2675  * @return
2676  *   0 on success, a negative errno value otherwise and rte_errno is set.
2677  */
2678 int
2679 mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
2680                             uint64_t item_flags,
2681                             uint8_t target_protocol,
2682                             struct rte_flow_error *error)
2683 {
2684         const struct rte_flow_item_gre *spec __rte_unused = item->spec;
2685         const struct rte_flow_item_gre *mask = item->mask;
2686         int ret;
2687         const struct rte_flow_item_gre nic_mask = {
2688                 .c_rsvd0_ver = RTE_BE16(0xB000),
2689                 .protocol = RTE_BE16(UINT16_MAX),
2690         };
2691
2692         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
2693                 return rte_flow_error_set(error, EINVAL,
2694                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2695                                           "protocol filtering not compatible"
2696                                           " with this GRE layer");
2697         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2698                 return rte_flow_error_set(error, ENOTSUP,
2699                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2700                                           "multiple tunnel layers not"
2701                                           " supported");
2702         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
2703                 return rte_flow_error_set(error, ENOTSUP,
2704                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2705                                           "L3 Layer is missing");
2706         if (!mask)
2707                 mask = &rte_flow_item_gre_mask;
2708         ret = mlx5_flow_item_acceptable
2709                 (item, (const uint8_t *)mask,
2710                  (const uint8_t *)&nic_mask,
2711                  sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED,
2712                  error);
2713         if (ret < 0)
2714                 return ret;
2715 #ifndef HAVE_MLX5DV_DR
2716 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
2717         if (spec && (spec->protocol & mask->protocol))
2718                 return rte_flow_error_set(error, ENOTSUP,
2719                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2720                                           "without MPLS support the"
2721                                           " specification cannot be used for"
2722                                           " filtering");
2723 #endif
2724 #endif
2725         return 0;
2726 }
2727
2728 /**
2729  * Validate Geneve item.
2730  *
2731  * @param[in] item
2732  *   Item specification.
2733  * @param[in] itemFlags
2734  *   Bit-fields that holds the items detected until now.
2735  * @param[in] enPriv
2736  *   Pointer to the private data structure.
2737  * @param[out] error
2738  *   Pointer to error structure.
2739  *
2740  * @return
2741  *   0 on success, a negative errno value otherwise and rte_errno is set.
2742  */
2743
2744 int
2745 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item,
2746                                uint64_t item_flags,
2747                                struct rte_eth_dev *dev,
2748                                struct rte_flow_error *error)
2749 {
2750         struct mlx5_priv *priv = dev->data->dev_private;
2751         const struct rte_flow_item_geneve *spec = item->spec;
2752         const struct rte_flow_item_geneve *mask = item->mask;
2753         int ret;
2754         uint16_t gbhdr;
2755         uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ?
2756                           MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0;
2757         const struct rte_flow_item_geneve nic_mask = {
2758                 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80),
2759                 .vni = "\xff\xff\xff",
2760                 .protocol = RTE_BE16(UINT16_MAX),
2761         };
2762
2763         if (!priv->config.hca_attr.tunnel_stateless_geneve_rx)
2764                 return rte_flow_error_set(error, ENOTSUP,
2765                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2766                                           "L3 Geneve is not enabled by device"
2767                                           " parameter and/or not configured in"
2768                                           " firmware");
2769         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2770                 return rte_flow_error_set(error, ENOTSUP,
2771                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2772                                           "multiple tunnel layers not"
2773                                           " supported");
2774         /*
2775          * Verify only UDPv4 is present as defined in
2776          * https://tools.ietf.org/html/rfc7348
2777          */
2778         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2779                 return rte_flow_error_set(error, EINVAL,
2780                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2781                                           "no outer UDP layer found");
2782         if (!mask)
2783                 mask = &rte_flow_item_geneve_mask;
2784         ret = mlx5_flow_item_acceptable
2785                                   (item, (const uint8_t *)mask,
2786                                    (const uint8_t *)&nic_mask,
2787                                    sizeof(struct rte_flow_item_geneve),
2788                                    MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2789         if (ret)
2790                 return ret;
2791         if (spec) {
2792                 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0);
2793                 if (MLX5_GENEVE_VER_VAL(gbhdr) ||
2794                      MLX5_GENEVE_CRITO_VAL(gbhdr) ||
2795                      MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1)
2796                         return rte_flow_error_set(error, ENOTSUP,
2797                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2798                                                   item,
2799                                                   "Geneve protocol unsupported"
2800                                                   " fields are being used");
2801                 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len)
2802                         return rte_flow_error_set
2803                                         (error, ENOTSUP,
2804                                          RTE_FLOW_ERROR_TYPE_ITEM,
2805                                          item,
2806                                          "Unsupported Geneve options length");
2807         }
2808         if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
2809                 return rte_flow_error_set
2810                                     (error, ENOTSUP,
2811                                      RTE_FLOW_ERROR_TYPE_ITEM, item,
2812                                      "Geneve tunnel must be fully defined");
2813         return 0;
2814 }
2815
2816 /**
2817  * Validate Geneve TLV option item.
2818  *
2819  * @param[in] item
2820  *   Item specification.
2821  * @param[in] last_item
2822  *   Previous validated item in the pattern items.
2823  * @param[in] geneve_item
2824  *   Previous GENEVE item specification.
2825  * @param[in] dev
2826  *   Pointer to the rte_eth_dev structure.
2827  * @param[out] error
2828  *   Pointer to error structure.
2829  *
2830  * @return
2831  *   0 on success, a negative errno value otherwise and rte_errno is set.
2832  */
2833 int
2834 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item,
2835                                    uint64_t last_item,
2836                                    const struct rte_flow_item *geneve_item,
2837                                    struct rte_eth_dev *dev,
2838                                    struct rte_flow_error *error)
2839 {
2840         struct mlx5_priv *priv = dev->data->dev_private;
2841         struct mlx5_dev_ctx_shared *sh = priv->sh;
2842         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource;
2843         struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr;
2844         uint8_t data_max_supported =
2845                         hca_attr->max_geneve_tlv_option_data_len * 4;
2846         struct mlx5_dev_config *config = &priv->config;
2847         const struct rte_flow_item_geneve *geneve_spec;
2848         const struct rte_flow_item_geneve *geneve_mask;
2849         const struct rte_flow_item_geneve_opt *spec = item->spec;
2850         const struct rte_flow_item_geneve_opt *mask = item->mask;
2851         unsigned int i;
2852         unsigned int data_len;
2853         uint8_t tlv_option_len;
2854         uint16_t optlen_m, optlen_v;
2855         const struct rte_flow_item_geneve_opt full_mask = {
2856                 .option_class = RTE_BE16(0xffff),
2857                 .option_type = 0xff,
2858                 .option_len = 0x1f,
2859         };
2860
2861         if (!mask)
2862                 mask = &rte_flow_item_geneve_opt_mask;
2863         if (!spec)
2864                 return rte_flow_error_set
2865                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2866                         "Geneve TLV opt class/type/length must be specified");
2867         if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK)
2868                 return rte_flow_error_set
2869                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2870                         "Geneve TLV opt length exceeeds the limit (31)");
2871         /* Check if class type and length masks are full. */
2872         if (full_mask.option_class != mask->option_class ||
2873             full_mask.option_type != mask->option_type ||
2874             full_mask.option_len != (mask->option_len & full_mask.option_len))
2875                 return rte_flow_error_set
2876                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2877                         "Geneve TLV opt class/type/length masks must be full");
2878         /* Check if length is supported */
2879         if ((uint32_t)spec->option_len >
2880                         config->hca_attr.max_geneve_tlv_option_data_len)
2881                 return rte_flow_error_set
2882                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2883                         "Geneve TLV opt length not supported");
2884         if (config->hca_attr.max_geneve_tlv_options > 1)
2885                 DRV_LOG(DEBUG,
2886                         "max_geneve_tlv_options supports more than 1 option");
2887         /* Check GENEVE item preceding. */
2888         if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE))
2889                 return rte_flow_error_set
2890                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2891                         "Geneve opt item must be preceded with Geneve item");
2892         geneve_spec = geneve_item->spec;
2893         geneve_mask = geneve_item->mask ? geneve_item->mask :
2894                                           &rte_flow_item_geneve_mask;
2895         /* Check if GENEVE TLV option size doesn't exceed option length */
2896         if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 ||
2897                             geneve_spec->ver_opt_len_o_c_rsvd0)) {
2898                 tlv_option_len = spec->option_len & mask->option_len;
2899                 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0);
2900                 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v);
2901                 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0);
2902                 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m);
2903                 if ((optlen_v & optlen_m) <= tlv_option_len)
2904                         return rte_flow_error_set
2905                                 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2906                                  "GENEVE TLV option length exceeds optlen");
2907         }
2908         /* Check if length is 0 or data is 0. */
2909         if (spec->data == NULL || spec->option_len == 0)
2910                 return rte_flow_error_set
2911                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2912                         "Geneve TLV opt with zero data/length not supported");
2913         /* Check not all data & mask are 0. */
2914         data_len = spec->option_len * 4;
2915         if (mask->data == NULL) {
2916                 for (i = 0; i < data_len; i++)
2917                         if (spec->data[i])
2918                                 break;
2919                 if (i == data_len)
2920                         return rte_flow_error_set(error, ENOTSUP,
2921                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2922                                 "Can't match on Geneve option data 0");
2923         } else {
2924                 for (i = 0; i < data_len; i++)
2925                         if (spec->data[i] & mask->data[i])
2926                                 break;
2927                 if (i == data_len)
2928                         return rte_flow_error_set(error, ENOTSUP,
2929                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2930                                 "Can't match on Geneve option data and mask 0");
2931                 /* Check data mask supported. */
2932                 for (i = data_max_supported; i < data_len ; i++)
2933                         if (mask->data[i])
2934                                 return rte_flow_error_set(error, ENOTSUP,
2935                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2936                                         "Data mask is of unsupported size");
2937         }
2938         /* Check GENEVE option is supported in NIC. */
2939         if (!config->hca_attr.geneve_tlv_opt)
2940                 return rte_flow_error_set
2941                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2942                         "Geneve TLV opt not supported");
2943         /* Check if we already have geneve option with different type/class. */
2944         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
2945         geneve_opt_resource = sh->geneve_tlv_option_resource;
2946         if (geneve_opt_resource != NULL)
2947                 if (geneve_opt_resource->option_class != spec->option_class ||
2948                     geneve_opt_resource->option_type != spec->option_type ||
2949                     geneve_opt_resource->length != spec->option_len) {
2950                         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
2951                         return rte_flow_error_set(error, ENOTSUP,
2952                                 RTE_FLOW_ERROR_TYPE_ITEM, item,
2953                                 "Only one Geneve TLV option supported");
2954                 }
2955         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
2956         return 0;
2957 }
2958
2959 /**
2960  * Validate MPLS item.
2961  *
2962  * @param[in] dev
2963  *   Pointer to the rte_eth_dev structure.
2964  * @param[in] item
2965  *   Item specification.
2966  * @param[in] item_flags
2967  *   Bit-fields that holds the items detected until now.
2968  * @param[in] prev_layer
2969  *   The protocol layer indicated in previous item.
2970  * @param[out] error
2971  *   Pointer to error structure.
2972  *
2973  * @return
2974  *   0 on success, a negative errno value otherwise and rte_errno is set.
2975  */
2976 int
2977 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused,
2978                              const struct rte_flow_item *item __rte_unused,
2979                              uint64_t item_flags __rte_unused,
2980                              uint64_t prev_layer __rte_unused,
2981                              struct rte_flow_error *error)
2982 {
2983 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
2984         const struct rte_flow_item_mpls *mask = item->mask;
2985         struct mlx5_priv *priv = dev->data->dev_private;
2986         int ret;
2987
2988         if (!priv->config.mpls_en)
2989                 return rte_flow_error_set(error, ENOTSUP,
2990                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2991                                           "MPLS not supported or"
2992                                           " disabled in firmware"
2993                                           " configuration.");
2994         /* MPLS over UDP, GRE is allowed */
2995         if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP |
2996                             MLX5_FLOW_LAYER_GRE |
2997                             MLX5_FLOW_LAYER_GRE_KEY)))
2998                 return rte_flow_error_set(error, EINVAL,
2999                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3000                                           "protocol filtering not compatible"
3001                                           " with MPLS layer");
3002         /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
3003         if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) &&
3004             !(item_flags & MLX5_FLOW_LAYER_GRE))
3005                 return rte_flow_error_set(error, ENOTSUP,
3006                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3007                                           "multiple tunnel layers not"
3008                                           " supported");
3009         if (!mask)
3010                 mask = &rte_flow_item_mpls_mask;
3011         ret = mlx5_flow_item_acceptable
3012                 (item, (const uint8_t *)mask,
3013                  (const uint8_t *)&rte_flow_item_mpls_mask,
3014                  sizeof(struct rte_flow_item_mpls),
3015                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3016         if (ret < 0)
3017                 return ret;
3018         return 0;
3019 #else
3020         return rte_flow_error_set(error, ENOTSUP,
3021                                   RTE_FLOW_ERROR_TYPE_ITEM, item,
3022                                   "MPLS is not supported by Verbs, please"
3023                                   " update.");
3024 #endif
3025 }
3026
3027 /**
3028  * Validate NVGRE item.
3029  *
3030  * @param[in] item
3031  *   Item specification.
3032  * @param[in] item_flags
3033  *   Bit flags to mark detected items.
3034  * @param[in] target_protocol
3035  *   The next protocol in the previous item.
3036  * @param[out] error
3037  *   Pointer to error structure.
3038  *
3039  * @return
3040  *   0 on success, a negative errno value otherwise and rte_errno is set.
3041  */
3042 int
3043 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item,
3044                               uint64_t item_flags,
3045                               uint8_t target_protocol,
3046                               struct rte_flow_error *error)
3047 {
3048         const struct rte_flow_item_nvgre *mask = item->mask;
3049         int ret;
3050
3051         if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
3052                 return rte_flow_error_set(error, EINVAL,
3053                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3054                                           "protocol filtering not compatible"
3055                                           " with this GRE layer");
3056         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3057                 return rte_flow_error_set(error, ENOTSUP,
3058                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3059                                           "multiple tunnel layers not"
3060                                           " supported");
3061         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
3062                 return rte_flow_error_set(error, ENOTSUP,
3063                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3064                                           "L3 Layer is missing");
3065         if (!mask)
3066                 mask = &rte_flow_item_nvgre_mask;
3067         ret = mlx5_flow_item_acceptable
3068                 (item, (const uint8_t *)mask,
3069                  (const uint8_t *)&rte_flow_item_nvgre_mask,
3070                  sizeof(struct rte_flow_item_nvgre),
3071                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3072         if (ret < 0)
3073                 return ret;
3074         return 0;
3075 }
3076
3077 /**
3078  * Validate eCPRI item.
3079  *
3080  * @param[in] item
3081  *   Item specification.
3082  * @param[in] item_flags
3083  *   Bit-fields that holds the items detected until now.
3084  * @param[in] last_item
3085  *   Previous validated item in the pattern items.
3086  * @param[in] ether_type
3087  *   Type in the ethernet layer header (including dot1q).
3088  * @param[in] acc_mask
3089  *   Acceptable mask, if NULL default internal default mask
3090  *   will be used to check whether item fields are supported.
3091  * @param[out] error
3092  *   Pointer to error structure.
3093  *
3094  * @return
3095  *   0 on success, a negative errno value otherwise and rte_errno is set.
3096  */
3097 int
3098 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
3099                               uint64_t item_flags,
3100                               uint64_t last_item,
3101                               uint16_t ether_type,
3102                               const struct rte_flow_item_ecpri *acc_mask,
3103                               struct rte_flow_error *error)
3104 {
3105         const struct rte_flow_item_ecpri *mask = item->mask;
3106         const struct rte_flow_item_ecpri nic_mask = {
3107                 .hdr = {
3108                         .common = {
3109                                 .u32 =
3110                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
3111                                         .type = 0xFF,
3112                                         }).u32),
3113                         },
3114                         .dummy[0] = 0xFFFFFFFF,
3115                 },
3116         };
3117         const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 |
3118                                         MLX5_FLOW_LAYER_OUTER_VLAN);
3119         struct rte_flow_item_ecpri mask_lo;
3120
3121         if (!(last_item & outer_l2_vlan) &&
3122             last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP)
3123                 return rte_flow_error_set(error, EINVAL,
3124                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3125                                           "eCPRI can only follow L2/VLAN layer or UDP layer");
3126         if ((last_item & outer_l2_vlan) && ether_type &&
3127             ether_type != RTE_ETHER_TYPE_ECPRI)
3128                 return rte_flow_error_set(error, EINVAL,
3129                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3130                                           "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE");
3131         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
3132                 return rte_flow_error_set(error, EINVAL,
3133                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3134                                           "eCPRI with tunnel is not supported right now");
3135         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3)
3136                 return rte_flow_error_set(error, ENOTSUP,
3137                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3138                                           "multiple L3 layers not supported");
3139         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
3140                 return rte_flow_error_set(error, EINVAL,
3141                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3142                                           "eCPRI cannot coexist with a TCP layer");
3143         /* In specification, eCPRI could be over UDP layer. */
3144         else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
3145                 return rte_flow_error_set(error, EINVAL,
3146                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
3147                                           "eCPRI over UDP layer is not yet supported right now");
3148         /* Mask for type field in common header could be zero. */
3149         if (!mask)
3150                 mask = &rte_flow_item_ecpri_mask;
3151         mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32);
3152         /* Input mask is in big-endian format. */
3153         if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff)
3154                 return rte_flow_error_set(error, EINVAL,
3155                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3156                                           "partial mask is not supported for protocol");
3157         else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0)
3158                 return rte_flow_error_set(error, EINVAL,
3159                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
3160                                           "message header mask must be after a type mask");
3161         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
3162                                          acc_mask ? (const uint8_t *)acc_mask
3163                                                   : (const uint8_t *)&nic_mask,
3164                                          sizeof(struct rte_flow_item_ecpri),
3165                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
3166 }
3167
3168 static int
3169 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
3170                    const struct rte_flow_attr *attr __rte_unused,
3171                    const struct rte_flow_item items[] __rte_unused,
3172                    const struct rte_flow_action actions[] __rte_unused,
3173                    bool external __rte_unused,
3174                    int hairpin __rte_unused,
3175                    struct rte_flow_error *error)
3176 {
3177         return rte_flow_error_set(error, ENOTSUP,
3178                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3179 }
3180
3181 static struct mlx5_flow *
3182 flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
3183                   const struct rte_flow_attr *attr __rte_unused,
3184                   const struct rte_flow_item items[] __rte_unused,
3185                   const struct rte_flow_action actions[] __rte_unused,
3186                   struct rte_flow_error *error)
3187 {
3188         rte_flow_error_set(error, ENOTSUP,
3189                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3190         return NULL;
3191 }
3192
3193 static int
3194 flow_null_translate(struct rte_eth_dev *dev __rte_unused,
3195                     struct mlx5_flow *dev_flow __rte_unused,
3196                     const struct rte_flow_attr *attr __rte_unused,
3197                     const struct rte_flow_item items[] __rte_unused,
3198                     const struct rte_flow_action actions[] __rte_unused,
3199                     struct rte_flow_error *error)
3200 {
3201         return rte_flow_error_set(error, ENOTSUP,
3202                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3203 }
3204
3205 static int
3206 flow_null_apply(struct rte_eth_dev *dev __rte_unused,
3207                 struct rte_flow *flow __rte_unused,
3208                 struct rte_flow_error *error)
3209 {
3210         return rte_flow_error_set(error, ENOTSUP,
3211                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3212 }
3213
3214 static void
3215 flow_null_remove(struct rte_eth_dev *dev __rte_unused,
3216                  struct rte_flow *flow __rte_unused)
3217 {
3218 }
3219
3220 static void
3221 flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
3222                   struct rte_flow *flow __rte_unused)
3223 {
3224 }
3225
3226 static int
3227 flow_null_query(struct rte_eth_dev *dev __rte_unused,
3228                 struct rte_flow *flow __rte_unused,
3229                 const struct rte_flow_action *actions __rte_unused,
3230                 void *data __rte_unused,
3231                 struct rte_flow_error *error)
3232 {
3233         return rte_flow_error_set(error, ENOTSUP,
3234                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL);
3235 }
3236
3237 static int
3238 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused,
3239                       uint32_t domains __rte_unused,
3240                       uint32_t flags __rte_unused)
3241 {
3242         return 0;
3243 }
3244
3245 /* Void driver to protect from null pointer reference. */
3246 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
3247         .validate = flow_null_validate,
3248         .prepare = flow_null_prepare,
3249         .translate = flow_null_translate,
3250         .apply = flow_null_apply,
3251         .remove = flow_null_remove,
3252         .destroy = flow_null_destroy,
3253         .query = flow_null_query,
3254         .sync_domain = flow_null_sync_domain,
3255 };
3256
3257 /**
3258  * Select flow driver type according to flow attributes and device
3259  * configuration.
3260  *
3261  * @param[in] dev
3262  *   Pointer to the dev structure.
3263  * @param[in] attr
3264  *   Pointer to the flow attributes.
3265  *
3266  * @return
3267  *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
3268  */
3269 static enum mlx5_flow_drv_type
3270 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
3271 {
3272         struct mlx5_priv *priv = dev->data->dev_private;
3273         /* The OS can determine first a specific flow type (DV, VERBS) */
3274         enum mlx5_flow_drv_type type = mlx5_flow_os_get_type();
3275
3276         if (type != MLX5_FLOW_TYPE_MAX)
3277                 return type;
3278         /* If no OS specific type - continue with DV/VERBS selection */
3279         if (attr->transfer && priv->config.dv_esw_en)
3280                 type = MLX5_FLOW_TYPE_DV;
3281         if (!attr->transfer)
3282                 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
3283                                                  MLX5_FLOW_TYPE_VERBS;
3284         return type;
3285 }
3286
3287 #define flow_get_drv_ops(type) flow_drv_ops[type]
3288
3289 /**
3290  * Flow driver validation API. This abstracts calling driver specific functions.
3291  * The type of flow driver is determined according to flow attributes.
3292  *
3293  * @param[in] dev
3294  *   Pointer to the dev structure.
3295  * @param[in] attr
3296  *   Pointer to the flow attributes.
3297  * @param[in] items
3298  *   Pointer to the list of items.
3299  * @param[in] actions
3300  *   Pointer to the list of actions.
3301  * @param[in] external
3302  *   This flow rule is created by request external to PMD.
3303  * @param[in] hairpin
3304  *   Number of hairpin TX actions, 0 means classic flow.
3305  * @param[out] error
3306  *   Pointer to the error structure.
3307  *
3308  * @return
3309  *   0 on success, a negative errno value otherwise and rte_errno is set.
3310  */
3311 static inline int
3312 flow_drv_validate(struct rte_eth_dev *dev,
3313                   const struct rte_flow_attr *attr,
3314                   const struct rte_flow_item items[],
3315                   const struct rte_flow_action actions[],
3316                   bool external, int hairpin, struct rte_flow_error *error)
3317 {
3318         const struct mlx5_flow_driver_ops *fops;
3319         enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
3320
3321         fops = flow_get_drv_ops(type);
3322         return fops->validate(dev, attr, items, actions, external,
3323                               hairpin, error);
3324 }
3325
3326 /**
3327  * Flow driver preparation API. This abstracts calling driver specific
3328  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3329  * calculates the size of memory required for device flow, allocates the memory,
3330  * initializes the device flow and returns the pointer.
3331  *
3332  * @note
3333  *   This function initializes device flow structure such as dv or verbs in
3334  *   struct mlx5_flow. However, it is caller's responsibility to initialize the
3335  *   rest. For example, adding returning device flow to flow->dev_flow list and
3336  *   setting backward reference to the flow should be done out of this function.
3337  *   layers field is not filled either.
3338  *
3339  * @param[in] dev
3340  *   Pointer to the dev structure.
3341  * @param[in] attr
3342  *   Pointer to the flow attributes.
3343  * @param[in] items
3344  *   Pointer to the list of items.
3345  * @param[in] actions
3346  *   Pointer to the list of actions.
3347  * @param[in] flow_idx
3348  *   This memory pool index to the flow.
3349  * @param[out] error
3350  *   Pointer to the error structure.
3351  *
3352  * @return
3353  *   Pointer to device flow on success, otherwise NULL and rte_errno is set.
3354  */
3355 static inline struct mlx5_flow *
3356 flow_drv_prepare(struct rte_eth_dev *dev,
3357                  const struct rte_flow *flow,
3358                  const struct rte_flow_attr *attr,
3359                  const struct rte_flow_item items[],
3360                  const struct rte_flow_action actions[],
3361                  uint32_t flow_idx,
3362                  struct rte_flow_error *error)
3363 {
3364         const struct mlx5_flow_driver_ops *fops;
3365         enum mlx5_flow_drv_type type = flow->drv_type;
3366         struct mlx5_flow *mlx5_flow = NULL;
3367
3368         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3369         fops = flow_get_drv_ops(type);
3370         mlx5_flow = fops->prepare(dev, attr, items, actions, error);
3371         if (mlx5_flow)
3372                 mlx5_flow->flow_idx = flow_idx;
3373         return mlx5_flow;
3374 }
3375
3376 /**
3377  * Flow driver translation API. This abstracts calling driver specific
3378  * functions. Parent flow (rte_flow) should have driver type (drv_type). It
3379  * translates a generic flow into a driver flow. flow_drv_prepare() must
3380  * precede.
3381  *
3382  * @note
3383  *   dev_flow->layers could be filled as a result of parsing during translation
3384  *   if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled
3385  *   if necessary. As a flow can have multiple dev_flows by RSS flow expansion,
3386  *   flow->actions could be overwritten even though all the expanded dev_flows
3387  *   have the same actions.
3388  *
3389  * @param[in] dev
3390  *   Pointer to the rte dev structure.
3391  * @param[in, out] dev_flow
3392  *   Pointer to the mlx5 flow.
3393  * @param[in] attr
3394  *   Pointer to the flow attributes.
3395  * @param[in] items
3396  *   Pointer to the list of items.
3397  * @param[in] actions
3398  *   Pointer to the list of actions.
3399  * @param[out] error
3400  *   Pointer to the error structure.
3401  *
3402  * @return
3403  *   0 on success, a negative errno value otherwise and rte_errno is set.
3404  */
3405 static inline int
3406 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
3407                    const struct rte_flow_attr *attr,
3408                    const struct rte_flow_item items[],
3409                    const struct rte_flow_action actions[],
3410                    struct rte_flow_error *error)
3411 {
3412         const struct mlx5_flow_driver_ops *fops;
3413         enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
3414
3415         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3416         fops = flow_get_drv_ops(type);
3417         return fops->translate(dev, dev_flow, attr, items, actions, error);
3418 }
3419
3420 /**
3421  * Flow driver apply API. This abstracts calling driver specific functions.
3422  * Parent flow (rte_flow) should have driver type (drv_type). It applies
3423  * translated driver flows on to device. flow_drv_translate() must precede.
3424  *
3425  * @param[in] dev
3426  *   Pointer to Ethernet device structure.
3427  * @param[in, out] flow
3428  *   Pointer to flow structure.
3429  * @param[out] error
3430  *   Pointer to error structure.
3431  *
3432  * @return
3433  *   0 on success, a negative errno value otherwise and rte_errno is set.
3434  */
3435 static inline int
3436 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
3437                struct rte_flow_error *error)
3438 {
3439         const struct mlx5_flow_driver_ops *fops;
3440         enum mlx5_flow_drv_type type = flow->drv_type;
3441
3442         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3443         fops = flow_get_drv_ops(type);
3444         return fops->apply(dev, flow, error);
3445 }
3446
3447 /**
3448  * Flow driver destroy API. This abstracts calling driver specific functions.
3449  * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
3450  * on device and releases resources of the flow.
3451  *
3452  * @param[in] dev
3453  *   Pointer to Ethernet device.
3454  * @param[in, out] flow
3455  *   Pointer to flow structure.
3456  */
3457 static inline void
3458 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
3459 {
3460         const struct mlx5_flow_driver_ops *fops;
3461         enum mlx5_flow_drv_type type = flow->drv_type;
3462
3463         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3464         fops = flow_get_drv_ops(type);
3465         fops->destroy(dev, flow);
3466 }
3467
3468 /**
3469  * Flow driver find RSS policy tbl API. This abstracts calling driver
3470  * specific functions. Parent flow (rte_flow) should have driver
3471  * type (drv_type). It will find the RSS policy table that has the rss_desc.
3472  *
3473  * @param[in] dev
3474  *   Pointer to Ethernet device.
3475  * @param[in, out] flow
3476  *   Pointer to flow structure.
3477  * @param[in] policy
3478  *   Pointer to meter policy table.
3479  * @param[in] rss_desc
3480  *   Pointer to rss_desc
3481  */
3482 static struct mlx5_flow_meter_sub_policy *
3483 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
3484                 struct rte_flow *flow,
3485                 struct mlx5_flow_meter_policy *policy,
3486                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
3487 {
3488         const struct mlx5_flow_driver_ops *fops;
3489         enum mlx5_flow_drv_type type = flow->drv_type;
3490
3491         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3492         fops = flow_get_drv_ops(type);
3493         return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc);
3494 }
3495
3496 /**
3497  * Flow driver color tag rule API. This abstracts calling driver
3498  * specific functions. Parent flow (rte_flow) should have driver
3499  * type (drv_type). It will create the color tag rules in hierarchy meter.
3500  *
3501  * @param[in] dev
3502  *   Pointer to Ethernet device.
3503  * @param[in, out] flow
3504  *   Pointer to flow structure.
3505  * @param[in] fm
3506  *   Pointer to flow meter structure.
3507  * @param[in] src_port
3508  *   The src port this extra rule should use.
3509  * @param[in] item
3510  *   The src port id match item.
3511  * @param[out] error
3512  *   Pointer to error structure.
3513  */
3514 static int
3515 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev,
3516                 struct rte_flow *flow,
3517                 struct mlx5_flow_meter_info *fm,
3518                 int32_t src_port,
3519                 const struct rte_flow_item *item,
3520                 struct rte_flow_error *error)
3521 {
3522         const struct mlx5_flow_driver_ops *fops;
3523         enum mlx5_flow_drv_type type = flow->drv_type;
3524
3525         MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
3526         fops = flow_get_drv_ops(type);
3527         return fops->meter_hierarchy_rule_create(dev, fm,
3528                                                 src_port, item, error);
3529 }
3530
3531 /**
3532  * Get RSS action from the action list.
3533  *
3534  * @param[in] dev
3535  *   Pointer to Ethernet device.
3536  * @param[in] actions
3537  *   Pointer to the list of actions.
3538  * @param[in] flow
3539  *   Parent flow structure pointer.
3540  *
3541  * @return
3542  *   Pointer to the RSS action if exist, else return NULL.
3543  */
3544 static const struct rte_flow_action_rss*
3545 flow_get_rss_action(struct rte_eth_dev *dev,
3546                     const struct rte_flow_action actions[])
3547 {
3548         struct mlx5_priv *priv = dev->data->dev_private;
3549         const struct rte_flow_action_rss *rss = NULL;
3550
3551         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3552                 switch (actions->type) {
3553                 case RTE_FLOW_ACTION_TYPE_RSS:
3554                         rss = actions->conf;
3555                         break;
3556                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
3557                 {
3558                         const struct rte_flow_action_sample *sample =
3559                                                                 actions->conf;
3560                         const struct rte_flow_action *act = sample->actions;
3561                         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++)
3562                                 if (act->type == RTE_FLOW_ACTION_TYPE_RSS)
3563                                         rss = act->conf;
3564                         break;
3565                 }
3566                 case RTE_FLOW_ACTION_TYPE_METER:
3567                 {
3568                         uint32_t mtr_idx;
3569                         struct mlx5_flow_meter_info *fm;
3570                         struct mlx5_flow_meter_policy *policy;
3571                         const struct rte_flow_action_meter *mtr = actions->conf;
3572
3573                         fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx);
3574                         if (fm && !fm->def_policy) {
3575                                 policy = mlx5_flow_meter_policy_find(dev,
3576                                                 fm->policy_id, NULL);
3577                                 MLX5_ASSERT(policy);
3578                                 if (policy->is_hierarchy) {
3579                                         policy =
3580                                 mlx5_flow_meter_hierarchy_get_final_policy(dev,
3581                                                                         policy);
3582                                         if (!policy)
3583                                                 return NULL;
3584                                 }
3585                                 if (policy->is_rss)
3586                                         rss =
3587                                 policy->act_cnt[RTE_COLOR_GREEN].rss->conf;
3588                         }
3589                         break;
3590                 }
3591                 default:
3592                         break;
3593                 }
3594         }
3595         return rss;
3596 }
3597
3598 /**
3599  * Get ASO age action by index.
3600  *
3601  * @param[in] dev
3602  *   Pointer to the Ethernet device structure.
3603  * @param[in] age_idx
3604  *   Index to the ASO age action.
3605  *
3606  * @return
3607  *   The specified ASO age action.
3608  */
3609 struct mlx5_aso_age_action*
3610 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx)
3611 {
3612         uint16_t pool_idx = age_idx & UINT16_MAX;
3613         uint16_t offset = (age_idx >> 16) & UINT16_MAX;
3614         struct mlx5_priv *priv = dev->data->dev_private;
3615         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
3616         struct mlx5_aso_age_pool *pool = mng->pools[pool_idx];
3617
3618         return &pool->actions[offset - 1];
3619 }
3620
3621 /* maps indirect action to translated direct in some actions array */
3622 struct mlx5_translated_action_handle {
3623         struct rte_flow_action_handle *action; /**< Indirect action handle. */
3624         int index; /**< Index in related array of rte_flow_action. */
3625 };
3626
3627 /**
3628  * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related
3629  * direct action if translation possible.
3630  * This functionality used to run same execution path for both direct and
3631  * indirect actions on flow create. All necessary preparations for indirect
3632  * action handling should be performed on *handle* actions list returned
3633  * from this call.
3634  *
3635  * @param[in] dev
3636  *   Pointer to Ethernet device.
3637  * @param[in] actions
3638  *   List of actions to translate.
3639  * @param[out] handle
3640  *   List to store translated indirect action object handles.
3641  * @param[in, out] indir_n
3642  *   Size of *handle* array. On return should be updated with number of
3643  *   indirect actions retrieved from the *actions* list.
3644  * @param[out] translated_actions
3645  *   List of actions where all indirect actions were translated to direct
3646  *   if possible. NULL if no translation took place.
3647  * @param[out] error
3648  *   Pointer to the error structure.
3649  *
3650  * @return
3651  *   0 on success, a negative errno value otherwise and rte_errno is set.
3652  */
3653 static int
3654 flow_action_handles_translate(struct rte_eth_dev *dev,
3655                               const struct rte_flow_action actions[],
3656                               struct mlx5_translated_action_handle *handle,
3657                               int *indir_n,
3658                               struct rte_flow_action **translated_actions,
3659                               struct rte_flow_error *error)
3660 {
3661         struct mlx5_priv *priv = dev->data->dev_private;
3662         struct rte_flow_action *translated = NULL;
3663         size_t actions_size;
3664         int n;
3665         int copied_n = 0;
3666         struct mlx5_translated_action_handle *handle_end = NULL;
3667
3668         for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) {
3669                 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT)
3670                         continue;
3671                 if (copied_n == *indir_n) {
3672                         return rte_flow_error_set
3673                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM,
3674                                  NULL, "too many shared actions");
3675                 }
3676                 rte_memcpy(&handle[copied_n].action, &actions[n].conf,
3677                            sizeof(actions[n].conf));
3678                 handle[copied_n].index = n;
3679                 copied_n++;
3680         }
3681         n++;
3682         *indir_n = copied_n;
3683         if (!copied_n)
3684                 return 0;
3685         actions_size = sizeof(struct rte_flow_action) * n;
3686         translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY);
3687         if (!translated) {
3688                 rte_errno = ENOMEM;
3689                 return -ENOMEM;
3690         }
3691         memcpy(translated, actions, actions_size);
3692         for (handle_end = handle + copied_n; handle < handle_end; handle++) {
3693                 struct mlx5_shared_action_rss *shared_rss;
3694                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
3695                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
3696                 uint32_t idx = act_idx &
3697                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
3698
3699                 switch (type) {
3700                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
3701                         shared_rss = mlx5_ipool_get
3702                           (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
3703                         translated[handle->index].type =
3704                                 RTE_FLOW_ACTION_TYPE_RSS;
3705                         translated[handle->index].conf =
3706                                 &shared_rss->origin;
3707                         break;
3708                 case MLX5_INDIRECT_ACTION_TYPE_COUNT:
3709                         translated[handle->index].type =
3710                                                 (enum rte_flow_action_type)
3711                                                 MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
3712                         translated[handle->index].conf = (void *)(uintptr_t)idx;
3713                         break;
3714                 case MLX5_INDIRECT_ACTION_TYPE_AGE:
3715                         if (priv->sh->flow_hit_aso_en) {
3716                                 translated[handle->index].type =
3717                                         (enum rte_flow_action_type)
3718                                         MLX5_RTE_FLOW_ACTION_TYPE_AGE;
3719                                 translated[handle->index].conf =
3720                                                          (void *)(uintptr_t)idx;
3721                                 break;
3722                         }
3723                         /* Fall-through */
3724                 case MLX5_INDIRECT_ACTION_TYPE_CT:
3725                         if (priv->sh->ct_aso_en) {
3726                                 translated[handle->index].type =
3727                                         RTE_FLOW_ACTION_TYPE_CONNTRACK;
3728                                 translated[handle->index].conf =
3729                                                          (void *)(uintptr_t)idx;
3730                                 break;
3731                         }
3732                         /* Fall-through */
3733                 default:
3734                         mlx5_free(translated);
3735                         return rte_flow_error_set
3736                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
3737                                  NULL, "invalid indirect action type");
3738                 }
3739         }
3740         *translated_actions = translated;
3741         return 0;
3742 }
3743
3744 /**
3745  * Get Shared RSS action from the action list.
3746  *
3747  * @param[in] dev
3748  *   Pointer to Ethernet device.
3749  * @param[in] shared
3750  *   Pointer to the list of actions.
3751  * @param[in] shared_n
3752  *   Actions list length.
3753  *
3754  * @return
3755  *   The MLX5 RSS action ID if exists, otherwise return 0.
3756  */
3757 static uint32_t
3758 flow_get_shared_rss_action(struct rte_eth_dev *dev,
3759                            struct mlx5_translated_action_handle *handle,
3760                            int shared_n)
3761 {
3762         struct mlx5_translated_action_handle *handle_end;
3763         struct mlx5_priv *priv = dev->data->dev_private;
3764         struct mlx5_shared_action_rss *shared_rss;
3765
3766
3767         for (handle_end = handle + shared_n; handle < handle_end; handle++) {
3768                 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action;
3769                 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
3770                 uint32_t idx = act_idx &
3771                                ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
3772                 switch (type) {
3773                 case MLX5_INDIRECT_ACTION_TYPE_RSS:
3774                         shared_rss = mlx5_ipool_get
3775                                 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
3776                                                                            idx);
3777                         __atomic_add_fetch(&shared_rss->refcnt, 1,
3778                                            __ATOMIC_RELAXED);
3779                         return idx;
3780                 default:
3781                         break;
3782                 }
3783         }
3784         return 0;
3785 }
3786
3787 static unsigned int
3788 find_graph_root(uint32_t rss_level)
3789 {
3790         return rss_level < 2 ? MLX5_EXPANSION_ROOT :
3791                                MLX5_EXPANSION_ROOT_OUTER;
3792 }
3793
3794 /**
3795  *  Get layer flags from the prefix flow.
3796  *
3797  *  Some flows may be split to several subflows, the prefix subflow gets the
3798  *  match items and the suffix sub flow gets the actions.
3799  *  Some actions need the user defined match item flags to get the detail for
3800  *  the action.
3801  *  This function helps the suffix flow to get the item layer flags from prefix
3802  *  subflow.
3803  *
3804  * @param[in] dev_flow
3805  *   Pointer the created preifx subflow.
3806  *
3807  * @return
3808  *   The layers get from prefix subflow.
3809  */
3810 static inline uint64_t
3811 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow)
3812 {
3813         uint64_t layers = 0;
3814
3815         /*
3816          * Layers bits could be localization, but usually the compiler will
3817          * help to do the optimization work for source code.
3818          * If no decap actions, use the layers directly.
3819          */
3820         if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP))
3821                 return dev_flow->handle->layers;
3822         /* Convert L3 layers with decap action. */
3823         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
3824                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3825         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
3826                 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3827         /* Convert L4 layers with decap action.  */
3828         if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
3829                 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
3830         else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
3831                 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
3832         return layers;
3833 }
3834
3835 /**
3836  * Get metadata split action information.
3837  *
3838  * @param[in] actions
3839  *   Pointer to the list of actions.
3840  * @param[out] qrss
3841  *   Pointer to the return pointer.
3842  * @param[out] qrss_type
3843  *   Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
3844  *   if no QUEUE/RSS is found.
3845  * @param[out] encap_idx
3846  *   Pointer to the index of the encap action if exists, otherwise the last
3847  *   action index.
3848  *
3849  * @return
3850  *   Total number of actions.
3851  */
3852 static int
3853 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
3854                                        const struct rte_flow_action **qrss,
3855                                        int *encap_idx)
3856 {
3857         const struct rte_flow_action_raw_encap *raw_encap;
3858         int actions_n = 0;
3859         int raw_decap_idx = -1;
3860
3861         *encap_idx = -1;
3862         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3863                 switch (actions->type) {
3864                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3865                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3866                         *encap_idx = actions_n;
3867                         break;
3868                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3869                         raw_decap_idx = actions_n;
3870                         break;
3871                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3872                         raw_encap = actions->conf;
3873                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3874                                 *encap_idx = raw_decap_idx != -1 ?
3875                                                       raw_decap_idx : actions_n;
3876                         break;
3877                 case RTE_FLOW_ACTION_TYPE_QUEUE:
3878                 case RTE_FLOW_ACTION_TYPE_RSS:
3879                         *qrss = actions;
3880                         break;
3881                 default:
3882                         break;
3883                 }
3884                 actions_n++;
3885         }
3886         if (*encap_idx == -1)
3887                 *encap_idx = actions_n;
3888         /* Count RTE_FLOW_ACTION_TYPE_END. */
3889         return actions_n + 1;
3890 }
3891
3892 /**
3893  * Check if the action will change packet.
3894  *
3895  * @param dev
3896  *   Pointer to Ethernet device.
3897  * @param[in] type
3898  *   action type.
3899  *
3900  * @return
3901  *   true if action will change packet, false otherwise.
3902  */
3903 static bool flow_check_modify_action_type(struct rte_eth_dev *dev,
3904                                           enum rte_flow_action_type type)
3905 {
3906         struct mlx5_priv *priv = dev->data->dev_private;
3907
3908         switch (type) {
3909         case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
3910         case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
3911         case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
3912         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
3913         case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
3914         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
3915         case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
3916         case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
3917         case RTE_FLOW_ACTION_TYPE_DEC_TTL:
3918         case RTE_FLOW_ACTION_TYPE_SET_TTL:
3919         case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
3920         case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
3921         case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
3922         case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
3923         case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
3924         case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
3925         case RTE_FLOW_ACTION_TYPE_SET_META:
3926         case RTE_FLOW_ACTION_TYPE_SET_TAG:
3927         case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
3928         case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
3929         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
3930         case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
3931         case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
3932         case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
3933         case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
3934         case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
3935         case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
3936         case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
3937         case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
3938                 return true;
3939         case RTE_FLOW_ACTION_TYPE_FLAG:
3940         case RTE_FLOW_ACTION_TYPE_MARK:
3941                 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
3942                         return true;
3943                 else
3944                         return false;
3945         default:
3946                 return false;
3947         }
3948 }
3949
3950 /**
3951  * Check meter action from the action list.
3952  *
3953  * @param dev
3954  *   Pointer to Ethernet device.
3955  * @param[in] actions
3956  *   Pointer to the list of actions.
3957  * @param[out] has_mtr
3958  *   Pointer to the meter exist flag.
3959  * @param[out] has_modify
3960  *   Pointer to the flag showing there's packet change action.
3961  * @param[out] meter_id
3962  *   Pointer to the meter id.
3963  *
3964  * @return
3965  *   Total number of actions.
3966  */
3967 static int
3968 flow_check_meter_action(struct rte_eth_dev *dev,
3969                         const struct rte_flow_action actions[],
3970                         bool *has_mtr, bool *has_modify, uint32_t *meter_id)
3971 {
3972         const struct rte_flow_action_meter *mtr = NULL;
3973         int actions_n = 0;
3974
3975         MLX5_ASSERT(has_mtr);
3976         *has_mtr = false;
3977         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
3978                 switch (actions->type) {
3979                 case RTE_FLOW_ACTION_TYPE_METER:
3980                         mtr = actions->conf;
3981                         *meter_id = mtr->mtr_id;
3982                         *has_mtr = true;
3983                         break;
3984                 default:
3985                         break;
3986                 }
3987                 if (!*has_mtr)
3988                         *has_modify |= flow_check_modify_action_type(dev,
3989                                                                 actions->type);
3990                 actions_n++;
3991         }
3992         /* Count RTE_FLOW_ACTION_TYPE_END. */
3993         return actions_n + 1;
3994 }
3995
3996 /**
3997  * Check if the flow should be split due to hairpin.
3998  * The reason for the split is that in current HW we can't
3999  * support encap and push-vlan on Rx, so if a flow contains
4000  * these actions we move it to Tx.
4001  *
4002  * @param dev
4003  *   Pointer to Ethernet device.
4004  * @param[in] attr
4005  *   Flow rule attributes.
4006  * @param[in] actions
4007  *   Associated actions (list terminated by the END action).
4008  *
4009  * @return
4010  *   > 0 the number of actions and the flow should be split,
4011  *   0 when no split required.
4012  */
4013 static int
4014 flow_check_hairpin_split(struct rte_eth_dev *dev,
4015                          const struct rte_flow_attr *attr,
4016                          const struct rte_flow_action actions[])
4017 {
4018         int queue_action = 0;
4019         int action_n = 0;
4020         int split = 0;
4021         const struct rte_flow_action_queue *queue;
4022         const struct rte_flow_action_rss *rss;
4023         const struct rte_flow_action_raw_encap *raw_encap;
4024         const struct rte_eth_hairpin_conf *conf;
4025
4026         if (!attr->ingress)
4027                 return 0;
4028         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4029                 switch (actions->type) {
4030                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4031                         queue = actions->conf;
4032                         if (queue == NULL)
4033                                 return 0;
4034                         conf = mlx5_rxq_get_hairpin_conf(dev, queue->index);
4035                         if (conf == NULL || conf->tx_explicit != 0)
4036                                 return 0;
4037                         queue_action = 1;
4038                         action_n++;
4039                         break;
4040                 case RTE_FLOW_ACTION_TYPE_RSS:
4041                         rss = actions->conf;
4042                         if (rss == NULL || rss->queue_num == 0)
4043                                 return 0;
4044                         conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]);
4045                         if (conf == NULL || conf->tx_explicit != 0)
4046                                 return 0;
4047                         queue_action = 1;
4048                         action_n++;
4049                         break;
4050                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4051                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4052                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4053                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4054                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4055                         split++;
4056                         action_n++;
4057                         break;
4058                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4059                         raw_encap = actions->conf;
4060                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4061                                 split++;
4062                         action_n++;
4063                         break;
4064                 default:
4065                         action_n++;
4066                         break;
4067                 }
4068         }
4069         if (split && queue_action)
4070                 return action_n;
4071         return 0;
4072 }
4073
4074 /* Declare flow create/destroy prototype in advance. */
4075 static uint32_t
4076 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
4077                  const struct rte_flow_attr *attr,
4078                  const struct rte_flow_item items[],
4079                  const struct rte_flow_action actions[],
4080                  bool external, struct rte_flow_error *error);
4081
4082 static void
4083 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
4084                   uint32_t flow_idx);
4085
4086 int
4087 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused,
4088                       struct mlx5_list_entry *entry, void *cb_ctx)
4089 {
4090         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4091         struct mlx5_flow_mreg_copy_resource *mcp_res =
4092                                container_of(entry, typeof(*mcp_res), hlist_ent);
4093
4094         return mcp_res->mark_id != *(uint32_t *)(ctx->data);
4095 }
4096
4097 struct mlx5_list_entry *
4098 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx)
4099 {
4100         struct rte_eth_dev *dev = tool_ctx;
4101         struct mlx5_priv *priv = dev->data->dev_private;
4102         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4103         struct mlx5_flow_mreg_copy_resource *mcp_res;
4104         struct rte_flow_error *error = ctx->error;
4105         uint32_t idx = 0;
4106         int ret;
4107         uint32_t mark_id = *(uint32_t *)(ctx->data);
4108         struct rte_flow_attr attr = {
4109                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
4110                 .ingress = 1,
4111         };
4112         struct mlx5_rte_flow_item_tag tag_spec = {
4113                 .data = mark_id,
4114         };
4115         struct rte_flow_item items[] = {
4116                 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
4117         };
4118         struct rte_flow_action_mark ftag = {
4119                 .id = mark_id,
4120         };
4121         struct mlx5_flow_action_copy_mreg cp_mreg = {
4122                 .dst = REG_B,
4123                 .src = REG_NON,
4124         };
4125         struct rte_flow_action_jump jump = {
4126                 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
4127         };
4128         struct rte_flow_action actions[] = {
4129                 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
4130         };
4131
4132         /* Fill the register fileds in the flow. */
4133         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
4134         if (ret < 0)
4135                 return NULL;
4136         tag_spec.id = ret;
4137         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
4138         if (ret < 0)
4139                 return NULL;
4140         cp_mreg.src = ret;
4141         /* Provide the full width of FLAG specific value. */
4142         if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
4143                 tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
4144         /* Build a new flow. */
4145         if (mark_id != MLX5_DEFAULT_COPY_ID) {
4146                 items[0] = (struct rte_flow_item){
4147                         .type = (enum rte_flow_item_type)
4148                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
4149                         .spec = &tag_spec,
4150                 };
4151                 items[1] = (struct rte_flow_item){
4152                         .type = RTE_FLOW_ITEM_TYPE_END,
4153                 };
4154                 actions[0] = (struct rte_flow_action){
4155                         .type = (enum rte_flow_action_type)
4156                                 MLX5_RTE_FLOW_ACTION_TYPE_MARK,
4157                         .conf = &ftag,
4158                 };
4159                 actions[1] = (struct rte_flow_action){
4160                         .type = (enum rte_flow_action_type)
4161                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4162                         .conf = &cp_mreg,
4163                 };
4164                 actions[2] = (struct rte_flow_action){
4165                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4166                         .conf = &jump,
4167                 };
4168                 actions[3] = (struct rte_flow_action){
4169                         .type = RTE_FLOW_ACTION_TYPE_END,
4170                 };
4171         } else {
4172                 /* Default rule, wildcard match. */
4173                 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR;
4174                 items[0] = (struct rte_flow_item){
4175                         .type = RTE_FLOW_ITEM_TYPE_END,
4176                 };
4177                 actions[0] = (struct rte_flow_action){
4178                         .type = (enum rte_flow_action_type)
4179                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
4180                         .conf = &cp_mreg,
4181                 };
4182                 actions[1] = (struct rte_flow_action){
4183                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
4184                         .conf = &jump,
4185                 };
4186                 actions[2] = (struct rte_flow_action){
4187                         .type = RTE_FLOW_ACTION_TYPE_END,
4188                 };
4189         }
4190         /* Build a new entry. */
4191         mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
4192         if (!mcp_res) {
4193                 rte_errno = ENOMEM;
4194                 return NULL;
4195         }
4196         mcp_res->idx = idx;
4197         mcp_res->mark_id = mark_id;
4198         /*
4199          * The copy Flows are not included in any list. There
4200          * ones are referenced from other Flows and can not
4201          * be applied, removed, deleted in ardbitrary order
4202          * by list traversing.
4203          */
4204         mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP,
4205                                         &attr, items, actions, false, error);
4206         if (!mcp_res->rix_flow) {
4207                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
4208                 return NULL;
4209         }
4210         return &mcp_res->hlist_ent;
4211 }
4212
4213 struct mlx5_list_entry *
4214 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
4215                       void *cb_ctx __rte_unused)
4216 {
4217         struct rte_eth_dev *dev = tool_ctx;
4218         struct mlx5_priv *priv = dev->data->dev_private;
4219         struct mlx5_flow_mreg_copy_resource *mcp_res;
4220         uint32_t idx = 0;
4221
4222         mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
4223         if (!mcp_res) {
4224                 rte_errno = ENOMEM;
4225                 return NULL;
4226         }
4227         memcpy(mcp_res, oentry, sizeof(*mcp_res));
4228         mcp_res->idx = idx;
4229         return &mcp_res->hlist_ent;
4230 }
4231
4232 void
4233 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
4234 {
4235         struct mlx5_flow_mreg_copy_resource *mcp_res =
4236                                container_of(entry, typeof(*mcp_res), hlist_ent);
4237         struct rte_eth_dev *dev = tool_ctx;
4238         struct mlx5_priv *priv = dev->data->dev_private;
4239
4240         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
4241 }
4242
4243 /**
4244  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4245  *
4246  * As mark_id is unique, if there's already a registered flow for the mark_id,
4247  * return by increasing the reference counter of the resource. Otherwise, create
4248  * the resource (mcp_res) and flow.
4249  *
4250  * Flow looks like,
4251  *   - If ingress port is ANY and reg_c[1] is mark_id,
4252  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4253  *
4254  * For default flow (zero mark_id), flow is like,
4255  *   - If ingress port is ANY,
4256  *     reg_b := reg_c[0] and jump to RX_ACT_TBL.
4257  *
4258  * @param dev
4259  *   Pointer to Ethernet device.
4260  * @param mark_id
4261  *   ID of MARK action, zero means default flow for META.
4262  * @param[out] error
4263  *   Perform verbose error reporting if not NULL.
4264  *
4265  * @return
4266  *   Associated resource on success, NULL otherwise and rte_errno is set.
4267  */
4268 static struct mlx5_flow_mreg_copy_resource *
4269 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
4270                           struct rte_flow_error *error)
4271 {
4272         struct mlx5_priv *priv = dev->data->dev_private;
4273         struct mlx5_list_entry *entry;
4274         struct mlx5_flow_cb_ctx ctx = {
4275                 .dev = dev,
4276                 .error = error,
4277                 .data = &mark_id,
4278         };
4279
4280         /* Check if already registered. */
4281         MLX5_ASSERT(priv->mreg_cp_tbl);
4282         entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx);
4283         if (!entry)
4284                 return NULL;
4285         return container_of(entry, struct mlx5_flow_mreg_copy_resource,
4286                             hlist_ent);
4287 }
4288
4289 void
4290 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
4291 {
4292         struct mlx5_flow_mreg_copy_resource *mcp_res =
4293                                container_of(entry, typeof(*mcp_res), hlist_ent);
4294         struct rte_eth_dev *dev = tool_ctx;
4295         struct mlx5_priv *priv = dev->data->dev_private;
4296
4297         MLX5_ASSERT(mcp_res->rix_flow);
4298         flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow);
4299         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
4300 }
4301
4302 /**
4303  * Release flow in RX_CP_TBL.
4304  *
4305  * @param dev
4306  *   Pointer to Ethernet device.
4307  * @flow
4308  *   Parent flow for wich copying is provided.
4309  */
4310 static void
4311 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
4312                           struct rte_flow *flow)
4313 {
4314         struct mlx5_flow_mreg_copy_resource *mcp_res;
4315         struct mlx5_priv *priv = dev->data->dev_private;
4316
4317         if (!flow->rix_mreg_copy)
4318                 return;
4319         mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
4320                                  flow->rix_mreg_copy);
4321         if (!mcp_res || !priv->mreg_cp_tbl)
4322                 return;
4323         MLX5_ASSERT(mcp_res->rix_flow);
4324         mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
4325         flow->rix_mreg_copy = 0;
4326 }
4327
4328 /**
4329  * Remove the default copy action from RX_CP_TBL.
4330  *
4331  * This functions is called in the mlx5_dev_start(). No thread safe
4332  * is guaranteed.
4333  *
4334  * @param dev
4335  *   Pointer to Ethernet device.
4336  */
4337 static void
4338 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
4339 {
4340         struct mlx5_list_entry *entry;
4341         struct mlx5_priv *priv = dev->data->dev_private;
4342         struct mlx5_flow_cb_ctx ctx;
4343         uint32_t mark_id;
4344
4345         /* Check if default flow is registered. */
4346         if (!priv->mreg_cp_tbl)
4347                 return;
4348         mark_id = MLX5_DEFAULT_COPY_ID;
4349         ctx.data = &mark_id;
4350         entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx);
4351         if (!entry)
4352                 return;
4353         mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
4354 }
4355
4356 /**
4357  * Add the default copy action in in RX_CP_TBL.
4358  *
4359  * This functions is called in the mlx5_dev_start(). No thread safe
4360  * is guaranteed.
4361  *
4362  * @param dev
4363  *   Pointer to Ethernet device.
4364  * @param[out] error
4365  *   Perform verbose error reporting if not NULL.
4366  *
4367  * @return
4368  *   0 for success, negative value otherwise and rte_errno is set.
4369  */
4370 static int
4371 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
4372                                   struct rte_flow_error *error)
4373 {
4374         struct mlx5_priv *priv = dev->data->dev_private;
4375         struct mlx5_flow_mreg_copy_resource *mcp_res;
4376         struct mlx5_flow_cb_ctx ctx;
4377         uint32_t mark_id;
4378
4379         /* Check whether extensive metadata feature is engaged. */
4380         if (!priv->config.dv_flow_en ||
4381             priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4382             !mlx5_flow_ext_mreg_supported(dev) ||
4383             !priv->sh->dv_regc0_mask)
4384                 return 0;
4385         /*
4386          * Add default mreg copy flow may be called multiple time, but
4387          * only be called once in stop. Avoid register it twice.
4388          */
4389         mark_id = MLX5_DEFAULT_COPY_ID;
4390         ctx.data = &mark_id;
4391         if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx))
4392                 return 0;
4393         mcp_res = flow_mreg_add_copy_action(dev, mark_id, error);
4394         if (!mcp_res)
4395                 return -rte_errno;
4396         return 0;
4397 }
4398
4399 /**
4400  * Add a flow of copying flow metadata registers in RX_CP_TBL.
4401  *
4402  * All the flow having Q/RSS action should be split by
4403  * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
4404  * performs the following,
4405  *   - CQE->flow_tag := reg_c[1] (MARK)
4406  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
4407  * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
4408  * but there should be a flow per each MARK ID set by MARK action.
4409  *
4410  * For the aforementioned reason, if there's a MARK action in flow's action
4411  * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
4412  * the MARK ID to CQE's flow_tag like,
4413  *   - If reg_c[1] is mark_id,
4414  *     flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4415  *
4416  * For SET_META action which stores value in reg_c[0], as the destination is
4417  * also a flow metadata register (reg_b), adding a default flow is enough. Zero
4418  * MARK ID means the default flow. The default flow looks like,
4419  *   - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
4420  *
4421  * @param dev
4422  *   Pointer to Ethernet device.
4423  * @param flow
4424  *   Pointer to flow structure.
4425  * @param[in] actions
4426  *   Pointer to the list of actions.
4427  * @param[out] error
4428  *   Perform verbose error reporting if not NULL.
4429  *
4430  * @return
4431  *   0 on success, negative value otherwise and rte_errno is set.
4432  */
4433 static int
4434 flow_mreg_update_copy_table(struct rte_eth_dev *dev,
4435                             struct rte_flow *flow,
4436                             const struct rte_flow_action *actions,
4437                             struct rte_flow_error *error)
4438 {
4439         struct mlx5_priv *priv = dev->data->dev_private;
4440         struct mlx5_dev_config *config = &priv->config;
4441         struct mlx5_flow_mreg_copy_resource *mcp_res;
4442         const struct rte_flow_action_mark *mark;
4443
4444         /* Check whether extensive metadata feature is engaged. */
4445         if (!config->dv_flow_en ||
4446             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4447             !mlx5_flow_ext_mreg_supported(dev) ||
4448             !priv->sh->dv_regc0_mask)
4449                 return 0;
4450         /* Find MARK action. */
4451         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4452                 switch (actions->type) {
4453                 case RTE_FLOW_ACTION_TYPE_FLAG:
4454                         mcp_res = flow_mreg_add_copy_action
4455                                 (dev, MLX5_FLOW_MARK_DEFAULT, error);
4456                         if (!mcp_res)
4457                                 return -rte_errno;
4458                         flow->rix_mreg_copy = mcp_res->idx;
4459                         return 0;
4460                 case RTE_FLOW_ACTION_TYPE_MARK:
4461                         mark = (const struct rte_flow_action_mark *)
4462                                 actions->conf;
4463                         mcp_res =
4464                                 flow_mreg_add_copy_action(dev, mark->id, error);
4465                         if (!mcp_res)
4466                                 return -rte_errno;
4467                         flow->rix_mreg_copy = mcp_res->idx;
4468                         return 0;
4469                 default:
4470                         break;
4471                 }
4472         }
4473         return 0;
4474 }
4475
4476 #define MLX5_MAX_SPLIT_ACTIONS 24
4477 #define MLX5_MAX_SPLIT_ITEMS 24
4478
4479 /**
4480  * Split the hairpin flow.
4481  * Since HW can't support encap and push-vlan on Rx, we move these
4482  * actions to Tx.
4483  * If the count action is after the encap then we also
4484  * move the count action. in this case the count will also measure
4485  * the outer bytes.
4486  *
4487  * @param dev
4488  *   Pointer to Ethernet device.
4489  * @param[in] actions
4490  *   Associated actions (list terminated by the END action).
4491  * @param[out] actions_rx
4492  *   Rx flow actions.
4493  * @param[out] actions_tx
4494  *   Tx flow actions..
4495  * @param[out] pattern_tx
4496  *   The pattern items for the Tx flow.
4497  * @param[out] flow_id
4498  *   The flow ID connected to this flow.
4499  *
4500  * @return
4501  *   0 on success.
4502  */
4503 static int
4504 flow_hairpin_split(struct rte_eth_dev *dev,
4505                    const struct rte_flow_action actions[],
4506                    struct rte_flow_action actions_rx[],
4507                    struct rte_flow_action actions_tx[],
4508                    struct rte_flow_item pattern_tx[],
4509                    uint32_t flow_id)
4510 {
4511         const struct rte_flow_action_raw_encap *raw_encap;
4512         const struct rte_flow_action_raw_decap *raw_decap;
4513         struct mlx5_rte_flow_action_set_tag *set_tag;
4514         struct rte_flow_action *tag_action;
4515         struct mlx5_rte_flow_item_tag *tag_item;
4516         struct rte_flow_item *item;
4517         char *addr;
4518         int encap = 0;
4519
4520         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4521                 switch (actions->type) {
4522                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4523                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4524                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4525                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4526                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
4527                         rte_memcpy(actions_tx, actions,
4528                                sizeof(struct rte_flow_action));
4529                         actions_tx++;
4530                         break;
4531                 case RTE_FLOW_ACTION_TYPE_COUNT:
4532                         if (encap) {
4533                                 rte_memcpy(actions_tx, actions,
4534                                            sizeof(struct rte_flow_action));
4535                                 actions_tx++;
4536                         } else {
4537                                 rte_memcpy(actions_rx, actions,
4538                                            sizeof(struct rte_flow_action));
4539                                 actions_rx++;
4540                         }
4541                         break;
4542                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4543                         raw_encap = actions->conf;
4544                         if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) {
4545                                 memcpy(actions_tx, actions,
4546                                        sizeof(struct rte_flow_action));
4547                                 actions_tx++;
4548                                 encap = 1;
4549                         } else {
4550                                 rte_memcpy(actions_rx, actions,
4551                                            sizeof(struct rte_flow_action));
4552                                 actions_rx++;
4553                         }
4554                         break;
4555                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4556                         raw_decap = actions->conf;
4557                         if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) {
4558                                 memcpy(actions_tx, actions,
4559                                        sizeof(struct rte_flow_action));
4560                                 actions_tx++;
4561                         } else {
4562                                 rte_memcpy(actions_rx, actions,
4563                                            sizeof(struct rte_flow_action));
4564                                 actions_rx++;
4565                         }
4566                         break;
4567                 default:
4568                         rte_memcpy(actions_rx, actions,
4569                                    sizeof(struct rte_flow_action));
4570                         actions_rx++;
4571                         break;
4572                 }
4573         }
4574         /* Add set meta action and end action for the Rx flow. */
4575         tag_action = actions_rx;
4576         tag_action->type = (enum rte_flow_action_type)
4577                            MLX5_RTE_FLOW_ACTION_TYPE_TAG;
4578         actions_rx++;
4579         rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
4580         actions_rx++;
4581         set_tag = (void *)actions_rx;
4582         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
4583                 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL),
4584                 .data = flow_id,
4585         };
4586         MLX5_ASSERT(set_tag->id > REG_NON);
4587         tag_action->conf = set_tag;
4588         /* Create Tx item list. */
4589         rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
4590         addr = (void *)&pattern_tx[2];
4591         item = pattern_tx;
4592         item->type = (enum rte_flow_item_type)
4593                      MLX5_RTE_FLOW_ITEM_TYPE_TAG;
4594         tag_item = (void *)addr;
4595         tag_item->data = flow_id;
4596         tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
4597         MLX5_ASSERT(set_tag->id > REG_NON);
4598         item->spec = tag_item;
4599         addr += sizeof(struct mlx5_rte_flow_item_tag);
4600         tag_item = (void *)addr;
4601         tag_item->data = UINT32_MAX;
4602         tag_item->id = UINT16_MAX;
4603         item->mask = tag_item;
4604         item->last = NULL;
4605         item++;
4606         item->type = RTE_FLOW_ITEM_TYPE_END;
4607         return 0;
4608 }
4609
4610 /**
4611  * The last stage of splitting chain, just creates the subflow
4612  * without any modification.
4613  *
4614  * @param[in] dev
4615  *   Pointer to Ethernet device.
4616  * @param[in] flow
4617  *   Parent flow structure pointer.
4618  * @param[in, out] sub_flow
4619  *   Pointer to return the created subflow, may be NULL.
4620  * @param[in] attr
4621  *   Flow rule attributes.
4622  * @param[in] items
4623  *   Pattern specification (list terminated by the END pattern item).
4624  * @param[in] actions
4625  *   Associated actions (list terminated by the END action).
4626  * @param[in] flow_split_info
4627  *   Pointer to flow split info structure.
4628  * @param[out] error
4629  *   Perform verbose error reporting if not NULL.
4630  * @return
4631  *   0 on success, negative value otherwise
4632  */
4633 static int
4634 flow_create_split_inner(struct rte_eth_dev *dev,
4635                         struct rte_flow *flow,
4636                         struct mlx5_flow **sub_flow,
4637                         const struct rte_flow_attr *attr,
4638                         const struct rte_flow_item items[],
4639                         const struct rte_flow_action actions[],
4640                         struct mlx5_flow_split_info *flow_split_info,
4641                         struct rte_flow_error *error)
4642 {
4643         struct mlx5_flow *dev_flow;
4644
4645         dev_flow = flow_drv_prepare(dev, flow, attr, items, actions,
4646                                     flow_split_info->flow_idx, error);
4647         if (!dev_flow)
4648                 return -rte_errno;
4649         dev_flow->flow = flow;
4650         dev_flow->external = flow_split_info->external;
4651         dev_flow->skip_scale = flow_split_info->skip_scale;
4652         /* Subflow object was created, we must include one in the list. */
4653         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
4654                       dev_flow->handle, next);
4655         /*
4656          * If dev_flow is as one of the suffix flow, some actions in suffix
4657          * flow may need some user defined item layer flags, and pass the
4658          * Metadate rxq mark flag to suffix flow as well.
4659          */
4660         if (flow_split_info->prefix_layers)
4661                 dev_flow->handle->layers = flow_split_info->prefix_layers;
4662         if (flow_split_info->prefix_mark)
4663                 dev_flow->handle->mark = 1;
4664         if (sub_flow)
4665                 *sub_flow = dev_flow;
4666 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
4667         dev_flow->dv.table_id = flow_split_info->table_id;
4668 #endif
4669         return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
4670 }
4671
4672 /**
4673  * Get the sub policy of a meter.
4674  *
4675  * @param[in] dev
4676  *   Pointer to Ethernet device.
4677  * @param[in] flow
4678  *   Parent flow structure pointer.
4679  * @param wks
4680  *   Pointer to thread flow work space.
4681  * @param[in] attr
4682  *   Flow rule attributes.
4683  * @param[in] items
4684  *   Pattern specification (list terminated by the END pattern item).
4685  * @param[out] error
4686  *   Perform verbose error reporting if not NULL.
4687  *
4688  * @return
4689  *   Pointer to the meter sub policy, NULL otherwise and rte_errno is set.
4690  */
4691 static struct mlx5_flow_meter_sub_policy *
4692 get_meter_sub_policy(struct rte_eth_dev *dev,
4693                      struct rte_flow *flow,
4694                      struct mlx5_flow_workspace *wks,
4695                      const struct rte_flow_attr *attr,
4696                      const struct rte_flow_item items[],
4697                      struct rte_flow_error *error)
4698 {
4699         struct mlx5_flow_meter_policy *policy;
4700         struct mlx5_flow_meter_policy *final_policy;
4701         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
4702
4703         policy = wks->policy;
4704         final_policy = policy->is_hierarchy ? wks->final_policy : policy;
4705         if (final_policy->is_rss || final_policy->is_queue) {
4706                 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS];
4707                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0};
4708                 uint32_t i;
4709
4710                 /*
4711                  * This is a tmp dev_flow,
4712                  * no need to register any matcher for it in translate.
4713                  */
4714                 wks->skip_matcher_reg = 1;
4715                 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
4716                         struct mlx5_flow dev_flow = {0};
4717                         struct mlx5_flow_handle dev_handle = { {0} };
4718                         uint8_t fate = final_policy->act_cnt[i].fate_action;
4719
4720                         if (fate == MLX5_FLOW_FATE_SHARED_RSS) {
4721                                 const struct rte_flow_action_rss *rss_act =
4722                                         final_policy->act_cnt[i].rss->conf;
4723                                 struct rte_flow_action rss_actions[2] = {
4724                                         [0] = {
4725                                         .type = RTE_FLOW_ACTION_TYPE_RSS,
4726                                         .conf = rss_act,
4727                                         },
4728                                         [1] = {
4729                                         .type = RTE_FLOW_ACTION_TYPE_END,
4730                                         .conf = NULL,
4731                                         }
4732                                 };
4733
4734                                 dev_flow.handle = &dev_handle;
4735                                 dev_flow.ingress = attr->ingress;
4736                                 dev_flow.flow = flow;
4737                                 dev_flow.external = 0;
4738 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
4739                                 dev_flow.dv.transfer = attr->transfer;
4740 #endif
4741                                 /**
4742                                  * Translate RSS action to get rss hash fields.
4743                                  */
4744                                 if (flow_drv_translate(dev, &dev_flow, attr,
4745                                                 items, rss_actions, error))
4746                                         goto exit;
4747                                 rss_desc_v[i] = wks->rss_desc;
4748                                 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN;
4749                                 rss_desc_v[i].hash_fields =
4750                                                 dev_flow.hash_fields;
4751                                 rss_desc_v[i].queue_num =
4752                                                 rss_desc_v[i].hash_fields ?
4753                                                 rss_desc_v[i].queue_num : 1;
4754                                 rss_desc_v[i].tunnel =
4755                                                 !!(dev_flow.handle->layers &
4756                                                    MLX5_FLOW_LAYER_TUNNEL);
4757                                 /* Use the RSS queues in the containers. */
4758                                 rss_desc_v[i].queue =
4759                                         (uint16_t *)(uintptr_t)rss_act->queue;
4760                                 rss_desc[i] = &rss_desc_v[i];
4761                         } else if (fate == MLX5_FLOW_FATE_QUEUE) {
4762                                 /* This is queue action. */
4763                                 rss_desc_v[i] = wks->rss_desc;
4764                                 rss_desc_v[i].key_len = 0;
4765                                 rss_desc_v[i].hash_fields = 0;
4766                                 rss_desc_v[i].queue =
4767                                         &final_policy->act_cnt[i].queue;
4768                                 rss_desc_v[i].queue_num = 1;
4769                                 rss_desc[i] = &rss_desc_v[i];
4770                         } else {
4771                                 rss_desc[i] = NULL;
4772                         }
4773                 }
4774                 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev,
4775                                                 flow, policy, rss_desc);
4776         } else {
4777                 enum mlx5_meter_domain mtr_domain =
4778                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
4779                                 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
4780                                                 MLX5_MTR_DOMAIN_INGRESS);
4781                 sub_policy = policy->sub_policys[mtr_domain][0];
4782         }
4783         if (!sub_policy)
4784                 rte_flow_error_set(error, EINVAL,
4785                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4786                                    "Failed to get meter sub-policy.");
4787 exit:
4788         return sub_policy;
4789 }
4790
4791 /**
4792  * Split the meter flow.
4793  *
4794  * As meter flow will split to three sub flow, other than meter
4795  * action, the other actions make sense to only meter accepts
4796  * the packet. If it need to be dropped, no other additional
4797  * actions should be take.
4798  *
4799  * One kind of special action which decapsulates the L3 tunnel
4800  * header will be in the prefix sub flow, as not to take the
4801  * L3 tunnel header into account.
4802  *
4803  * @param[in] dev
4804  *   Pointer to Ethernet device.
4805  * @param[in] flow
4806  *   Parent flow structure pointer.
4807  * @param wks
4808  *   Pointer to thread flow work space.
4809  * @param[in] attr
4810  *   Flow rule attributes.
4811  * @param[in] items
4812  *   Pattern specification (list terminated by the END pattern item).
4813  * @param[out] sfx_items
4814  *   Suffix flow match items (list terminated by the END pattern item).
4815  * @param[in] actions
4816  *   Associated actions (list terminated by the END action).
4817  * @param[out] actions_sfx
4818  *   Suffix flow actions.
4819  * @param[out] actions_pre
4820  *   Prefix flow actions.
4821  * @param[out] mtr_flow_id
4822  *   Pointer to meter flow id.
4823  * @param[out] error
4824  *   Perform verbose error reporting if not NULL.
4825  *
4826  * @return
4827  *   0 on success, a negative errno value otherwise and rte_errno is set.
4828  */
4829 static int
4830 flow_meter_split_prep(struct rte_eth_dev *dev,
4831                       struct rte_flow *flow,
4832                       struct mlx5_flow_workspace *wks,
4833                       const struct rte_flow_attr *attr,
4834                       const struct rte_flow_item items[],
4835                       struct rte_flow_item sfx_items[],
4836                       const struct rte_flow_action actions[],
4837                       struct rte_flow_action actions_sfx[],
4838                       struct rte_flow_action actions_pre[],
4839                       uint32_t *mtr_flow_id,
4840                       struct rte_flow_error *error)
4841 {
4842         struct mlx5_priv *priv = dev->data->dev_private;
4843         struct mlx5_flow_meter_info *fm = wks->fm;
4844         struct rte_flow_action *tag_action = NULL;
4845         struct rte_flow_item *tag_item;
4846         struct mlx5_rte_flow_action_set_tag *set_tag;
4847         const struct rte_flow_action_raw_encap *raw_encap;
4848         const struct rte_flow_action_raw_decap *raw_decap;
4849         struct mlx5_rte_flow_item_tag *tag_item_spec;
4850         struct mlx5_rte_flow_item_tag *tag_item_mask;
4851         uint32_t tag_id = 0;
4852         struct rte_flow_item *vlan_item_dst = NULL;
4853         const struct rte_flow_item *vlan_item_src = NULL;
4854         struct rte_flow_action *hw_mtr_action;
4855         struct rte_flow_action *action_pre_head = NULL;
4856         int32_t flow_src_port = priv->representor_id;
4857         bool mtr_first;
4858         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
4859         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
4860                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
4861         uint32_t flow_id = 0;
4862         uint32_t flow_id_reversed = 0;
4863         uint8_t flow_id_bits = 0;
4864         int shift;
4865
4866         /* Prepare the suffix subflow items. */
4867         tag_item = sfx_items++;
4868         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4869                 struct mlx5_priv *port_priv;
4870                 const struct rte_flow_item_port_id *pid_v;
4871                 int item_type = items->type;
4872
4873                 switch (item_type) {
4874                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4875                         pid_v = items->spec;
4876                         MLX5_ASSERT(pid_v);
4877                         port_priv = mlx5_port_to_eswitch_info(pid_v->id, false);
4878                         if (!port_priv)
4879                                 return rte_flow_error_set(error,
4880                                                 rte_errno,
4881                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
4882                                                 pid_v,
4883                                                 "Failed to get port info.");
4884                         flow_src_port = port_priv->representor_id;
4885                         if (!fm->def_policy && wks->policy->is_hierarchy &&
4886                             flow_src_port != priv->representor_id) {
4887                                 if (flow_drv_mtr_hierarchy_rule_create(dev,
4888                                                                 flow, fm,
4889                                                                 flow_src_port,
4890                                                                 items,
4891                                                                 error))
4892                                         return -rte_errno;
4893                         }
4894                         memcpy(sfx_items, items, sizeof(*sfx_items));
4895                         sfx_items++;
4896                         break;
4897                 case RTE_FLOW_ITEM_TYPE_VLAN:
4898                         /* Determine if copy vlan item below. */
4899                         vlan_item_src = items;
4900                         vlan_item_dst = sfx_items++;
4901                         vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID;
4902                         break;
4903                 default:
4904                         break;
4905                 }
4906         }
4907         sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
4908         sfx_items++;
4909         mtr_first = priv->sh->meter_aso_en &&
4910                 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX));
4911         /* For ASO meter, meter must be before tag in TX direction. */
4912         if (mtr_first) {
4913                 action_pre_head = actions_pre++;
4914                 /* Leave space for tag action. */
4915                 tag_action = actions_pre++;
4916         }
4917         /* Prepare the actions for prefix and suffix flow. */
4918         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4919                 struct rte_flow_action *action_cur = NULL;
4920
4921                 switch (actions->type) {
4922                 case RTE_FLOW_ACTION_TYPE_METER:
4923                         if (mtr_first) {
4924                                 action_cur = action_pre_head;
4925                         } else {
4926                                 /* Leave space for tag action. */
4927                                 tag_action = actions_pre++;
4928                                 action_cur = actions_pre++;
4929                         }
4930                         break;
4931                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4932                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4933                         action_cur = actions_pre++;
4934                         break;
4935                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4936                         raw_encap = actions->conf;
4937                         if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
4938                                 action_cur = actions_pre++;
4939                         break;
4940                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4941                         raw_decap = actions->conf;
4942                         if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
4943                                 action_cur = actions_pre++;
4944                         break;
4945                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
4946                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
4947                         if (vlan_item_dst && vlan_item_src) {
4948                                 memcpy(vlan_item_dst, vlan_item_src,
4949                                         sizeof(*vlan_item_dst));
4950                                 /*
4951                                  * Convert to internal match item, it is used
4952                                  * for vlan push and set vid.
4953                                  */
4954                                 vlan_item_dst->type = (enum rte_flow_item_type)
4955                                                 MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
4956                         }
4957                         break;
4958                 default:
4959                         break;
4960                 }
4961                 if (!action_cur)
4962                         action_cur = (fm->def_policy) ?
4963                                         actions_sfx++ : actions_pre++;
4964                 memcpy(action_cur, actions, sizeof(struct rte_flow_action));
4965         }
4966         /* Add end action to the actions. */
4967         actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
4968         if (priv->sh->meter_aso_en) {
4969                 /**
4970                  * For ASO meter, need to add an extra jump action explicitly,
4971                  * to jump from meter to policer table.
4972                  */
4973                 struct mlx5_flow_meter_sub_policy *sub_policy;
4974                 struct mlx5_flow_tbl_data_entry *tbl_data;
4975
4976                 if (!fm->def_policy) {
4977                         sub_policy = get_meter_sub_policy(dev, flow, wks,
4978                                                           attr, items, error);
4979                         if (!sub_policy)
4980                                 return -rte_errno;
4981                 } else {
4982                         enum mlx5_meter_domain mtr_domain =
4983                         attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
4984                                 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS :
4985                                                 MLX5_MTR_DOMAIN_INGRESS);
4986
4987                         sub_policy =
4988                         &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy;
4989                 }
4990                 tbl_data = container_of(sub_policy->tbl_rsc,
4991                                         struct mlx5_flow_tbl_data_entry, tbl);
4992                 hw_mtr_action = actions_pre++;
4993                 hw_mtr_action->type = (enum rte_flow_action_type)
4994                                       MLX5_RTE_FLOW_ACTION_TYPE_JUMP;
4995                 hw_mtr_action->conf = tbl_data->jump.action;
4996         }
4997         actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
4998         actions_pre++;
4999         if (!tag_action)
5000                 return rte_flow_error_set(error, ENOMEM,
5001                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5002                                           NULL, "No tag action space.");
5003         if (!mtr_flow_id) {
5004                 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID;
5005                 goto exit;
5006         }
5007         /* Only default-policy Meter creates mtr flow id. */
5008         if (fm->def_policy) {
5009                 mlx5_ipool_malloc(fm->flow_ipool, &tag_id);
5010                 if (!tag_id)
5011                         return rte_flow_error_set(error, ENOMEM,
5012                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5013                                         "Failed to allocate meter flow id.");
5014                 flow_id = tag_id - 1;
5015                 flow_id_bits = (!flow_id) ? 1 :
5016                                 (MLX5_REG_BITS - __builtin_clz(flow_id));
5017                 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) >
5018                     mtr_reg_bits) {
5019                         mlx5_ipool_free(fm->flow_ipool, tag_id);
5020                         return rte_flow_error_set(error, EINVAL,
5021                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5022                                         "Meter flow id exceeds max limit.");
5023                 }
5024                 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits)
5025                         priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits;
5026         }
5027         /* Build tag actions and items for meter_id/meter flow_id. */
5028         set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre;
5029         tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
5030         tag_item_mask = tag_item_spec + 1;
5031         /* Both flow_id and meter_id share the same register. */
5032         *set_tag = (struct mlx5_rte_flow_action_set_tag) {
5033                 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
5034                                                             0, error),
5035                 .offset = mtr_id_offset,
5036                 .length = mtr_reg_bits,
5037                 .data = flow->meter,
5038         };
5039         /*
5040          * The color Reg bits used by flow_id are growing from
5041          * msb to lsb, so must do bit reverse for flow_id val in RegC.
5042          */
5043         for (shift = 0; shift < flow_id_bits; shift++)
5044                 flow_id_reversed = (flow_id_reversed << 1) |
5045                                 ((flow_id >> shift) & 0x1);
5046         set_tag->data |=
5047                 flow_id_reversed << (mtr_reg_bits - flow_id_bits);
5048         tag_item_spec->id = set_tag->id;
5049         tag_item_spec->data = set_tag->data << mtr_id_offset;
5050         tag_item_mask->data = UINT32_MAX << mtr_id_offset;
5051         tag_action->type = (enum rte_flow_action_type)
5052                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5053         tag_action->conf = set_tag;
5054         tag_item->type = (enum rte_flow_item_type)
5055                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG;
5056         tag_item->spec = tag_item_spec;
5057         tag_item->last = NULL;
5058         tag_item->mask = tag_item_mask;
5059 exit:
5060         if (mtr_flow_id)
5061                 *mtr_flow_id = tag_id;
5062         return 0;
5063 }
5064
5065 /**
5066  * Split action list having QUEUE/RSS for metadata register copy.
5067  *
5068  * Once Q/RSS action is detected in user's action list, the flow action
5069  * should be split in order to copy metadata registers, which will happen in
5070  * RX_CP_TBL like,
5071  *   - CQE->flow_tag := reg_c[1] (MARK)
5072  *   - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
5073  * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL.
5074  * This is because the last action of each flow must be a terminal action
5075  * (QUEUE, RSS or DROP).
5076  *
5077  * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is
5078  * stored and kept in the mlx5_flow structure per each sub_flow.
5079  *
5080  * The Q/RSS action is replaced with,
5081  *   - SET_TAG, setting the allocated flow ID to reg_c[2].
5082  * And the following JUMP action is added at the end,
5083  *   - JUMP, to RX_CP_TBL.
5084  *
5085  * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by
5086  * flow_create_split_metadata() routine. The flow will look like,
5087  *   - If flow ID matches (reg_c[2]), perform Q/RSS.
5088  *
5089  * @param dev
5090  *   Pointer to Ethernet device.
5091  * @param[out] split_actions
5092  *   Pointer to store split actions to jump to CP_TBL.
5093  * @param[in] actions
5094  *   Pointer to the list of original flow actions.
5095  * @param[in] qrss
5096  *   Pointer to the Q/RSS action.
5097  * @param[in] actions_n
5098  *   Number of original actions.
5099  * @param[out] error
5100  *   Perform verbose error reporting if not NULL.
5101  *
5102  * @return
5103  *   non-zero unique flow_id on success, otherwise 0 and
5104  *   error/rte_error are set.
5105  */
5106 static uint32_t
5107 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev,
5108                           struct rte_flow_action *split_actions,
5109                           const struct rte_flow_action *actions,
5110                           const struct rte_flow_action *qrss,
5111                           int actions_n, struct rte_flow_error *error)
5112 {
5113         struct mlx5_priv *priv = dev->data->dev_private;
5114         struct mlx5_rte_flow_action_set_tag *set_tag;
5115         struct rte_flow_action_jump *jump;
5116         const int qrss_idx = qrss - actions;
5117         uint32_t flow_id = 0;
5118         int ret = 0;
5119
5120         /*
5121          * Given actions will be split
5122          * - Replace QUEUE/RSS action with SET_TAG to set flow ID.
5123          * - Add jump to mreg CP_TBL.
5124          * As a result, there will be one more action.
5125          */
5126         ++actions_n;
5127         memcpy(split_actions, actions, sizeof(*split_actions) * actions_n);
5128         set_tag = (void *)(split_actions + actions_n);
5129         /*
5130          * If tag action is not set to void(it means we are not the meter
5131          * suffix flow), add the tag action. Since meter suffix flow already
5132          * has the tag added.
5133          */
5134         if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) {
5135                 /*
5136                  * Allocate the new subflow ID. This one is unique within
5137                  * device and not shared with representors. Otherwise,
5138                  * we would have to resolve multi-thread access synch
5139                  * issue. Each flow on the shared device is appended
5140                  * with source vport identifier, so the resulting
5141                  * flows will be unique in the shared (by master and
5142                  * representors) domain even if they have coinciding
5143                  * IDs.
5144                  */
5145                 mlx5_ipool_malloc(priv->sh->ipool
5146                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id);
5147                 if (!flow_id)
5148                         return rte_flow_error_set(error, ENOMEM,
5149                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5150                                                   NULL, "can't allocate id "
5151                                                   "for split Q/RSS subflow");
5152                 /* Internal SET_TAG action to set flow ID. */
5153                 *set_tag = (struct mlx5_rte_flow_action_set_tag){
5154                         .data = flow_id,
5155                 };
5156                 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error);
5157                 if (ret < 0)
5158                         return ret;
5159                 set_tag->id = ret;
5160                 /* Construct new actions array. */
5161                 /* Replace QUEUE/RSS action. */
5162                 split_actions[qrss_idx] = (struct rte_flow_action){
5163                         .type = (enum rte_flow_action_type)
5164                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
5165                         .conf = set_tag,
5166                 };
5167         }
5168         /* JUMP action to jump to mreg copy table (CP_TBL). */
5169         jump = (void *)(set_tag + 1);
5170         *jump = (struct rte_flow_action_jump){
5171                 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
5172         };
5173         split_actions[actions_n - 2] = (struct rte_flow_action){
5174                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
5175                 .conf = jump,
5176         };
5177         split_actions[actions_n - 1] = (struct rte_flow_action){
5178                 .type = RTE_FLOW_ACTION_TYPE_END,
5179         };
5180         return flow_id;
5181 }
5182
5183 /**
5184  * Extend the given action list for Tx metadata copy.
5185  *
5186  * Copy the given action list to the ext_actions and add flow metadata register
5187  * copy action in order to copy reg_a set by WQE to reg_c[0].
5188  *
5189  * @param[out] ext_actions
5190  *   Pointer to the extended action list.
5191  * @param[in] actions
5192  *   Pointer to the list of actions.
5193  * @param[in] actions_n
5194  *   Number of actions in the list.
5195  * @param[out] error
5196  *   Perform verbose error reporting if not NULL.
5197  * @param[in] encap_idx
5198  *   The encap action inndex.
5199  *
5200  * @return
5201  *   0 on success, negative value otherwise
5202  */
5203 static int
5204 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev,
5205                        struct rte_flow_action *ext_actions,
5206                        const struct rte_flow_action *actions,
5207                        int actions_n, struct rte_flow_error *error,
5208                        int encap_idx)
5209 {
5210         struct mlx5_flow_action_copy_mreg *cp_mreg =
5211                 (struct mlx5_flow_action_copy_mreg *)
5212                         (ext_actions + actions_n + 1);
5213         int ret;
5214
5215         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
5216         if (ret < 0)
5217                 return ret;
5218         cp_mreg->dst = ret;
5219         ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error);
5220         if (ret < 0)
5221                 return ret;
5222         cp_mreg->src = ret;
5223         if (encap_idx != 0)
5224                 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx);
5225         if (encap_idx == actions_n - 1) {
5226                 ext_actions[actions_n - 1] = (struct rte_flow_action){
5227                         .type = (enum rte_flow_action_type)
5228                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5229                         .conf = cp_mreg,
5230                 };
5231                 ext_actions[actions_n] = (struct rte_flow_action){
5232                         .type = RTE_FLOW_ACTION_TYPE_END,
5233                 };
5234         } else {
5235                 ext_actions[encap_idx] = (struct rte_flow_action){
5236                         .type = (enum rte_flow_action_type)
5237                                 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
5238                         .conf = cp_mreg,
5239                 };
5240                 memcpy(ext_actions + encap_idx + 1, actions + encap_idx,
5241                                 sizeof(*ext_actions) * (actions_n - encap_idx));
5242         }
5243         return 0;
5244 }
5245
5246 /**
5247  * Check the match action from the action list.
5248  *
5249  * @param[in] actions
5250  *   Pointer to the list of actions.
5251  * @param[in] attr
5252  *   Flow rule attributes.
5253  * @param[in] action
5254  *   The action to be check if exist.
5255  * @param[out] match_action_pos
5256  *   Pointer to the position of the matched action if exists, otherwise is -1.
5257  * @param[out] qrss_action_pos
5258  *   Pointer to the position of the Queue/RSS action if exists, otherwise is -1.
5259  * @param[out] modify_after_mirror
5260  *   Pointer to the flag of modify action after FDB mirroring.
5261  *
5262  * @return
5263  *   > 0 the total number of actions.
5264  *   0 if not found match action in action list.
5265  */
5266 static int
5267 flow_check_match_action(const struct rte_flow_action actions[],
5268                         const struct rte_flow_attr *attr,
5269                         enum rte_flow_action_type action,
5270                         int *match_action_pos, int *qrss_action_pos,
5271                         int *modify_after_mirror)
5272 {
5273         const struct rte_flow_action_sample *sample;
5274         const struct rte_flow_action_raw_decap *decap;
5275         int actions_n = 0;
5276         uint32_t ratio = 0;
5277         int sub_type = 0;
5278         int flag = 0;
5279         int fdb_mirror = 0;
5280
5281         *match_action_pos = -1;
5282         *qrss_action_pos = -1;
5283         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5284                 if (actions->type == action) {
5285                         flag = 1;
5286                         *match_action_pos = actions_n;
5287                 }
5288                 switch (actions->type) {
5289                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5290                 case RTE_FLOW_ACTION_TYPE_RSS:
5291                         *qrss_action_pos = actions_n;
5292                         break;
5293                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
5294                         sample = actions->conf;
5295                         ratio = sample->ratio;
5296                         sub_type = ((const struct rte_flow_action *)
5297                                         (sample->actions))->type;
5298                         if (ratio == 1 && attr->transfer)
5299                                 fdb_mirror = 1;
5300                         break;
5301                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5302                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5303                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5304                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5305                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5306                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5307                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5308                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5309                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5310                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5311                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5312                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5313                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5314                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5315                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5316                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5317                 case RTE_FLOW_ACTION_TYPE_FLAG:
5318                 case RTE_FLOW_ACTION_TYPE_MARK:
5319                 case RTE_FLOW_ACTION_TYPE_SET_META:
5320                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5321                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5322                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5323                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5324                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5325                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5326                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5327                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
5328                 case RTE_FLOW_ACTION_TYPE_METER:
5329                         if (fdb_mirror)
5330                                 *modify_after_mirror = 1;
5331                         break;
5332                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5333                         decap = actions->conf;
5334                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5335                                 ;
5336                         actions_n++;
5337                         if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5338                                 const struct rte_flow_action_raw_encap *encap =
5339                                                                 actions->conf;
5340                                 if (decap->size <=
5341                                         MLX5_ENCAPSULATION_DECISION_SIZE &&
5342                                     encap->size >
5343                                         MLX5_ENCAPSULATION_DECISION_SIZE)
5344                                         /* L3 encap. */
5345                                         break;
5346                         }
5347                         if (fdb_mirror)
5348                                 *modify_after_mirror = 1;
5349                         break;
5350                 default:
5351                         break;
5352                 }
5353                 actions_n++;
5354         }
5355         if (flag && fdb_mirror && !*modify_after_mirror) {
5356                 /* FDB mirroring uses the destination array to implement
5357                  * instead of FLOW_SAMPLER object.
5358                  */
5359                 if (sub_type != RTE_FLOW_ACTION_TYPE_END)
5360                         flag = 0;
5361         }
5362         /* Count RTE_FLOW_ACTION_TYPE_END. */
5363         return flag ? actions_n + 1 : 0;
5364 }
5365
5366 #define SAMPLE_SUFFIX_ITEM 2
5367
5368 /**
5369  * Split the sample flow.
5370  *
5371  * As sample flow will split to two sub flow, sample flow with
5372  * sample action, the other actions will move to new suffix flow.
5373  *
5374  * Also add unique tag id with tag action in the sample flow,
5375  * the same tag id will be as match in the suffix flow.
5376  *
5377  * @param dev
5378  *   Pointer to Ethernet device.
5379  * @param[in] add_tag
5380  *   Add extra tag action flag.
5381  * @param[out] sfx_items
5382  *   Suffix flow match items (list terminated by the END pattern item).
5383  * @param[in] actions
5384  *   Associated actions (list terminated by the END action).
5385  * @param[out] actions_sfx
5386  *   Suffix flow actions.
5387  * @param[out] actions_pre
5388  *   Prefix flow actions.
5389  * @param[in] actions_n
5390  *  The total number of actions.
5391  * @param[in] sample_action_pos
5392  *   The sample action position.
5393  * @param[in] qrss_action_pos
5394  *   The Queue/RSS action position.
5395  * @param[in] jump_table
5396  *   Add extra jump action flag.
5397  * @param[out] error
5398  *   Perform verbose error reporting if not NULL.
5399  *
5400  * @return
5401  *   0 on success, or unique flow_id, a negative errno value
5402  *   otherwise and rte_errno is set.
5403  */
5404 static int
5405 flow_sample_split_prep(struct rte_eth_dev *dev,
5406                        int add_tag,
5407                        struct rte_flow_item sfx_items[],
5408                        const struct rte_flow_action actions[],
5409                        struct rte_flow_action actions_sfx[],
5410                        struct rte_flow_action actions_pre[],
5411                        int actions_n,
5412                        int sample_action_pos,
5413                        int qrss_action_pos,
5414                        int jump_table,
5415                        struct rte_flow_error *error)
5416 {
5417         struct mlx5_priv *priv = dev->data->dev_private;
5418         struct mlx5_rte_flow_action_set_tag *set_tag;
5419         struct mlx5_rte_flow_item_tag *tag_spec;
5420         struct mlx5_rte_flow_item_tag *tag_mask;
5421         struct rte_flow_action_jump *jump_action;
5422         uint32_t tag_id = 0;
5423         int index;
5424         int append_index = 0;
5425         int ret;
5426
5427         if (sample_action_pos < 0)
5428                 return rte_flow_error_set(error, EINVAL,
5429                                           RTE_FLOW_ERROR_TYPE_ACTION,
5430                                           NULL, "invalid position of sample "
5431                                           "action in list");
5432         /* Prepare the actions for prefix and suffix flow. */
5433         if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) {
5434                 index = qrss_action_pos;
5435                 /* Put the preceding the Queue/RSS action into prefix flow. */
5436                 if (index != 0)
5437                         memcpy(actions_pre, actions,
5438                                sizeof(struct rte_flow_action) * index);
5439                 /* Put others preceding the sample action into prefix flow. */
5440                 if (sample_action_pos > index + 1)
5441                         memcpy(actions_pre + index, actions + index + 1,
5442                                sizeof(struct rte_flow_action) *
5443                                (sample_action_pos - index - 1));
5444                 index = sample_action_pos - 1;
5445                 /* Put Queue/RSS action into Suffix flow. */
5446                 memcpy(actions_sfx, actions + qrss_action_pos,
5447                        sizeof(struct rte_flow_action));
5448                 actions_sfx++;
5449         } else {
5450                 index = sample_action_pos;
5451                 if (index != 0)
5452                         memcpy(actions_pre, actions,
5453                                sizeof(struct rte_flow_action) * index);
5454         }
5455         /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress.
5456          * For CX6DX and above, metadata registers Cx preserve their value,
5457          * add an extra tag action for NIC-RX and E-Switch Domain.
5458          */
5459         if (add_tag) {
5460                 /* Prepare the prefix tag action. */
5461                 append_index++;
5462                 set_tag = (void *)(actions_pre + actions_n + append_index);
5463                 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error);
5464                 if (ret < 0)
5465                         return ret;
5466                 mlx5_ipool_malloc(priv->sh->ipool
5467                                   [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id);
5468                 *set_tag = (struct mlx5_rte_flow_action_set_tag) {
5469                         .id = ret,
5470                         .data = tag_id,
5471                 };
5472                 /* Prepare the suffix subflow items. */
5473                 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM);
5474                 tag_spec->data = tag_id;
5475                 tag_spec->id = set_tag->id;
5476                 tag_mask = tag_spec + 1;
5477                 tag_mask->data = UINT32_MAX;
5478                 sfx_items[0] = (struct rte_flow_item){
5479                         .type = (enum rte_flow_item_type)
5480                                 MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5481                         .spec = tag_spec,
5482                         .last = NULL,
5483                         .mask = tag_mask,
5484                 };
5485                 sfx_items[1] = (struct rte_flow_item){
5486                         .type = (enum rte_flow_item_type)
5487                                 RTE_FLOW_ITEM_TYPE_END,
5488                 };
5489                 /* Prepare the tag action in prefix subflow. */
5490                 actions_pre[index++] =
5491                         (struct rte_flow_action){
5492                         .type = (enum rte_flow_action_type)
5493                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG,
5494                         .conf = set_tag,
5495                 };
5496         }
5497         memcpy(actions_pre + index, actions + sample_action_pos,
5498                sizeof(struct rte_flow_action));
5499         index += 1;
5500         /* For the modify action after the sample action in E-Switch mirroring,
5501          * Add the extra jump action in prefix subflow and jump into the next
5502          * table, then do the modify action in the new table.
5503          */
5504         if (jump_table) {
5505                 /* Prepare the prefix jump action. */
5506                 append_index++;
5507                 jump_action = (void *)(actions_pre + actions_n + append_index);
5508                 jump_action->group = jump_table;
5509                 actions_pre[index++] =
5510                         (struct rte_flow_action){
5511                         .type = (enum rte_flow_action_type)
5512                                 RTE_FLOW_ACTION_TYPE_JUMP,
5513                         .conf = jump_action,
5514                 };
5515         }
5516         actions_pre[index] = (struct rte_flow_action){
5517                 .type = (enum rte_flow_action_type)
5518                         RTE_FLOW_ACTION_TYPE_END,
5519         };
5520         /* Put the actions after sample into Suffix flow. */
5521         memcpy(actions_sfx, actions + sample_action_pos + 1,
5522                sizeof(struct rte_flow_action) *
5523                (actions_n - sample_action_pos - 1));
5524         return tag_id;
5525 }
5526
5527 /**
5528  * The splitting for metadata feature.
5529  *
5530  * - Q/RSS action on NIC Rx should be split in order to pass by
5531  *   the mreg copy table (RX_CP_TBL) and then it jumps to the
5532  *   action table (RX_ACT_TBL) which has the split Q/RSS action.
5533  *
5534  * - All the actions on NIC Tx should have a mreg copy action to
5535  *   copy reg_a from WQE to reg_c[0].
5536  *
5537  * @param dev
5538  *   Pointer to Ethernet device.
5539  * @param[in] flow
5540  *   Parent flow structure pointer.
5541  * @param[in] attr
5542  *   Flow rule attributes.
5543  * @param[in] items
5544  *   Pattern specification (list terminated by the END pattern item).
5545  * @param[in] actions
5546  *   Associated actions (list terminated by the END action).
5547  * @param[in] flow_split_info
5548  *   Pointer to flow split info structure.
5549  * @param[out] error
5550  *   Perform verbose error reporting if not NULL.
5551  * @return
5552  *   0 on success, negative value otherwise
5553  */
5554 static int
5555 flow_create_split_metadata(struct rte_eth_dev *dev,
5556                            struct rte_flow *flow,
5557                            const struct rte_flow_attr *attr,
5558                            const struct rte_flow_item items[],
5559                            const struct rte_flow_action actions[],
5560                            struct mlx5_flow_split_info *flow_split_info,
5561                            struct rte_flow_error *error)
5562 {
5563         struct mlx5_priv *priv = dev->data->dev_private;
5564         struct mlx5_dev_config *config = &priv->config;
5565         const struct rte_flow_action *qrss = NULL;
5566         struct rte_flow_action *ext_actions = NULL;
5567         struct mlx5_flow *dev_flow = NULL;
5568         uint32_t qrss_id = 0;
5569         int mtr_sfx = 0;
5570         size_t act_size;
5571         int actions_n;
5572         int encap_idx;
5573         int ret;
5574
5575         /* Check whether extensive metadata feature is engaged. */
5576         if (!config->dv_flow_en ||
5577             config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
5578             !mlx5_flow_ext_mreg_supported(dev))
5579                 return flow_create_split_inner(dev, flow, NULL, attr, items,
5580                                                actions, flow_split_info, error);
5581         actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
5582                                                            &encap_idx);
5583         if (qrss) {
5584                 /* Exclude hairpin flows from splitting. */
5585                 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
5586                         const struct rte_flow_action_queue *queue;
5587
5588                         queue = qrss->conf;
5589                         if (mlx5_rxq_get_type(dev, queue->index) ==
5590                             MLX5_RXQ_TYPE_HAIRPIN)
5591                                 qrss = NULL;
5592                 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) {
5593                         const struct rte_flow_action_rss *rss;
5594
5595                         rss = qrss->conf;
5596                         if (mlx5_rxq_get_type(dev, rss->queue[0]) ==
5597                             MLX5_RXQ_TYPE_HAIRPIN)
5598                                 qrss = NULL;
5599                 }
5600         }
5601         if (qrss) {
5602                 /* Check if it is in meter suffix table. */
5603                 mtr_sfx = attr->group == (attr->transfer ?
5604                           (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
5605                           MLX5_FLOW_TABLE_LEVEL_METER);
5606                 /*
5607                  * Q/RSS action on NIC Rx should be split in order to pass by
5608                  * the mreg copy table (RX_CP_TBL) and then it jumps to the
5609                  * action table (RX_ACT_TBL) which has the split Q/RSS action.
5610                  */
5611                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5612                            sizeof(struct rte_flow_action_set_tag) +
5613                            sizeof(struct rte_flow_action_jump);
5614                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5615                                           SOCKET_ID_ANY);
5616                 if (!ext_actions)
5617                         return rte_flow_error_set(error, ENOMEM,
5618                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5619                                                   NULL, "no memory to split "
5620                                                   "metadata flow");
5621                 /*
5622                  * If we are the suffix flow of meter, tag already exist.
5623                  * Set the tag action to void.
5624                  */
5625                 if (mtr_sfx)
5626                         ext_actions[qrss - actions].type =
5627                                                 RTE_FLOW_ACTION_TYPE_VOID;
5628                 else
5629                         ext_actions[qrss - actions].type =
5630                                                 (enum rte_flow_action_type)
5631                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
5632                 /*
5633                  * Create the new actions list with removed Q/RSS action
5634                  * and appended set tag and jump to register copy table
5635                  * (RX_CP_TBL). We should preallocate unique tag ID here
5636                  * in advance, because it is needed for set tag action.
5637                  */
5638                 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions,
5639                                                     qrss, actions_n, error);
5640                 if (!mtr_sfx && !qrss_id) {
5641                         ret = -rte_errno;
5642                         goto exit;
5643                 }
5644         } else if (attr->egress && !attr->transfer) {
5645                 /*
5646                  * All the actions on NIC Tx should have a metadata register
5647                  * copy action to copy reg_a from WQE to reg_c[meta]
5648                  */
5649                 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) +
5650                            sizeof(struct mlx5_flow_action_copy_mreg);
5651                 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0,
5652                                           SOCKET_ID_ANY);
5653                 if (!ext_actions)
5654                         return rte_flow_error_set(error, ENOMEM,
5655                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5656                                                   NULL, "no memory to split "
5657                                                   "metadata flow");
5658                 /* Create the action list appended with copy register. */
5659                 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions,
5660                                              actions_n, error, encap_idx);
5661                 if (ret < 0)
5662                         goto exit;
5663         }
5664         /* Add the unmodified original or prefix subflow. */
5665         ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
5666                                       items, ext_actions ? ext_actions :
5667                                       actions, flow_split_info, error);
5668         if (ret < 0)
5669                 goto exit;
5670         MLX5_ASSERT(dev_flow);
5671         if (qrss) {
5672                 const struct rte_flow_attr q_attr = {
5673                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
5674                         .ingress = 1,
5675                 };
5676                 /* Internal PMD action to set register. */
5677                 struct mlx5_rte_flow_item_tag q_tag_spec = {
5678                         .data = qrss_id,
5679                         .id = REG_NON,
5680                 };
5681                 struct rte_flow_item q_items[] = {
5682                         {
5683                                 .type = (enum rte_flow_item_type)
5684                                         MLX5_RTE_FLOW_ITEM_TYPE_TAG,
5685                                 .spec = &q_tag_spec,
5686                                 .last = NULL,
5687                                 .mask = NULL,
5688                         },
5689                         {
5690                                 .type = RTE_FLOW_ITEM_TYPE_END,
5691                         },
5692                 };
5693                 struct rte_flow_action q_actions[] = {
5694                         {
5695                                 .type = qrss->type,
5696                                 .conf = qrss->conf,
5697                         },
5698                         {
5699                                 .type = RTE_FLOW_ACTION_TYPE_END,
5700                         },
5701                 };
5702                 uint64_t layers = flow_get_prefix_layer_flags(dev_flow);
5703
5704                 /*
5705                  * Configure the tag item only if there is no meter subflow.
5706                  * Since tag is already marked in the meter suffix subflow
5707                  * we can just use the meter suffix items as is.
5708                  */
5709                 if (qrss_id) {
5710                         /* Not meter subflow. */
5711                         MLX5_ASSERT(!mtr_sfx);
5712                         /*
5713                          * Put unique id in prefix flow due to it is destroyed
5714                          * after suffix flow and id will be freed after there
5715                          * is no actual flows with this id and identifier
5716                          * reallocation becomes possible (for example, for
5717                          * other flows in other threads).
5718                          */
5719                         dev_flow->handle->split_flow_id = qrss_id;
5720                         ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
5721                                                    error);
5722                         if (ret < 0)
5723                                 goto exit;
5724                         q_tag_spec.id = ret;
5725                 }
5726                 dev_flow = NULL;
5727                 /* Add suffix subflow to execute Q/RSS. */
5728                 flow_split_info->prefix_layers = layers;
5729                 flow_split_info->prefix_mark = 0;
5730                 ret = flow_create_split_inner(dev, flow, &dev_flow,
5731                                               &q_attr, mtr_sfx ? items :
5732                                               q_items, q_actions,
5733                                               flow_split_info, error);
5734                 if (ret < 0)
5735                         goto exit;
5736                 /* qrss ID should be freed if failed. */
5737                 qrss_id = 0;
5738                 MLX5_ASSERT(dev_flow);
5739         }
5740
5741 exit:
5742         /*
5743          * We do not destroy the partially created sub_flows in case of error.
5744          * These ones are included into parent flow list and will be destroyed
5745          * by flow_drv_destroy.
5746          */
5747         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
5748                         qrss_id);
5749         mlx5_free(ext_actions);
5750         return ret;
5751 }
5752
5753 /**
5754  * Create meter internal drop flow with the original pattern.
5755  *
5756  * @param dev
5757  *   Pointer to Ethernet device.
5758  * @param[in] flow
5759  *   Parent flow structure pointer.
5760  * @param[in] attr
5761  *   Flow rule attributes.
5762  * @param[in] items
5763  *   Pattern specification (list terminated by the END pattern item).
5764  * @param[in] flow_split_info
5765  *   Pointer to flow split info structure.
5766  * @param[in] fm
5767  *   Pointer to flow meter structure.
5768  * @param[out] error
5769  *   Perform verbose error reporting if not NULL.
5770  * @return
5771  *   0 on success, negative value otherwise
5772  */
5773 static uint32_t
5774 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev,
5775                         struct rte_flow *flow,
5776                         const struct rte_flow_attr *attr,
5777                         const struct rte_flow_item items[],
5778                         struct mlx5_flow_split_info *flow_split_info,
5779                         struct mlx5_flow_meter_info *fm,
5780                         struct rte_flow_error *error)
5781 {
5782         struct mlx5_flow *dev_flow = NULL;
5783         struct rte_flow_attr drop_attr = *attr;
5784         struct rte_flow_action drop_actions[3];
5785         struct mlx5_flow_split_info drop_split_info = *flow_split_info;
5786
5787         MLX5_ASSERT(fm->drop_cnt);
5788         drop_actions[0].type =
5789                 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT;
5790         drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt;
5791         drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP;
5792         drop_actions[1].conf = NULL;
5793         drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END;
5794         drop_actions[2].conf = NULL;
5795         drop_split_info.external = false;
5796         drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT;
5797         drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP;
5798         drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER;
5799         return flow_create_split_inner(dev, flow, &dev_flow,
5800                                 &drop_attr, items, drop_actions,
5801                                 &drop_split_info, error);
5802 }
5803
5804 /**
5805  * The splitting for meter feature.
5806  *
5807  * - The meter flow will be split to two flows as prefix and
5808  *   suffix flow. The packets make sense only it pass the prefix
5809  *   meter action.
5810  *
5811  * - Reg_C_5 is used for the packet to match betweend prefix and
5812  *   suffix flow.
5813  *
5814  * @param dev
5815  *   Pointer to Ethernet device.
5816  * @param[in] flow
5817  *   Parent flow structure pointer.
5818  * @param[in] attr
5819  *   Flow rule attributes.
5820  * @param[in] items
5821  *   Pattern specification (list terminated by the END pattern item).
5822  * @param[in] actions
5823  *   Associated actions (list terminated by the END action).
5824  * @param[in] flow_split_info
5825  *   Pointer to flow split info structure.
5826  * @param[out] error
5827  *   Perform verbose error reporting if not NULL.
5828  * @return
5829  *   0 on success, negative value otherwise
5830  */
5831 static int
5832 flow_create_split_meter(struct rte_eth_dev *dev,
5833                         struct rte_flow *flow,
5834                         const struct rte_flow_attr *attr,
5835                         const struct rte_flow_item items[],
5836                         const struct rte_flow_action actions[],
5837                         struct mlx5_flow_split_info *flow_split_info,
5838                         struct rte_flow_error *error)
5839 {
5840         struct mlx5_priv *priv = dev->data->dev_private;
5841         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
5842         struct rte_flow_action *sfx_actions = NULL;
5843         struct rte_flow_action *pre_actions = NULL;
5844         struct rte_flow_item *sfx_items = NULL;
5845         struct mlx5_flow *dev_flow = NULL;
5846         struct rte_flow_attr sfx_attr = *attr;
5847         struct mlx5_flow_meter_info *fm = NULL;
5848         uint8_t skip_scale_restore;
5849         bool has_mtr = false;
5850         bool has_modify = false;
5851         bool set_mtr_reg = true;
5852         bool is_mtr_hierarchy = false;
5853         uint32_t meter_id = 0;
5854         uint32_t mtr_idx = 0;
5855         uint32_t mtr_flow_id = 0;
5856         size_t act_size;
5857         size_t item_size;
5858         int actions_n = 0;
5859         int ret = 0;
5860
5861         if (priv->mtr_en)
5862                 actions_n = flow_check_meter_action(dev, actions, &has_mtr,
5863                                                     &has_modify, &meter_id);
5864         if (has_mtr) {
5865                 if (flow->meter) {
5866                         fm = flow_dv_meter_find_by_idx(priv, flow->meter);
5867                         if (!fm)
5868                                 return rte_flow_error_set(error, EINVAL,
5869                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5870                                                 NULL, "Meter not found.");
5871                 } else {
5872                         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
5873                         if (!fm)
5874                                 return rte_flow_error_set(error, EINVAL,
5875                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5876                                                 NULL, "Meter not found.");
5877                         ret = mlx5_flow_meter_attach(priv, fm,
5878                                                      &sfx_attr, error);
5879                         if (ret)
5880                                 return -rte_errno;
5881                         flow->meter = mtr_idx;
5882                 }
5883                 MLX5_ASSERT(wks);
5884                 wks->fm = fm;
5885                 if (!fm->def_policy) {
5886                         wks->policy = mlx5_flow_meter_policy_find(dev,
5887                                                                   fm->policy_id,
5888                                                                   NULL);
5889                         MLX5_ASSERT(wks->policy);
5890                         if (wks->policy->is_hierarchy) {
5891                                 wks->final_policy =
5892                                 mlx5_flow_meter_hierarchy_get_final_policy(dev,
5893                                                                 wks->policy);
5894                                 if (!wks->final_policy)
5895                                         return rte_flow_error_set(error,
5896                                         EINVAL,
5897                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5898                                 "Failed to find terminal policy of hierarchy.");
5899                                 is_mtr_hierarchy = true;
5900                         }
5901                 }
5902                 /*
5903                  * If it isn't default-policy Meter, and
5904                  * 1. There's no action in flow to change
5905                  *    packet (modify/encap/decap etc.), OR
5906                  * 2. No drop count needed for this meter.
5907                  * 3. It's not meter hierarchy.
5908                  * Then no need to use regC to save meter id anymore.
5909                  */
5910                 if (!fm->def_policy && !is_mtr_hierarchy &&
5911                     (!has_modify || !fm->drop_cnt))
5912                         set_mtr_reg = false;
5913                 /* Prefix actions: meter, decap, encap, tag, jump, end. */
5914                 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) +
5915                            sizeof(struct mlx5_rte_flow_action_set_tag);
5916                 /* Suffix items: tag, vlan, port id, end. */
5917 #define METER_SUFFIX_ITEM 4
5918                 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
5919                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
5920                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size),
5921                                           0, SOCKET_ID_ANY);
5922                 if (!sfx_actions)
5923                         return rte_flow_error_set(error, ENOMEM,
5924                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5925                                                   NULL, "no memory to split "
5926                                                   "meter flow");
5927                 sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
5928                              act_size);
5929                 /* There's no suffix flow for meter of non-default policy. */
5930                 if (!fm->def_policy)
5931                         pre_actions = sfx_actions + 1;
5932                 else
5933                         pre_actions = sfx_actions + actions_n;
5934                 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr,
5935                                             items, sfx_items, actions,
5936                                             sfx_actions, pre_actions,
5937                                             (set_mtr_reg ? &mtr_flow_id : NULL),
5938                                             error);
5939                 if (ret) {
5940                         ret = -rte_errno;
5941                         goto exit;
5942                 }
5943                 /* Add the prefix subflow. */
5944                 flow_split_info->prefix_mark = 0;
5945                 skip_scale_restore = flow_split_info->skip_scale;
5946                 flow_split_info->skip_scale |=
5947                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
5948                 ret = flow_create_split_inner(dev, flow, &dev_flow,
5949                                               attr, items, pre_actions,
5950                                               flow_split_info, error);
5951                 flow_split_info->skip_scale = skip_scale_restore;
5952                 if (ret) {
5953                         if (mtr_flow_id)
5954                                 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id);
5955                         ret = -rte_errno;
5956                         goto exit;
5957                 }
5958                 if (mtr_flow_id) {
5959                         dev_flow->handle->split_flow_id = mtr_flow_id;
5960                         dev_flow->handle->is_meter_flow_id = 1;
5961                 }
5962                 if (!fm->def_policy) {
5963                         if (!set_mtr_reg && fm->drop_cnt)
5964                                 ret =
5965                         flow_meter_create_drop_flow_with_org_pattern(dev, flow,
5966                                                         &sfx_attr, items,
5967                                                         flow_split_info,
5968                                                         fm, error);
5969                         goto exit;
5970                 }
5971                 /* Setting the sfx group atrr. */
5972                 sfx_attr.group = sfx_attr.transfer ?
5973                                 (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
5974                                  MLX5_FLOW_TABLE_LEVEL_METER;
5975                 flow_split_info->prefix_layers =
5976                                 flow_get_prefix_layer_flags(dev_flow);
5977                 flow_split_info->prefix_mark = dev_flow->handle->mark;
5978                 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX;
5979         }
5980         /* Add the prefix subflow. */
5981         ret = flow_create_split_metadata(dev, flow,
5982                                          &sfx_attr, sfx_items ?
5983                                          sfx_items : items,
5984                                          sfx_actions ? sfx_actions : actions,
5985                                          flow_split_info, error);
5986 exit:
5987         if (sfx_actions)
5988                 mlx5_free(sfx_actions);
5989         return ret;
5990 }
5991
5992 /**
5993  * The splitting for sample feature.
5994  *
5995  * Once Sample action is detected in the action list, the flow actions should
5996  * be split into prefix sub flow and suffix sub flow.
5997  *
5998  * The original items remain in the prefix sub flow, all actions preceding the
5999  * sample action and the sample action itself will be copied to the prefix
6000  * sub flow, the actions following the sample action will be copied to the
6001  * suffix sub flow, Queue action always be located in the suffix sub flow.
6002  *
6003  * In order to make the packet from prefix sub flow matches with suffix sub
6004  * flow, an extra tag action be added into prefix sub flow, and the suffix sub
6005  * flow uses tag item with the unique flow id.
6006  *
6007  * @param dev
6008  *   Pointer to Ethernet device.
6009  * @param[in] flow
6010  *   Parent flow structure pointer.
6011  * @param[in] attr
6012  *   Flow rule attributes.
6013  * @param[in] items
6014  *   Pattern specification (list terminated by the END pattern item).
6015  * @param[in] actions
6016  *   Associated actions (list terminated by the END action).
6017  * @param[in] flow_split_info
6018  *   Pointer to flow split info structure.
6019  * @param[out] error
6020  *   Perform verbose error reporting if not NULL.
6021  * @return
6022  *   0 on success, negative value otherwise
6023  */
6024 static int
6025 flow_create_split_sample(struct rte_eth_dev *dev,
6026                          struct rte_flow *flow,
6027                          const struct rte_flow_attr *attr,
6028                          const struct rte_flow_item items[],
6029                          const struct rte_flow_action actions[],
6030                          struct mlx5_flow_split_info *flow_split_info,
6031                          struct rte_flow_error *error)
6032 {
6033         struct mlx5_priv *priv = dev->data->dev_private;
6034         struct rte_flow_action *sfx_actions = NULL;
6035         struct rte_flow_action *pre_actions = NULL;
6036         struct rte_flow_item *sfx_items = NULL;
6037         struct mlx5_flow *dev_flow = NULL;
6038         struct rte_flow_attr sfx_attr = *attr;
6039 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
6040         struct mlx5_flow_dv_sample_resource *sample_res;
6041         struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
6042         struct mlx5_flow_tbl_resource *sfx_tbl;
6043 #endif
6044         size_t act_size;
6045         size_t item_size;
6046         uint32_t fdb_tx = 0;
6047         int32_t tag_id = 0;
6048         int actions_n = 0;
6049         int sample_action_pos;
6050         int qrss_action_pos;
6051         int add_tag = 0;
6052         int modify_after_mirror = 0;
6053         uint16_t jump_table = 0;
6054         const uint32_t next_ft_step = 1;
6055         int ret = 0;
6056
6057         if (priv->sampler_en)
6058                 actions_n = flow_check_match_action(actions, attr,
6059                                         RTE_FLOW_ACTION_TYPE_SAMPLE,
6060                                         &sample_action_pos, &qrss_action_pos,
6061                                         &modify_after_mirror);
6062         if (actions_n) {
6063                 /* The prefix actions must includes sample, tag, end. */
6064                 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
6065                            + sizeof(struct mlx5_rte_flow_action_set_tag);
6066                 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
6067                             sizeof(struct mlx5_rte_flow_item_tag) * 2;
6068                 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size +
6069                                           item_size), 0, SOCKET_ID_ANY);
6070                 if (!sfx_actions)
6071                         return rte_flow_error_set(error, ENOMEM,
6072                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6073                                                   NULL, "no memory to split "
6074                                                   "sample flow");
6075                 /* The representor_id is UINT16_MAX for uplink. */
6076                 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX);
6077                 /*
6078                  * When reg_c_preserve is set, metadata registers Cx preserve
6079                  * their value even through packet duplication.
6080                  */
6081                 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve);
6082                 if (add_tag)
6083                         sfx_items = (struct rte_flow_item *)((char *)sfx_actions
6084                                         + act_size);
6085                 if (modify_after_mirror)
6086                         jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR +
6087                                      next_ft_step;
6088                 pre_actions = sfx_actions + actions_n;
6089                 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items,
6090                                                 actions, sfx_actions,
6091                                                 pre_actions, actions_n,
6092                                                 sample_action_pos,
6093                                                 qrss_action_pos, jump_table,
6094                                                 error);
6095                 if (tag_id < 0 || (add_tag && !tag_id)) {
6096                         ret = -rte_errno;
6097                         goto exit;
6098                 }
6099                 if (modify_after_mirror)
6100                         flow_split_info->skip_scale =
6101                                         1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT;
6102                 /* Add the prefix subflow. */
6103                 ret = flow_create_split_inner(dev, flow, &dev_flow, attr,
6104                                               items, pre_actions,
6105                                               flow_split_info, error);
6106                 if (ret) {
6107                         ret = -rte_errno;
6108                         goto exit;
6109                 }
6110                 dev_flow->handle->split_flow_id = tag_id;
6111 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
6112                 if (!modify_after_mirror) {
6113                         /* Set the sfx group attr. */
6114                         sample_res = (struct mlx5_flow_dv_sample_resource *)
6115                                                 dev_flow->dv.sample_res;
6116                         sfx_tbl = (struct mlx5_flow_tbl_resource *)
6117                                                 sample_res->normal_path_tbl;
6118                         sfx_tbl_data = container_of(sfx_tbl,
6119                                                 struct mlx5_flow_tbl_data_entry,
6120                                                 tbl);
6121                         sfx_attr.group = sfx_attr.transfer ?
6122                         (sfx_tbl_data->level - 1) : sfx_tbl_data->level;
6123                 } else {
6124                         MLX5_ASSERT(attr->transfer);
6125                         sfx_attr.group = jump_table;
6126                 }
6127                 flow_split_info->prefix_layers =
6128                                 flow_get_prefix_layer_flags(dev_flow);
6129                 flow_split_info->prefix_mark = dev_flow->handle->mark;
6130                 /* Suffix group level already be scaled with factor, set
6131                  * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale
6132                  * again in translation.
6133                  */
6134                 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT;
6135 #endif
6136         }
6137         /* Add the suffix subflow. */
6138         ret = flow_create_split_meter(dev, flow, &sfx_attr,
6139                                       sfx_items ? sfx_items : items,
6140                                       sfx_actions ? sfx_actions : actions,
6141                                       flow_split_info, error);
6142 exit:
6143         if (sfx_actions)
6144                 mlx5_free(sfx_actions);
6145         return ret;
6146 }
6147
6148 /**
6149  * Split the flow to subflow set. The splitters might be linked
6150  * in the chain, like this:
6151  * flow_create_split_outer() calls:
6152  *   flow_create_split_meter() calls:
6153  *     flow_create_split_metadata(meter_subflow_0) calls:
6154  *       flow_create_split_inner(metadata_subflow_0)
6155  *       flow_create_split_inner(metadata_subflow_1)
6156  *       flow_create_split_inner(metadata_subflow_2)
6157  *     flow_create_split_metadata(meter_subflow_1) calls:
6158  *       flow_create_split_inner(metadata_subflow_0)
6159  *       flow_create_split_inner(metadata_subflow_1)
6160  *       flow_create_split_inner(metadata_subflow_2)
6161  *
6162  * This provide flexible way to add new levels of flow splitting.
6163  * The all of successfully created subflows are included to the
6164  * parent flow dev_flow list.
6165  *
6166  * @param dev
6167  *   Pointer to Ethernet device.
6168  * @param[in] flow
6169  *   Parent flow structure pointer.
6170  * @param[in] attr
6171  *   Flow rule attributes.
6172  * @param[in] items
6173  *   Pattern specification (list terminated by the END pattern item).
6174  * @param[in] actions
6175  *   Associated actions (list terminated by the END action).
6176  * @param[in] flow_split_info
6177  *   Pointer to flow split info structure.
6178  * @param[out] error
6179  *   Perform verbose error reporting if not NULL.
6180  * @return
6181  *   0 on success, negative value otherwise
6182  */
6183 static int
6184 flow_create_split_outer(struct rte_eth_dev *dev,
6185                         struct rte_flow *flow,
6186                         const struct rte_flow_attr *attr,
6187                         const struct rte_flow_item items[],
6188                         const struct rte_flow_action actions[],
6189                         struct mlx5_flow_split_info *flow_split_info,
6190                         struct rte_flow_error *error)
6191 {
6192         int ret;
6193
6194         ret = flow_create_split_sample(dev, flow, attr, items,
6195                                        actions, flow_split_info, error);
6196         MLX5_ASSERT(ret <= 0);
6197         return ret;
6198 }
6199
6200 static inline struct mlx5_flow_tunnel *
6201 flow_tunnel_from_rule(const struct mlx5_flow *flow)
6202 {
6203         struct mlx5_flow_tunnel *tunnel;
6204
6205 #pragma GCC diagnostic push
6206 #pragma GCC diagnostic ignored "-Wcast-qual"
6207         tunnel = (typeof(tunnel))flow->tunnel;
6208 #pragma GCC diagnostic pop
6209
6210         return tunnel;
6211 }
6212
6213 /**
6214  * Adjust flow RSS workspace if needed.
6215  *
6216  * @param wks
6217  *   Pointer to thread flow work space.
6218  * @param rss_desc
6219  *   Pointer to RSS descriptor.
6220  * @param[in] nrssq_num
6221  *   New RSS queue number.
6222  *
6223  * @return
6224  *   0 on success, -1 otherwise and rte_errno is set.
6225  */
6226 static int
6227 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
6228                           struct mlx5_flow_rss_desc *rss_desc,
6229                           uint32_t nrssq_num)
6230 {
6231         if (likely(nrssq_num <= wks->rssq_num))
6232                 return 0;
6233         rss_desc->queue = realloc(rss_desc->queue,
6234                           sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2));
6235         if (!rss_desc->queue) {
6236                 rte_errno = ENOMEM;
6237                 return -1;
6238         }
6239         wks->rssq_num = RTE_ALIGN(nrssq_num, 2);
6240         return 0;
6241 }
6242
6243 /**
6244  * Create a flow and add it to @p list.
6245  *
6246  * @param dev
6247  *   Pointer to Ethernet device.
6248  * @param list
6249  *   Pointer to a TAILQ flow list. If this parameter NULL,
6250  *   no list insertion occurred, flow is just created,
6251  *   this is caller's responsibility to track the
6252  *   created flow.
6253  * @param[in] attr
6254  *   Flow rule attributes.
6255  * @param[in] items
6256  *   Pattern specification (list terminated by the END pattern item).
6257  * @param[in] actions
6258  *   Associated actions (list terminated by the END action).
6259  * @param[in] external
6260  *   This flow rule is created by request external to PMD.
6261  * @param[out] error
6262  *   Perform verbose error reporting if not NULL.
6263  *
6264  * @return
6265  *   A flow index on success, 0 otherwise and rte_errno is set.
6266  */
6267 static uint32_t
6268 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
6269                  const struct rte_flow_attr *attr,
6270                  const struct rte_flow_item items[],
6271                  const struct rte_flow_action original_actions[],
6272                  bool external, struct rte_flow_error *error)
6273 {
6274         struct mlx5_priv *priv = dev->data->dev_private;
6275         struct rte_flow *flow = NULL;
6276         struct mlx5_flow *dev_flow;
6277         const struct rte_flow_action_rss *rss = NULL;
6278         struct mlx5_translated_action_handle
6279                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
6280         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
6281         union {
6282                 struct mlx5_flow_expand_rss buf;
6283                 uint8_t buffer[4096];
6284         } expand_buffer;
6285         union {
6286                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6287                 uint8_t buffer[2048];
6288         } actions_rx;
6289         union {
6290                 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
6291                 uint8_t buffer[2048];
6292         } actions_hairpin_tx;
6293         union {
6294                 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS];
6295                 uint8_t buffer[2048];
6296         } items_tx;
6297         struct mlx5_flow_expand_rss *buf = &expand_buffer.buf;
6298         struct mlx5_flow_rss_desc *rss_desc;
6299         const struct rte_flow_action *p_actions_rx;
6300         uint32_t i;
6301         uint32_t idx = 0;
6302         int hairpin_flow;
6303         struct rte_flow_attr attr_tx = { .priority = 0 };
6304         const struct rte_flow_action *actions;
6305         struct rte_flow_action *translated_actions = NULL;
6306         struct mlx5_flow_tunnel *tunnel;
6307         struct tunnel_default_miss_ctx default_miss_ctx = { 0, };
6308         struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace();
6309         struct mlx5_flow_split_info flow_split_info = {
6310                 .external = !!external,
6311                 .skip_scale = 0,
6312                 .flow_idx = 0,
6313                 .prefix_mark = 0,
6314                 .prefix_layers = 0,
6315                 .table_id = 0
6316         };
6317         int ret;
6318
6319         MLX5_ASSERT(wks);
6320         rss_desc = &wks->rss_desc;
6321         ret = flow_action_handles_translate(dev, original_actions,
6322                                             indir_actions,
6323                                             &indir_actions_n,
6324                                             &translated_actions, error);
6325         if (ret < 0) {
6326                 MLX5_ASSERT(translated_actions == NULL);
6327                 return 0;
6328         }
6329         actions = translated_actions ? translated_actions : original_actions;
6330         p_actions_rx = actions;
6331         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
6332         ret = flow_drv_validate(dev, attr, items, p_actions_rx,
6333                                 external, hairpin_flow, error);
6334         if (ret < 0)
6335                 goto error_before_hairpin_split;
6336         flow = mlx5_ipool_zmalloc(priv->flows[type], &idx);
6337         if (!flow) {
6338                 rte_errno = ENOMEM;
6339                 goto error_before_hairpin_split;
6340         }
6341         if (hairpin_flow > 0) {
6342                 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
6343                         rte_errno = EINVAL;
6344                         goto error_before_hairpin_split;
6345                 }
6346                 flow_hairpin_split(dev, actions, actions_rx.actions,
6347                                    actions_hairpin_tx.actions, items_tx.items,
6348                                    idx);
6349                 p_actions_rx = actions_rx.actions;
6350         }
6351         flow_split_info.flow_idx = idx;
6352         flow->drv_type = flow_get_drv_type(dev, attr);
6353         MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
6354                     flow->drv_type < MLX5_FLOW_TYPE_MAX);
6355         memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
6356         /* RSS Action only works on NIC RX domain */
6357         if (attr->ingress && !attr->transfer)
6358                 rss = flow_get_rss_action(dev, p_actions_rx);
6359         if (rss) {
6360                 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num))
6361                         return 0;
6362                 /*
6363                  * The following information is required by
6364                  * mlx5_flow_hashfields_adjust() in advance.
6365                  */
6366                 rss_desc->level = rss->level;
6367                 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
6368                 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types;
6369         }
6370         flow->dev_handles = 0;
6371         if (rss && rss->types) {
6372                 unsigned int graph_root;
6373
6374                 graph_root = find_graph_root(rss->level);
6375                 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
6376                                            items, rss->types,
6377                                            mlx5_support_expansion, graph_root);
6378                 MLX5_ASSERT(ret > 0 &&
6379                        (unsigned int)ret < sizeof(expand_buffer.buffer));
6380                 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) {
6381                         for (i = 0; i < buf->entries; ++i)
6382                                 mlx5_dbg__print_pattern(buf->entry[i].pattern);
6383                 }
6384         } else {
6385                 buf->entries = 1;
6386                 buf->entry[0].pattern = (void *)(uintptr_t)items;
6387         }
6388         rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions,
6389                                                       indir_actions_n);
6390         for (i = 0; i < buf->entries; ++i) {
6391                 /* Initialize flow split data. */
6392                 flow_split_info.prefix_layers = 0;
6393                 flow_split_info.prefix_mark = 0;
6394                 flow_split_info.skip_scale = 0;
6395                 /*
6396                  * The splitter may create multiple dev_flows,
6397                  * depending on configuration. In the simplest
6398                  * case it just creates unmodified original flow.
6399                  */
6400                 ret = flow_create_split_outer(dev, flow, attr,
6401                                               buf->entry[i].pattern,
6402                                               p_actions_rx, &flow_split_info,
6403                                               error);
6404                 if (ret < 0)
6405                         goto error;
6406                 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) {
6407                         ret = flow_tunnel_add_default_miss(dev, flow, attr,
6408                                                            p_actions_rx,
6409                                                            idx,
6410                                                            wks->flows[0].tunnel,
6411                                                            &default_miss_ctx,
6412                                                            error);
6413                         if (ret < 0) {
6414                                 mlx5_free(default_miss_ctx.queue);
6415                                 goto error;
6416                         }
6417                 }
6418         }
6419         /* Create the tx flow. */
6420         if (hairpin_flow) {
6421                 attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
6422                 attr_tx.ingress = 0;
6423                 attr_tx.egress = 1;
6424                 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
6425                                          actions_hairpin_tx.actions,
6426                                          idx, error);
6427                 if (!dev_flow)
6428                         goto error;
6429                 dev_flow->flow = flow;
6430                 dev_flow->external = 0;
6431                 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
6432                               dev_flow->handle, next);
6433                 ret = flow_drv_translate(dev, dev_flow, &attr_tx,
6434                                          items_tx.items,
6435                                          actions_hairpin_tx.actions, error);
6436                 if (ret < 0)
6437                         goto error;
6438         }
6439         /*
6440          * Update the metadata register copy table. If extensive
6441          * metadata feature is enabled and registers are supported
6442          * we might create the extra rte_flow for each unique
6443          * MARK/FLAG action ID.
6444          *
6445          * The table is updated for ingress Flows only, because
6446          * the egress Flows belong to the different device and
6447          * copy table should be updated in peer NIC Rx domain.
6448          */
6449         if (attr->ingress &&
6450             (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) {
6451                 ret = flow_mreg_update_copy_table(dev, flow, actions, error);
6452                 if (ret)
6453                         goto error;
6454         }
6455         /*
6456          * If the flow is external (from application) OR device is started,
6457          * OR mreg discover, then apply immediately.
6458          */
6459         if (external || dev->data->dev_started ||
6460             (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP &&
6461              attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) {
6462                 ret = flow_drv_apply(dev, flow, error);
6463                 if (ret < 0)
6464                         goto error;
6465         }
6466         flow->type = type;
6467         flow_rxq_flags_set(dev, flow);
6468         rte_free(translated_actions);
6469         tunnel = flow_tunnel_from_rule(wks->flows);
6470         if (tunnel) {
6471                 flow->tunnel = 1;
6472                 flow->tunnel_id = tunnel->tunnel_id;
6473                 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED);
6474                 mlx5_free(default_miss_ctx.queue);
6475         }
6476         mlx5_flow_pop_thread_workspace();
6477         return idx;
6478 error:
6479         MLX5_ASSERT(flow);
6480         ret = rte_errno; /* Save rte_errno before cleanup. */
6481         flow_mreg_del_copy_action(dev, flow);
6482         flow_drv_destroy(dev, flow);
6483         if (rss_desc->shared_rss)
6484                 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *)
6485                         mlx5_ipool_get
6486                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
6487                         rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
6488         mlx5_ipool_free(priv->flows[type], idx);
6489         rte_errno = ret; /* Restore rte_errno. */
6490         ret = rte_errno;
6491         rte_errno = ret;
6492         mlx5_flow_pop_thread_workspace();
6493 error_before_hairpin_split:
6494         rte_free(translated_actions);
6495         return 0;
6496 }
6497
6498 /**
6499  * Create a dedicated flow rule on e-switch table 0 (root table), to direct all
6500  * incoming packets to table 1.
6501  *
6502  * Other flow rules, requested for group n, will be created in
6503  * e-switch table n+1.
6504  * Jump action to e-switch group n will be created to group n+1.
6505  *
6506  * Used when working in switchdev mode, to utilise advantages of table 1
6507  * and above.
6508  *
6509  * @param dev
6510  *   Pointer to Ethernet device.
6511  *
6512  * @return
6513  *   Pointer to flow on success, NULL otherwise and rte_errno is set.
6514  */
6515 struct rte_flow *
6516 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
6517 {
6518         const struct rte_flow_attr attr = {
6519                 .group = 0,
6520                 .priority = 0,
6521                 .ingress = 1,
6522                 .egress = 0,
6523                 .transfer = 1,
6524         };
6525         const struct rte_flow_item pattern = {
6526                 .type = RTE_FLOW_ITEM_TYPE_END,
6527         };
6528         struct rte_flow_action_jump jump = {
6529                 .group = 1,
6530         };
6531         const struct rte_flow_action actions[] = {
6532                 {
6533                         .type = RTE_FLOW_ACTION_TYPE_JUMP,
6534                         .conf = &jump,
6535                 },
6536                 {
6537                         .type = RTE_FLOW_ACTION_TYPE_END,
6538                 },
6539         };
6540         struct rte_flow_error error;
6541
6542         return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
6543                                                    &attr, &pattern,
6544                                                    actions, false, &error);
6545 }
6546
6547 /**
6548  * Validate a flow supported by the NIC.
6549  *
6550  * @see rte_flow_validate()
6551  * @see rte_flow_ops
6552  */
6553 int
6554 mlx5_flow_validate(struct rte_eth_dev *dev,
6555                    const struct rte_flow_attr *attr,
6556                    const struct rte_flow_item items[],
6557                    const struct rte_flow_action original_actions[],
6558                    struct rte_flow_error *error)
6559 {
6560         int hairpin_flow;
6561         struct mlx5_translated_action_handle
6562                 indir_actions[MLX5_MAX_INDIRECT_ACTIONS];
6563         int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS;
6564         const struct rte_flow_action *actions;
6565         struct rte_flow_action *translated_actions = NULL;
6566         int ret = flow_action_handles_translate(dev, original_actions,
6567                                                 indir_actions,
6568                                                 &indir_actions_n,
6569                                                 &translated_actions, error);
6570
6571         if (ret)
6572                 return ret;
6573         actions = translated_actions ? translated_actions : original_actions;
6574         hairpin_flow = flow_check_hairpin_split(dev, attr, actions);
6575         ret = flow_drv_validate(dev, attr, items, actions,
6576                                 true, hairpin_flow, error);
6577         rte_free(translated_actions);
6578         return ret;
6579 }
6580
6581 /**
6582  * Create a flow.
6583  *
6584  * @see rte_flow_create()
6585  * @see rte_flow_ops
6586  */
6587 struct rte_flow *
6588 mlx5_flow_create(struct rte_eth_dev *dev,
6589                  const struct rte_flow_attr *attr,
6590                  const struct rte_flow_item items[],
6591                  const struct rte_flow_action actions[],
6592                  struct rte_flow_error *error)
6593 {
6594         /*
6595          * If the device is not started yet, it is not allowed to created a
6596          * flow from application. PMD default flows and traffic control flows
6597          * are not affected.
6598          */
6599         if (unlikely(!dev->data->dev_started)) {
6600                 DRV_LOG(DEBUG, "port %u is not started when "
6601                         "inserting a flow", dev->data->port_id);
6602                 rte_flow_error_set(error, ENODEV,
6603                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6604                                    NULL,
6605                                    "port not started");
6606                 return NULL;
6607         }
6608
6609         return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN,
6610                                                    attr, items, actions,
6611                                                    true, error);
6612 }
6613
6614 /**
6615  * Destroy a flow in a list.
6616  *
6617  * @param dev
6618  *   Pointer to Ethernet device.
6619  * @param[in] flow_idx
6620  *   Index of flow to destroy.
6621  */
6622 static void
6623 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
6624                   uint32_t flow_idx)
6625 {
6626         struct mlx5_priv *priv = dev->data->dev_private;
6627         struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx);
6628
6629         if (!flow)
6630                 return;
6631         MLX5_ASSERT(flow->type == type);
6632         /*
6633          * Update RX queue flags only if port is started, otherwise it is
6634          * already clean.
6635          */
6636         if (dev->data->dev_started)
6637                 flow_rxq_flags_trim(dev, flow);
6638         flow_drv_destroy(dev, flow);
6639         if (flow->tunnel) {
6640                 struct mlx5_flow_tunnel *tunnel;
6641
6642                 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
6643                 RTE_VERIFY(tunnel);
6644                 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
6645                         mlx5_flow_tunnel_free(dev, tunnel);
6646         }
6647         flow_mreg_del_copy_action(dev, flow);
6648         mlx5_ipool_free(priv->flows[type], flow_idx);
6649 }
6650
6651 /**
6652  * Destroy all flows.
6653  *
6654  * @param dev
6655  *   Pointer to Ethernet device.
6656  * @param type
6657  *   Flow type to be flushed.
6658  * @param active
6659  *   If flushing is called avtively.
6660  */
6661 void
6662 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
6663                      bool active)
6664 {
6665         struct mlx5_priv *priv = dev->data->dev_private;
6666         uint32_t num_flushed = 0, fidx = 1;
6667         struct rte_flow *flow;
6668
6669         MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) {
6670                 flow_list_destroy(dev, type, fidx);
6671                 num_flushed++;
6672         }
6673         if (active) {
6674                 DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
6675                         dev->data->port_id, num_flushed);
6676         }
6677 }
6678
6679 /**
6680  * Stop all default actions for flows.
6681  *
6682  * @param dev
6683  *   Pointer to Ethernet device.
6684  */
6685 void
6686 mlx5_flow_stop_default(struct rte_eth_dev *dev)
6687 {
6688         flow_mreg_del_default_copy_action(dev);
6689         flow_rxq_flags_clear(dev);
6690 }
6691
6692 /**
6693  * Start all default actions for flows.
6694  *
6695  * @param dev
6696  *   Pointer to Ethernet device.
6697  * @return
6698  *   0 on success, a negative errno value otherwise and rte_errno is set.
6699  */
6700 int
6701 mlx5_flow_start_default(struct rte_eth_dev *dev)
6702 {
6703         struct rte_flow_error error;
6704
6705         /* Make sure default copy action (reg_c[0] -> reg_b) is created. */
6706         return flow_mreg_add_default_copy_action(dev, &error);
6707 }
6708
6709 /**
6710  * Release key of thread specific flow workspace data.
6711  */
6712 void
6713 flow_release_workspace(void *data)
6714 {
6715         struct mlx5_flow_workspace *wks = data;
6716         struct mlx5_flow_workspace *next;
6717
6718         while (wks) {
6719                 next = wks->next;
6720                 free(wks->rss_desc.queue);
6721                 free(wks);
6722                 wks = next;
6723         }
6724 }
6725
6726 /**
6727  * Get thread specific current flow workspace.
6728  *
6729  * @return pointer to thread specific flow workspace data, NULL on error.
6730  */
6731 struct mlx5_flow_workspace*
6732 mlx5_flow_get_thread_workspace(void)
6733 {
6734         struct mlx5_flow_workspace *data;
6735
6736         data = mlx5_flow_os_get_specific_workspace();
6737         MLX5_ASSERT(data && data->inuse);
6738         if (!data || !data->inuse)
6739                 DRV_LOG(ERR, "flow workspace not initialized.");
6740         return data;
6741 }
6742
6743 /**
6744  * Allocate and init new flow workspace.
6745  *
6746  * @return pointer to flow workspace data, NULL on error.
6747  */
6748 static struct mlx5_flow_workspace*
6749 flow_alloc_thread_workspace(void)
6750 {
6751         struct mlx5_flow_workspace *data = calloc(1, sizeof(*data));
6752
6753         if (!data) {
6754                 DRV_LOG(ERR, "Failed to allocate flow workspace "
6755                         "memory.");
6756                 return NULL;
6757         }
6758         data->rss_desc.queue = calloc(1,
6759                         sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM);
6760         if (!data->rss_desc.queue)
6761                 goto err;
6762         data->rssq_num = MLX5_RSSQ_DEFAULT_NUM;
6763         return data;
6764 err:
6765         if (data->rss_desc.queue)
6766                 free(data->rss_desc.queue);
6767         free(data);
6768         return NULL;
6769 }
6770
6771 /**
6772  * Get new thread specific flow workspace.
6773  *
6774  * If current workspace inuse, create new one and set as current.
6775  *
6776  * @return pointer to thread specific flow workspace data, NULL on error.
6777  */
6778 static struct mlx5_flow_workspace*
6779 mlx5_flow_push_thread_workspace(void)
6780 {
6781         struct mlx5_flow_workspace *curr;
6782         struct mlx5_flow_workspace *data;
6783
6784         curr = mlx5_flow_os_get_specific_workspace();
6785         if (!curr) {
6786                 data = flow_alloc_thread_workspace();
6787                 if (!data)
6788                         return NULL;
6789         } else if (!curr->inuse) {
6790                 data = curr;
6791         } else if (curr->next) {
6792                 data = curr->next;
6793         } else {
6794                 data = flow_alloc_thread_workspace();
6795                 if (!data)
6796                         return NULL;
6797                 curr->next = data;
6798                 data->prev = curr;
6799         }
6800         data->inuse = 1;
6801         data->flow_idx = 0;
6802         /* Set as current workspace */
6803         if (mlx5_flow_os_set_specific_workspace(data))
6804                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
6805         return data;
6806 }
6807
6808 /**
6809  * Close current thread specific flow workspace.
6810  *
6811  * If previous workspace available, set it as current.
6812  *
6813  * @return pointer to thread specific flow workspace data, NULL on error.
6814  */
6815 static void
6816 mlx5_flow_pop_thread_workspace(void)
6817 {
6818         struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace();
6819
6820         if (!data)
6821                 return;
6822         if (!data->inuse) {
6823                 DRV_LOG(ERR, "Failed to close unused flow workspace.");
6824                 return;
6825         }
6826         data->inuse = 0;
6827         if (!data->prev)
6828                 return;
6829         if (mlx5_flow_os_set_specific_workspace(data->prev))
6830                 DRV_LOG(ERR, "Failed to set flow workspace to thread.");
6831 }
6832
6833 /**
6834  * Verify the flow list is empty
6835  *
6836  * @param dev
6837  *  Pointer to Ethernet device.
6838  *
6839  * @return the number of flows not released.
6840  */
6841 int
6842 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused)
6843 {
6844         struct mlx5_priv *priv = dev->data->dev_private;
6845         struct rte_flow *flow;
6846         uint32_t idx = 0;
6847         int ret = 0, i;
6848
6849         for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
6850                 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) {
6851                         DRV_LOG(DEBUG, "port %u flow %p still referenced",
6852                                 dev->data->port_id, (void *)flow);
6853                         ret++;
6854                 }
6855         }
6856         return ret;
6857 }
6858
6859 /**
6860  * Enable default hairpin egress flow.
6861  *
6862  * @param dev
6863  *   Pointer to Ethernet device.
6864  * @param queue
6865  *   The queue index.
6866  *
6867  * @return
6868  *   0 on success, a negative errno value otherwise and rte_errno is set.
6869  */
6870 int
6871 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
6872                             uint32_t queue)
6873 {
6874         const struct rte_flow_attr attr = {
6875                 .egress = 1,
6876                 .priority = 0,
6877         };
6878         struct mlx5_rte_flow_item_tx_queue queue_spec = {
6879                 .queue = queue,
6880         };
6881         struct mlx5_rte_flow_item_tx_queue queue_mask = {
6882                 .queue = UINT32_MAX,
6883         };
6884         struct rte_flow_item items[] = {
6885                 {
6886                         .type = (enum rte_flow_item_type)
6887                                 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
6888                         .spec = &queue_spec,
6889                         .last = NULL,
6890                         .mask = &queue_mask,
6891                 },
6892                 {
6893                         .type = RTE_FLOW_ITEM_TYPE_END,
6894                 },
6895         };
6896         struct rte_flow_action_jump jump = {
6897                 .group = MLX5_HAIRPIN_TX_TABLE,
6898         };
6899         struct rte_flow_action actions[2];
6900         uint32_t flow_idx;
6901         struct rte_flow_error error;
6902
6903         actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
6904         actions[0].conf = &jump;
6905         actions[1].type = RTE_FLOW_ACTION_TYPE_END;
6906         flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
6907                                     &attr, items, actions, false, &error);
6908         if (!flow_idx) {
6909                 DRV_LOG(DEBUG,
6910                         "Failed to create ctrl flow: rte_errno(%d),"
6911                         " type(%d), message(%s)",
6912                         rte_errno, error.type,
6913                         error.message ? error.message : " (no stated reason)");
6914                 return -rte_errno;
6915         }
6916         return 0;
6917 }
6918
6919 /**
6920  * Enable a control flow configured from the control plane.
6921  *
6922  * @param dev
6923  *   Pointer to Ethernet device.
6924  * @param eth_spec
6925  *   An Ethernet flow spec to apply.
6926  * @param eth_mask
6927  *   An Ethernet flow mask to apply.
6928  * @param vlan_spec
6929  *   A VLAN flow spec to apply.
6930  * @param vlan_mask
6931  *   A VLAN flow mask to apply.
6932  *
6933  * @return
6934  *   0 on success, a negative errno value otherwise and rte_errno is set.
6935  */
6936 int
6937 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
6938                     struct rte_flow_item_eth *eth_spec,
6939                     struct rte_flow_item_eth *eth_mask,
6940                     struct rte_flow_item_vlan *vlan_spec,
6941                     struct rte_flow_item_vlan *vlan_mask)
6942 {
6943         struct mlx5_priv *priv = dev->data->dev_private;
6944         const struct rte_flow_attr attr = {
6945                 .ingress = 1,
6946                 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
6947         };
6948         struct rte_flow_item items[] = {
6949                 {
6950                         .type = RTE_FLOW_ITEM_TYPE_ETH,
6951                         .spec = eth_spec,
6952                         .last = NULL,
6953                         .mask = eth_mask,
6954                 },
6955                 {
6956                         .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
6957                                               RTE_FLOW_ITEM_TYPE_END,
6958                         .spec = vlan_spec,
6959                         .last = NULL,
6960                         .mask = vlan_mask,
6961                 },
6962                 {
6963                         .type = RTE_FLOW_ITEM_TYPE_END,
6964                 },
6965         };
6966         uint16_t queue[priv->reta_idx_n];
6967         struct rte_flow_action_rss action_rss = {
6968                 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
6969                 .level = 0,
6970                 .types = priv->rss_conf.rss_hf,
6971                 .key_len = priv->rss_conf.rss_key_len,
6972                 .queue_num = priv->reta_idx_n,
6973                 .key = priv->rss_conf.rss_key,
6974                 .queue = queue,
6975         };
6976         struct rte_flow_action actions[] = {
6977                 {
6978                         .type = RTE_FLOW_ACTION_TYPE_RSS,
6979                         .conf = &action_rss,
6980                 },
6981                 {
6982                         .type = RTE_FLOW_ACTION_TYPE_END,
6983                 },
6984         };
6985         uint32_t flow_idx;
6986         struct rte_flow_error error;
6987         unsigned int i;
6988
6989         if (!priv->reta_idx_n || !priv->rxqs_n) {
6990                 return 0;
6991         }
6992         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
6993                 action_rss.types = 0;
6994         for (i = 0; i != priv->reta_idx_n; ++i)
6995                 queue[i] = (*priv->reta_idx)[i];
6996         flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
6997                                     &attr, items, actions, false, &error);
6998         if (!flow_idx)
6999                 return -rte_errno;
7000         return 0;
7001 }
7002
7003 /**
7004  * Enable a flow control configured from the control plane.
7005  *
7006  * @param dev
7007  *   Pointer to Ethernet device.
7008  * @param eth_spec
7009  *   An Ethernet flow spec to apply.
7010  * @param eth_mask
7011  *   An Ethernet flow mask to apply.
7012  *
7013  * @return
7014  *   0 on success, a negative errno value otherwise and rte_errno is set.
7015  */
7016 int
7017 mlx5_ctrl_flow(struct rte_eth_dev *dev,
7018                struct rte_flow_item_eth *eth_spec,
7019                struct rte_flow_item_eth *eth_mask)
7020 {
7021         return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
7022 }
7023
7024 /**
7025  * Create default miss flow rule matching lacp traffic
7026  *
7027  * @param dev
7028  *   Pointer to Ethernet device.
7029  * @param eth_spec
7030  *   An Ethernet flow spec to apply.
7031  *
7032  * @return
7033  *   0 on success, a negative errno value otherwise and rte_errno is set.
7034  */
7035 int
7036 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
7037 {
7038         /*
7039          * The LACP matching is done by only using ether type since using
7040          * a multicast dst mac causes kernel to give low priority to this flow.
7041          */
7042         static const struct rte_flow_item_eth lacp_spec = {
7043                 .type = RTE_BE16(0x8809),
7044         };
7045         static const struct rte_flow_item_eth lacp_mask = {
7046                 .type = 0xffff,
7047         };
7048         const struct rte_flow_attr attr = {
7049                 .ingress = 1,
7050         };
7051         struct rte_flow_item items[] = {
7052                 {
7053                         .type = RTE_FLOW_ITEM_TYPE_ETH,
7054                         .spec = &lacp_spec,
7055                         .mask = &lacp_mask,
7056                 },
7057                 {
7058                         .type = RTE_FLOW_ITEM_TYPE_END,
7059                 },
7060         };
7061         struct rte_flow_action actions[] = {
7062                 {
7063                         .type = (enum rte_flow_action_type)
7064                                 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS,
7065                 },
7066                 {
7067                         .type = RTE_FLOW_ACTION_TYPE_END,
7068                 },
7069         };
7070         struct rte_flow_error error;
7071         uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
7072                                         &attr, items, actions,
7073                                         false, &error);
7074
7075         if (!flow_idx)
7076                 return -rte_errno;
7077         return 0;
7078 }
7079
7080 /**
7081  * Destroy a flow.
7082  *
7083  * @see rte_flow_destroy()
7084  * @see rte_flow_ops
7085  */
7086 int
7087 mlx5_flow_destroy(struct rte_eth_dev *dev,
7088                   struct rte_flow *flow,
7089                   struct rte_flow_error *error __rte_unused)
7090 {
7091         flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
7092                                 (uintptr_t)(void *)flow);
7093         return 0;
7094 }
7095
7096 /**
7097  * Destroy all flows.
7098  *
7099  * @see rte_flow_flush()
7100  * @see rte_flow_ops
7101  */
7102 int
7103 mlx5_flow_flush(struct rte_eth_dev *dev,
7104                 struct rte_flow_error *error __rte_unused)
7105 {
7106         mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false);
7107         return 0;
7108 }
7109
7110 /**
7111  * Isolated mode.
7112  *
7113  * @see rte_flow_isolate()
7114  * @see rte_flow_ops
7115  */
7116 int
7117 mlx5_flow_isolate(struct rte_eth_dev *dev,
7118                   int enable,
7119                   struct rte_flow_error *error)
7120 {
7121         struct mlx5_priv *priv = dev->data->dev_private;
7122
7123         if (dev->data->dev_started) {
7124                 rte_flow_error_set(error, EBUSY,
7125                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7126                                    NULL,
7127                                    "port must be stopped first");
7128                 return -rte_errno;
7129         }
7130         priv->isolated = !!enable;
7131         if (enable)
7132                 dev->dev_ops = &mlx5_dev_ops_isolate;
7133         else
7134                 dev->dev_ops = &mlx5_dev_ops;
7135
7136         dev->rx_descriptor_status = mlx5_rx_descriptor_status;
7137         dev->tx_descriptor_status = mlx5_tx_descriptor_status;
7138
7139         return 0;
7140 }
7141
7142 /**
7143  * Query a flow.
7144  *
7145  * @see rte_flow_query()
7146  * @see rte_flow_ops
7147  */
7148 static int
7149 flow_drv_query(struct rte_eth_dev *dev,
7150                uint32_t flow_idx,
7151                const struct rte_flow_action *actions,
7152                void *data,
7153                struct rte_flow_error *error)
7154 {
7155         struct mlx5_priv *priv = dev->data->dev_private;
7156         const struct mlx5_flow_driver_ops *fops;
7157         struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
7158                                                flow_idx);
7159         enum mlx5_flow_drv_type ftype;
7160
7161         if (!flow) {
7162                 return rte_flow_error_set(error, ENOENT,
7163                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7164                           NULL,
7165                           "invalid flow handle");
7166         }
7167         ftype = flow->drv_type;
7168         MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
7169         fops = flow_get_drv_ops(ftype);
7170
7171         return fops->query(dev, flow, actions, data, error);
7172 }
7173
7174 /**
7175  * Query a flow.
7176  *
7177  * @see rte_flow_query()
7178  * @see rte_flow_ops
7179  */
7180 int
7181 mlx5_flow_query(struct rte_eth_dev *dev,
7182                 struct rte_flow *flow,
7183                 const struct rte_flow_action *actions,
7184                 void *data,
7185                 struct rte_flow_error *error)
7186 {
7187         int ret;
7188
7189         ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data,
7190                              error);
7191         if (ret < 0)
7192                 return ret;
7193         return 0;
7194 }
7195
7196 /**
7197  * Get rte_flow callbacks.
7198  *
7199  * @param dev
7200  *   Pointer to Ethernet device structure.
7201  * @param ops
7202  *   Pointer to operation-specific structure.
7203  *
7204  * @return 0
7205  */
7206 int
7207 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused,
7208                   const struct rte_flow_ops **ops)
7209 {
7210         *ops = &mlx5_flow_ops;
7211         return 0;
7212 }
7213
7214 /**
7215  * Validate meter policy actions.
7216  * Dispatcher for action type specific validation.
7217  *
7218  * @param[in] dev
7219  *   Pointer to the Ethernet device structure.
7220  * @param[in] action
7221  *   The meter policy action object to validate.
7222  * @param[in] attr
7223  *   Attributes of flow to determine steering domain.
7224  * @param[out] is_rss
7225  *   Is RSS or not.
7226  * @param[out] domain_bitmap
7227  *   Domain bitmap.
7228  * @param[out] is_def_policy
7229  *   Is default policy or not.
7230  * @param[out] error
7231  *   Perform verbose error reporting if not NULL. Initialized in case of
7232  *   error only.
7233  *
7234  * @return
7235  *   0 on success, otherwise negative errno value.
7236  */
7237 int
7238 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev,
7239                         const struct rte_flow_action *actions[RTE_COLORS],
7240                         struct rte_flow_attr *attr,
7241                         bool *is_rss,
7242                         uint8_t *domain_bitmap,
7243                         uint8_t *policy_mode,
7244                         struct rte_mtr_error *error)
7245 {
7246         const struct mlx5_flow_driver_ops *fops;
7247
7248         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7249         return fops->validate_mtr_acts(dev, actions, attr, is_rss,
7250                                        domain_bitmap, policy_mode, error);
7251 }
7252
7253 /**
7254  * Destroy the meter table set.
7255  *
7256  * @param[in] dev
7257  *   Pointer to Ethernet device.
7258  * @param[in] mtr_policy
7259  *   Meter policy struct.
7260  */
7261 void
7262 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev,
7263                       struct mlx5_flow_meter_policy *mtr_policy)
7264 {
7265         const struct mlx5_flow_driver_ops *fops;
7266
7267         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7268         fops->destroy_mtr_acts(dev, mtr_policy);
7269 }
7270
7271 /**
7272  * Create policy action, lock free,
7273  * (mutex should be acquired by caller).
7274  * Dispatcher for action type specific call.
7275  *
7276  * @param[in] dev
7277  *   Pointer to the Ethernet device structure.
7278  * @param[in] mtr_policy
7279  *   Meter policy struct.
7280  * @param[in] action
7281  *   Action specification used to create meter actions.
7282  * @param[out] error
7283  *   Perform verbose error reporting if not NULL. Initialized in case of
7284  *   error only.
7285  *
7286  * @return
7287  *   0 on success, otherwise negative errno value.
7288  */
7289 int
7290 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev,
7291                       struct mlx5_flow_meter_policy *mtr_policy,
7292                       const struct rte_flow_action *actions[RTE_COLORS],
7293                       struct rte_mtr_error *error)
7294 {
7295         const struct mlx5_flow_driver_ops *fops;
7296
7297         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7298         return fops->create_mtr_acts(dev, mtr_policy, actions, error);
7299 }
7300
7301 /**
7302  * Create policy rules, lock free,
7303  * (mutex should be acquired by caller).
7304  * Dispatcher for action type specific call.
7305  *
7306  * @param[in] dev
7307  *   Pointer to the Ethernet device structure.
7308  * @param[in] mtr_policy
7309  *   Meter policy struct.
7310  *
7311  * @return
7312  *   0 on success, -1 otherwise.
7313  */
7314 int
7315 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev,
7316                              struct mlx5_flow_meter_policy *mtr_policy)
7317 {
7318         const struct mlx5_flow_driver_ops *fops;
7319
7320         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7321         return fops->create_policy_rules(dev, mtr_policy);
7322 }
7323
7324 /**
7325  * Destroy policy rules, lock free,
7326  * (mutex should be acquired by caller).
7327  * Dispatcher for action type specific call.
7328  *
7329  * @param[in] dev
7330  *   Pointer to the Ethernet device structure.
7331  * @param[in] mtr_policy
7332  *   Meter policy struct.
7333  */
7334 void
7335 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev,
7336                              struct mlx5_flow_meter_policy *mtr_policy)
7337 {
7338         const struct mlx5_flow_driver_ops *fops;
7339
7340         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7341         fops->destroy_policy_rules(dev, mtr_policy);
7342 }
7343
7344 /**
7345  * Destroy the default policy table set.
7346  *
7347  * @param[in] dev
7348  *   Pointer to Ethernet device.
7349  */
7350 void
7351 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev)
7352 {
7353         const struct mlx5_flow_driver_ops *fops;
7354
7355         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7356         fops->destroy_def_policy(dev);
7357 }
7358
7359 /**
7360  * Destroy the default policy table set.
7361  *
7362  * @param[in] dev
7363  *   Pointer to Ethernet device.
7364  *
7365  * @return
7366  *   0 on success, -1 otherwise.
7367  */
7368 int
7369 mlx5_flow_create_def_policy(struct rte_eth_dev *dev)
7370 {
7371         const struct mlx5_flow_driver_ops *fops;
7372
7373         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7374         return fops->create_def_policy(dev);
7375 }
7376
7377 /**
7378  * Create the needed meter and suffix tables.
7379  *
7380  * @param[in] dev
7381  *   Pointer to Ethernet device.
7382  *
7383  * @return
7384  *   0 on success, -1 otherwise.
7385  */
7386 int
7387 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
7388                         struct mlx5_flow_meter_info *fm,
7389                         uint32_t mtr_idx,
7390                         uint8_t domain_bitmap)
7391 {
7392         const struct mlx5_flow_driver_ops *fops;
7393
7394         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7395         return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap);
7396 }
7397
7398 /**
7399  * Destroy the meter table set.
7400  *
7401  * @param[in] dev
7402  *   Pointer to Ethernet device.
7403  * @param[in] tbl
7404  *   Pointer to the meter table set.
7405  */
7406 void
7407 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
7408                            struct mlx5_flow_meter_info *fm)
7409 {
7410         const struct mlx5_flow_driver_ops *fops;
7411
7412         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7413         fops->destroy_mtr_tbls(dev, fm);
7414 }
7415
7416 /**
7417  * Destroy the global meter drop table.
7418  *
7419  * @param[in] dev
7420  *   Pointer to Ethernet device.
7421  */
7422 void
7423 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
7424 {
7425         const struct mlx5_flow_driver_ops *fops;
7426
7427         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7428         fops->destroy_mtr_drop_tbls(dev);
7429 }
7430
7431 /**
7432  * Destroy the sub policy table with RX queue.
7433  *
7434  * @param[in] dev
7435  *   Pointer to Ethernet device.
7436  * @param[in] mtr_policy
7437  *   Pointer to meter policy table.
7438  */
7439 void
7440 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
7441                 struct mlx5_flow_meter_policy *mtr_policy)
7442 {
7443         const struct mlx5_flow_driver_ops *fops;
7444
7445         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7446         fops->destroy_sub_policy_with_rxq(dev, mtr_policy);
7447 }
7448
7449 /**
7450  * Allocate the needed aso flow meter id.
7451  *
7452  * @param[in] dev
7453  *   Pointer to Ethernet device.
7454  *
7455  * @return
7456  *   Index to aso flow meter on success, NULL otherwise.
7457  */
7458 uint32_t
7459 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev)
7460 {
7461         const struct mlx5_flow_driver_ops *fops;
7462
7463         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7464         return fops->create_meter(dev);
7465 }
7466
7467 /**
7468  * Free the aso flow meter id.
7469  *
7470  * @param[in] dev
7471  *   Pointer to Ethernet device.
7472  * @param[in] mtr_idx
7473  *  Index to aso flow meter to be free.
7474  *
7475  * @return
7476  *   0 on success.
7477  */
7478 void
7479 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx)
7480 {
7481         const struct mlx5_flow_driver_ops *fops;
7482
7483         fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7484         fops->free_meter(dev, mtr_idx);
7485 }
7486
7487 /**
7488  * Allocate a counter.
7489  *
7490  * @param[in] dev
7491  *   Pointer to Ethernet device structure.
7492  *
7493  * @return
7494  *   Index to allocated counter  on success, 0 otherwise.
7495  */
7496 uint32_t
7497 mlx5_counter_alloc(struct rte_eth_dev *dev)
7498 {
7499         const struct mlx5_flow_driver_ops *fops;
7500         struct rte_flow_attr attr = { .transfer = 0 };
7501
7502         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7503                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7504                 return fops->counter_alloc(dev);
7505         }
7506         DRV_LOG(ERR,
7507                 "port %u counter allocate is not supported.",
7508                  dev->data->port_id);
7509         return 0;
7510 }
7511
7512 /**
7513  * Free a counter.
7514  *
7515  * @param[in] dev
7516  *   Pointer to Ethernet device structure.
7517  * @param[in] cnt
7518  *   Index to counter to be free.
7519  */
7520 void
7521 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
7522 {
7523         const struct mlx5_flow_driver_ops *fops;
7524         struct rte_flow_attr attr = { .transfer = 0 };
7525
7526         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7527                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7528                 fops->counter_free(dev, cnt);
7529                 return;
7530         }
7531         DRV_LOG(ERR,
7532                 "port %u counter free is not supported.",
7533                  dev->data->port_id);
7534 }
7535
7536 /**
7537  * Query counter statistics.
7538  *
7539  * @param[in] dev
7540  *   Pointer to Ethernet device structure.
7541  * @param[in] cnt
7542  *   Index to counter to query.
7543  * @param[in] clear
7544  *   Set to clear counter statistics.
7545  * @param[out] pkts
7546  *   The counter hits packets number to save.
7547  * @param[out] bytes
7548  *   The counter hits bytes number to save.
7549  *
7550  * @return
7551  *   0 on success, a negative errno value otherwise.
7552  */
7553 int
7554 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
7555                    bool clear, uint64_t *pkts, uint64_t *bytes)
7556 {
7557         const struct mlx5_flow_driver_ops *fops;
7558         struct rte_flow_attr attr = { .transfer = 0 };
7559
7560         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
7561                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
7562                 return fops->counter_query(dev, cnt, clear, pkts, bytes);
7563         }
7564         DRV_LOG(ERR,
7565                 "port %u counter query is not supported.",
7566                  dev->data->port_id);
7567         return -ENOTSUP;
7568 }
7569
7570 /**
7571  * Allocate a new memory for the counter values wrapped by all the needed
7572  * management.
7573  *
7574  * @param[in] sh
7575  *   Pointer to mlx5_dev_ctx_shared object.
7576  *
7577  * @return
7578  *   0 on success, a negative errno value otherwise.
7579  */
7580 static int
7581 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
7582 {
7583         struct mlx5_devx_mkey_attr mkey_attr;
7584         struct mlx5_counter_stats_mem_mng *mem_mng;
7585         volatile struct flow_counter_stats *raw_data;
7586         int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
7587         int size = (sizeof(struct flow_counter_stats) *
7588                         MLX5_COUNTERS_PER_POOL +
7589                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
7590                         sizeof(struct mlx5_counter_stats_mem_mng);
7591         size_t pgsize = rte_mem_page_size();
7592         uint8_t *mem;
7593         int i;
7594
7595         if (pgsize == (size_t)-1) {
7596                 DRV_LOG(ERR, "Failed to get mem page size");
7597                 rte_errno = ENOMEM;
7598                 return -ENOMEM;
7599         }
7600         mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY);
7601         if (!mem) {
7602                 rte_errno = ENOMEM;
7603                 return -ENOMEM;
7604         }
7605         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
7606         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
7607         mem_mng->umem = mlx5_os_umem_reg(sh->ctx, mem, size,
7608                                                  IBV_ACCESS_LOCAL_WRITE);
7609         if (!mem_mng->umem) {
7610                 rte_errno = errno;
7611                 mlx5_free(mem);
7612                 return -rte_errno;
7613         }
7614         memset(&mkey_attr, 0, sizeof(mkey_attr));
7615         mkey_attr.addr = (uintptr_t)mem;
7616         mkey_attr.size = size;
7617         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
7618         mkey_attr.pd = sh->pdn;
7619         mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write;
7620         mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read;
7621         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
7622         if (!mem_mng->dm) {
7623                 mlx5_os_umem_dereg(mem_mng->umem);
7624                 rte_errno = errno;
7625                 mlx5_free(mem);
7626                 return -rte_errno;
7627         }
7628         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
7629         raw_data = (volatile struct flow_counter_stats *)mem;
7630         for (i = 0; i < raws_n; ++i) {
7631                 mem_mng->raws[i].mem_mng = mem_mng;
7632                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
7633         }
7634         for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
7635                 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
7636                                  mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
7637                                  next);
7638         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
7639         sh->cmng.mem_mng = mem_mng;
7640         return 0;
7641 }
7642
7643 /**
7644  * Set the statistic memory to the new counter pool.
7645  *
7646  * @param[in] sh
7647  *   Pointer to mlx5_dev_ctx_shared object.
7648  * @param[in] pool
7649  *   Pointer to the pool to set the statistic memory.
7650  *
7651  * @return
7652  *   0 on success, a negative errno value otherwise.
7653  */
7654 static int
7655 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
7656                                struct mlx5_flow_counter_pool *pool)
7657 {
7658         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7659         /* Resize statistic memory once used out. */
7660         if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
7661             mlx5_flow_create_counter_stat_mem_mng(sh)) {
7662                 DRV_LOG(ERR, "Cannot resize counter stat mem.");
7663                 return -1;
7664         }
7665         rte_spinlock_lock(&pool->sl);
7666         pool->raw = cmng->mem_mng->raws + pool->index %
7667                     MLX5_CNT_CONTAINER_RESIZE;
7668         rte_spinlock_unlock(&pool->sl);
7669         pool->raw_hw = NULL;
7670         return 0;
7671 }
7672
7673 #define MLX5_POOL_QUERY_FREQ_US 1000000
7674
7675 /**
7676  * Set the periodic procedure for triggering asynchronous batch queries for all
7677  * the counter pools.
7678  *
7679  * @param[in] sh
7680  *   Pointer to mlx5_dev_ctx_shared object.
7681  */
7682 void
7683 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh)
7684 {
7685         uint32_t pools_n, us;
7686
7687         pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED);
7688         us = MLX5_POOL_QUERY_FREQ_US / pools_n;
7689         DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us);
7690         if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) {
7691                 sh->cmng.query_thread_on = 0;
7692                 DRV_LOG(ERR, "Cannot reinitialize query alarm");
7693         } else {
7694                 sh->cmng.query_thread_on = 1;
7695         }
7696 }
7697
7698 /**
7699  * The periodic procedure for triggering asynchronous batch queries for all the
7700  * counter pools. This function is probably called by the host thread.
7701  *
7702  * @param[in] arg
7703  *   The parameter for the alarm process.
7704  */
7705 void
7706 mlx5_flow_query_alarm(void *arg)
7707 {
7708         struct mlx5_dev_ctx_shared *sh = arg;
7709         int ret;
7710         uint16_t pool_index = sh->cmng.pool_index;
7711         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7712         struct mlx5_flow_counter_pool *pool;
7713         uint16_t n_valid;
7714
7715         if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
7716                 goto set_alarm;
7717         rte_spinlock_lock(&cmng->pool_update_sl);
7718         pool = cmng->pools[pool_index];
7719         n_valid = cmng->n_valid;
7720         rte_spinlock_unlock(&cmng->pool_update_sl);
7721         /* Set the statistic memory to the new created pool. */
7722         if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool)))
7723                 goto set_alarm;
7724         if (pool->raw_hw)
7725                 /* There is a pool query in progress. */
7726                 goto set_alarm;
7727         pool->raw_hw =
7728                 LIST_FIRST(&sh->cmng.free_stat_raws);
7729         if (!pool->raw_hw)
7730                 /* No free counter statistics raw memory. */
7731                 goto set_alarm;
7732         /*
7733          * Identify the counters released between query trigger and query
7734          * handle more efficiently. The counter released in this gap period
7735          * should wait for a new round of query as the new arrived packets
7736          * will not be taken into account.
7737          */
7738         pool->query_gen++;
7739         ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
7740                                                MLX5_COUNTERS_PER_POOL,
7741                                                NULL, NULL,
7742                                                pool->raw_hw->mem_mng->dm->id,
7743                                                (void *)(uintptr_t)
7744                                                pool->raw_hw->data,
7745                                                sh->devx_comp,
7746                                                (uint64_t)(uintptr_t)pool);
7747         if (ret) {
7748                 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID"
7749                         " %d", pool->min_dcs->id);
7750                 pool->raw_hw = NULL;
7751                 goto set_alarm;
7752         }
7753         LIST_REMOVE(pool->raw_hw, next);
7754         sh->cmng.pending_queries++;
7755         pool_index++;
7756         if (pool_index >= n_valid)
7757                 pool_index = 0;
7758 set_alarm:
7759         sh->cmng.pool_index = pool_index;
7760         mlx5_set_query_alarm(sh);
7761 }
7762
7763 /**
7764  * Check and callback event for new aged flow in the counter pool
7765  *
7766  * @param[in] sh
7767  *   Pointer to mlx5_dev_ctx_shared object.
7768  * @param[in] pool
7769  *   Pointer to Current counter pool.
7770  */
7771 static void
7772 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh,
7773                    struct mlx5_flow_counter_pool *pool)
7774 {
7775         struct mlx5_priv *priv;
7776         struct mlx5_flow_counter *cnt;
7777         struct mlx5_age_info *age_info;
7778         struct mlx5_age_param *age_param;
7779         struct mlx5_counter_stats_raw *cur = pool->raw_hw;
7780         struct mlx5_counter_stats_raw *prev = pool->raw;
7781         const uint64_t curr_time = MLX5_CURR_TIME_SEC;
7782         const uint32_t time_delta = curr_time - pool->time_of_last_age_check;
7783         uint16_t expected = AGE_CANDIDATE;
7784         uint32_t i;
7785
7786         pool->time_of_last_age_check = curr_time;
7787         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
7788                 cnt = MLX5_POOL_GET_CNT(pool, i);
7789                 age_param = MLX5_CNT_TO_AGE(cnt);
7790                 if (__atomic_load_n(&age_param->state,
7791                                     __ATOMIC_RELAXED) != AGE_CANDIDATE)
7792                         continue;
7793                 if (cur->data[i].hits != prev->data[i].hits) {
7794                         __atomic_store_n(&age_param->sec_since_last_hit, 0,
7795                                          __ATOMIC_RELAXED);
7796                         continue;
7797                 }
7798                 if (__atomic_add_fetch(&age_param->sec_since_last_hit,
7799                                        time_delta,
7800                                        __ATOMIC_RELAXED) <= age_param->timeout)
7801                         continue;
7802                 /**
7803                  * Hold the lock first, or if between the
7804                  * state AGE_TMOUT and tailq operation the
7805                  * release happened, the release procedure
7806                  * may delete a non-existent tailq node.
7807                  */
7808                 priv = rte_eth_devices[age_param->port_id].data->dev_private;
7809                 age_info = GET_PORT_AGE_INFO(priv);
7810                 rte_spinlock_lock(&age_info->aged_sl);
7811                 if (__atomic_compare_exchange_n(&age_param->state, &expected,
7812                                                 AGE_TMOUT, false,
7813                                                 __ATOMIC_RELAXED,
7814                                                 __ATOMIC_RELAXED)) {
7815                         TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next);
7816                         MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW);
7817                 }
7818                 rte_spinlock_unlock(&age_info->aged_sl);
7819         }
7820         mlx5_age_event_prepare(sh);
7821 }
7822
7823 /**
7824  * Handler for the HW respond about ready values from an asynchronous batch
7825  * query. This function is probably called by the host thread.
7826  *
7827  * @param[in] sh
7828  *   The pointer to the shared device context.
7829  * @param[in] async_id
7830  *   The Devx async ID.
7831  * @param[in] status
7832  *   The status of the completion.
7833  */
7834 void
7835 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh,
7836                                   uint64_t async_id, int status)
7837 {
7838         struct mlx5_flow_counter_pool *pool =
7839                 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id;
7840         struct mlx5_counter_stats_raw *raw_to_free;
7841         uint8_t query_gen = pool->query_gen ^ 1;
7842         struct mlx5_flow_counter_mng *cmng = &sh->cmng;
7843         enum mlx5_counter_type cnt_type =
7844                 pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
7845                                 MLX5_COUNTER_TYPE_ORIGIN;
7846
7847         if (unlikely(status)) {
7848                 raw_to_free = pool->raw_hw;
7849         } else {
7850                 raw_to_free = pool->raw;
7851                 if (pool->is_aged)
7852                         mlx5_flow_aging_check(sh, pool);
7853                 rte_spinlock_lock(&pool->sl);
7854                 pool->raw = pool->raw_hw;
7855                 rte_spinlock_unlock(&pool->sl);
7856                 /* Be sure the new raw counters data is updated in memory. */
7857                 rte_io_wmb();
7858                 if (!TAILQ_EMPTY(&pool->counters[query_gen])) {
7859                         rte_spinlock_lock(&cmng->csl[cnt_type]);
7860                         TAILQ_CONCAT(&cmng->counters[cnt_type],
7861                                      &pool->counters[query_gen], next);
7862                         rte_spinlock_unlock(&cmng->csl[cnt_type]);
7863                 }
7864         }
7865         LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next);
7866         pool->raw_hw = NULL;
7867         sh->cmng.pending_queries--;
7868 }
7869
7870 static int
7871 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table,
7872                     const struct flow_grp_info *grp_info,
7873                     struct rte_flow_error *error)
7874 {
7875         if (grp_info->transfer && grp_info->external &&
7876             grp_info->fdb_def_rule) {
7877                 if (group == UINT32_MAX)
7878                         return rte_flow_error_set
7879                                                 (error, EINVAL,
7880                                                  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
7881                                                  NULL,
7882                                                  "group index not supported");
7883                 *table = group + 1;
7884         } else {
7885                 *table = group;
7886         }
7887         DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table);
7888         return 0;
7889 }
7890
7891 /**
7892  * Translate the rte_flow group index to HW table value.
7893  *
7894  * If tunnel offload is disabled, all group ids converted to flow table
7895  * id using the standard method.
7896  * If tunnel offload is enabled, group id can be converted using the
7897  * standard or tunnel conversion method. Group conversion method
7898  * selection depends on flags in `grp_info` parameter:
7899  * - Internal (grp_info.external == 0) groups conversion uses the
7900  *   standard method.
7901  * - Group ids in JUMP action converted with the tunnel conversion.
7902  * - Group id in rule attribute conversion depends on a rule type and
7903  *   group id value:
7904  *   ** non zero group attributes converted with the tunnel method
7905  *   ** zero group attribute in non-tunnel rule is converted using the
7906  *      standard method - there's only one root table
7907  *   ** zero group attribute in steer tunnel rule is converted with the
7908  *      standard method - single root table
7909  *   ** zero group attribute in match tunnel rule is a special OvS
7910  *      case: that value is used for portability reasons. That group
7911  *      id is converted with the tunnel conversion method.
7912  *
7913  * @param[in] dev
7914  *   Port device
7915  * @param[in] tunnel
7916  *   PMD tunnel offload object
7917  * @param[in] group
7918  *   rte_flow group index value.
7919  * @param[out] table
7920  *   HW table value.
7921  * @param[in] grp_info
7922  *   flags used for conversion
7923  * @param[out] error
7924  *   Pointer to error structure.
7925  *
7926  * @return
7927  *   0 on success, a negative errno value otherwise and rte_errno is set.
7928  */
7929 int
7930 mlx5_flow_group_to_table(struct rte_eth_dev *dev,
7931                          const struct mlx5_flow_tunnel *tunnel,
7932                          uint32_t group, uint32_t *table,
7933                          const struct flow_grp_info *grp_info,
7934                          struct rte_flow_error *error)
7935 {
7936         int ret;
7937         bool standard_translation;
7938
7939         if (!grp_info->skip_scale && grp_info->external &&
7940             group < MLX5_MAX_TABLES_EXTERNAL)
7941                 group *= MLX5_FLOW_TABLE_FACTOR;
7942         if (is_tunnel_offload_active(dev)) {
7943                 standard_translation = !grp_info->external ||
7944                                         grp_info->std_tbl_fix;
7945         } else {
7946                 standard_translation = true;
7947         }
7948         DRV_LOG(DEBUG,
7949                 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s",
7950                 dev->data->port_id, group, grp_info->transfer,
7951                 grp_info->external, grp_info->fdb_def_rule,
7952                 standard_translation ? "STANDARD" : "TUNNEL");
7953         if (standard_translation)
7954                 ret = flow_group_to_table(dev->data->port_id, group, table,
7955                                           grp_info, error);
7956         else
7957                 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group,
7958                                                       table, error);
7959
7960         return ret;
7961 }
7962
7963 /**
7964  * Discover availability of metadata reg_c's.
7965  *
7966  * Iteratively use test flows to check availability.
7967  *
7968  * @param[in] dev
7969  *   Pointer to the Ethernet device structure.
7970  *
7971  * @return
7972  *   0 on success, a negative errno value otherwise and rte_errno is set.
7973  */
7974 int
7975 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
7976 {
7977         struct mlx5_priv *priv = dev->data->dev_private;
7978         struct mlx5_dev_config *config = &priv->config;
7979         enum modify_reg idx;
7980         int n = 0;
7981
7982         /* reg_c[0] and reg_c[1] are reserved. */
7983         config->flow_mreg_c[n++] = REG_C_0;
7984         config->flow_mreg_c[n++] = REG_C_1;
7985         /* Discover availability of other reg_c's. */
7986         for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
7987                 struct rte_flow_attr attr = {
7988                         .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
7989                         .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR,
7990                         .ingress = 1,
7991                 };
7992                 struct rte_flow_item items[] = {
7993                         [0] = {
7994                                 .type = RTE_FLOW_ITEM_TYPE_END,
7995                         },
7996                 };
7997                 struct rte_flow_action actions[] = {
7998                         [0] = {
7999                                 .type = (enum rte_flow_action_type)
8000                                         MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
8001                                 .conf = &(struct mlx5_flow_action_copy_mreg){
8002                                         .src = REG_C_1,
8003                                         .dst = idx,
8004                                 },
8005                         },
8006                         [1] = {
8007                                 .type = RTE_FLOW_ACTION_TYPE_JUMP,
8008                                 .conf = &(struct rte_flow_action_jump){
8009                                         .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
8010                                 },
8011                         },
8012                         [2] = {
8013                                 .type = RTE_FLOW_ACTION_TYPE_END,
8014                         },
8015                 };
8016                 uint32_t flow_idx;
8017                 struct rte_flow *flow;
8018                 struct rte_flow_error error;
8019
8020                 if (!config->dv_flow_en)
8021                         break;
8022                 /* Create internal flow, validation skips copy action. */
8023                 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
8024                                         items, actions, false, &error);
8025                 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
8026                                       flow_idx);
8027                 if (!flow)
8028                         continue;
8029                 config->flow_mreg_c[n++] = idx;
8030                 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
8031         }
8032         for (; n < MLX5_MREG_C_NUM; ++n)
8033                 config->flow_mreg_c[n] = REG_NON;
8034         return 0;
8035 }
8036
8037 int
8038 save_dump_file(const uint8_t *data, uint32_t size,
8039         uint32_t type, uint32_t id, void *arg, FILE *file)
8040 {
8041         char line[BUF_SIZE];
8042         uint32_t out = 0;
8043         uint32_t k;
8044         uint32_t actions_num;
8045         struct rte_flow_query_count *count;
8046
8047         memset(line, 0, BUF_SIZE);
8048         switch (type) {
8049         case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR:
8050                 actions_num = *(uint32_t *)(arg);
8051                 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,%d,",
8052                                 type, id, actions_num);
8053                 break;
8054         case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT:
8055                 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,",
8056                                 type, id);
8057                 break;
8058         case DR_DUMP_REC_TYPE_PMD_COUNTER:
8059                 count = (struct rte_flow_query_count *)arg;
8060                 fprintf(file, "%d,0x%x,%" PRIu64 ",%" PRIu64 "\n", type,
8061                                 id, count->hits, count->bytes);
8062                 return 0;
8063         default:
8064                 return -1;
8065         }
8066
8067         for (k = 0; k < size; k++) {
8068                 /* Make sure we do not overrun the line buffer length. */
8069                 if (out >= BUF_SIZE - 4) {
8070                         line[out] = '\0';
8071                         break;
8072                 }
8073                 out += snprintf(line + out, BUF_SIZE - out, "%02x",
8074                                 (data[k]) & 0xff);
8075         }
8076         fprintf(file, "%s\n", line);
8077         return 0;
8078 }
8079
8080 int
8081 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow,
8082         struct rte_flow_query_count *count, struct rte_flow_error *error)
8083 {
8084         struct rte_flow_action action[2];
8085         enum mlx5_flow_drv_type ftype;
8086         const struct mlx5_flow_driver_ops *fops;
8087
8088         if (!flow) {
8089                 return rte_flow_error_set(error, ENOENT,
8090                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8091                                 NULL,
8092                                 "invalid flow handle");
8093         }
8094         action[0].type = RTE_FLOW_ACTION_TYPE_COUNT;
8095         action[1].type = RTE_FLOW_ACTION_TYPE_END;
8096         if (flow->counter) {
8097                 memset(count, 0, sizeof(struct rte_flow_query_count));
8098                 ftype = (enum mlx5_flow_drv_type)(flow->drv_type);
8099                 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN &&
8100                                                 ftype < MLX5_FLOW_TYPE_MAX);
8101                 fops = flow_get_drv_ops(ftype);
8102                 return fops->query(dev, flow, action, count, error);
8103         }
8104         return -1;
8105 }
8106
8107 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8108 /**
8109  * Dump flow ipool data to file
8110  *
8111  * @param[in] dev
8112  *   The pointer to Ethernet device.
8113  * @param[in] file
8114  *   A pointer to a file for output.
8115  * @param[out] error
8116  *   Perform verbose error reporting if not NULL. PMDs initialize this
8117  *   structure in case of error only.
8118  * @return
8119  *   0 on success, a negative value otherwise.
8120  */
8121 int
8122 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev,
8123         struct rte_flow *flow, FILE *file,
8124         struct rte_flow_error *error)
8125 {
8126         struct mlx5_priv *priv = dev->data->dev_private;
8127         struct mlx5_flow_dv_modify_hdr_resource  *modify_hdr;
8128         struct mlx5_flow_dv_encap_decap_resource *encap_decap;
8129         uint32_t handle_idx;
8130         struct mlx5_flow_handle *dh;
8131         struct rte_flow_query_count count;
8132         uint32_t actions_num;
8133         const uint8_t *data;
8134         size_t size;
8135         uint32_t id;
8136         uint32_t type;
8137
8138         if (!flow) {
8139                 return rte_flow_error_set(error, ENOENT,
8140                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8141                         NULL,
8142                         "invalid flow handle");
8143         }
8144         handle_idx = flow->dev_handles;
8145         while (handle_idx) {
8146                 dh = mlx5_ipool_get(priv->sh->ipool
8147                         [MLX5_IPOOL_MLX5_FLOW], handle_idx);
8148                 if (!dh)
8149                         continue;
8150                 handle_idx = dh->next.next;
8151                 id = (uint32_t)(uintptr_t)dh->drv_flow;
8152
8153                 /* query counter */
8154                 type = DR_DUMP_REC_TYPE_PMD_COUNTER;
8155                 if (!mlx5_flow_query_counter(dev, flow, &count, error))
8156                         save_dump_file(NULL, 0, type,
8157                                         id, (void *)&count, file);
8158
8159                 /* Get modify_hdr and encap_decap buf from ipools. */
8160                 encap_decap = NULL;
8161                 modify_hdr = dh->dvh.modify_hdr;
8162
8163                 if (dh->dvh.rix_encap_decap) {
8164                         encap_decap = mlx5_ipool_get(priv->sh->ipool
8165                                                 [MLX5_IPOOL_DECAP_ENCAP],
8166                                                 dh->dvh.rix_encap_decap);
8167                 }
8168                 if (modify_hdr) {
8169                         data = (const uint8_t *)modify_hdr->actions;
8170                         size = (size_t)(modify_hdr->actions_num) * 8;
8171                         actions_num = modify_hdr->actions_num;
8172                         type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR;
8173                         save_dump_file(data, size, type, id,
8174                                         (void *)(&actions_num), file);
8175                 }
8176                 if (encap_decap) {
8177                         data = encap_decap->buf;
8178                         size = encap_decap->size;
8179                         type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT;
8180                         save_dump_file(data, size, type,
8181                                                 id, NULL, file);
8182                 }
8183         }
8184         return 0;
8185 }
8186 #endif
8187
8188 /**
8189  * Dump flow raw hw data to file
8190  *
8191  * @param[in] dev
8192  *    The pointer to Ethernet device.
8193  * @param[in] file
8194  *   A pointer to a file for output.
8195  * @param[out] error
8196  *   Perform verbose error reporting if not NULL. PMDs initialize this
8197  *   structure in case of error only.
8198  * @return
8199  *   0 on success, a nagative value otherwise.
8200  */
8201 int
8202 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
8203                    FILE *file,
8204                    struct rte_flow_error *error __rte_unused)
8205 {
8206         struct mlx5_priv *priv = dev->data->dev_private;
8207         struct mlx5_dev_ctx_shared *sh = priv->sh;
8208         uint32_t handle_idx;
8209         int ret;
8210         struct mlx5_flow_handle *dh;
8211         struct rte_flow *flow;
8212 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8213         uint32_t idx;
8214 #endif
8215
8216         if (!priv->config.dv_flow_en) {
8217                 if (fputs("device dv flow disabled\n", file) <= 0)
8218                         return -errno;
8219                 return -ENOTSUP;
8220         }
8221
8222         /* dump all */
8223         if (!flow_idx) {
8224 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8225                 MLX5_IPOOL_FOREACH(priv->flows[MLX5_FLOW_TYPE_GEN], idx, flow)
8226                         mlx5_flow_dev_dump_ipool(dev, flow, file, error);
8227 #endif
8228                 return mlx5_devx_cmd_flow_dump(sh->fdb_domain,
8229                                         sh->rx_domain,
8230                                         sh->tx_domain, file);
8231         }
8232         /* dump one */
8233         flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
8234                         (uintptr_t)(void *)flow_idx);
8235         if (!flow)
8236                 return -ENOENT;
8237
8238 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8239         mlx5_flow_dev_dump_ipool(dev, flow, file, error);
8240 #endif
8241         handle_idx = flow->dev_handles;
8242         while (handle_idx) {
8243                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
8244                                 handle_idx);
8245                 if (!dh)
8246                         return -ENOENT;
8247                 if (dh->drv_flow) {
8248                         ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow,
8249                                         file);
8250                         if (ret)
8251                                 return -ENOENT;
8252                 }
8253                 handle_idx = dh->next.next;
8254         }
8255         return 0;
8256 }
8257
8258 /**
8259  * Get aged-out flows.
8260  *
8261  * @param[in] dev
8262  *   Pointer to the Ethernet device structure.
8263  * @param[in] context
8264  *   The address of an array of pointers to the aged-out flows contexts.
8265  * @param[in] nb_countexts
8266  *   The length of context array pointers.
8267  * @param[out] error
8268  *   Perform verbose error reporting if not NULL. Initialized in case of
8269  *   error only.
8270  *
8271  * @return
8272  *   how many contexts get in success, otherwise negative errno value.
8273  *   if nb_contexts is 0, return the amount of all aged contexts.
8274  *   if nb_contexts is not 0 , return the amount of aged flows reported
8275  *   in the context array.
8276  */
8277 int
8278 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
8279                         uint32_t nb_contexts, struct rte_flow_error *error)
8280 {
8281         const struct mlx5_flow_driver_ops *fops;
8282         struct rte_flow_attr attr = { .transfer = 0 };
8283
8284         if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) {
8285                 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
8286                 return fops->get_aged_flows(dev, contexts, nb_contexts,
8287                                                     error);
8288         }
8289         DRV_LOG(ERR,
8290                 "port %u get aged flows is not supported.",
8291                  dev->data->port_id);
8292         return -ENOTSUP;
8293 }
8294
8295 /* Wrapper for driver action_validate op callback */
8296 static int
8297 flow_drv_action_validate(struct rte_eth_dev *dev,
8298                          const struct rte_flow_indir_action_conf *conf,
8299                          const struct rte_flow_action *action,
8300                          const struct mlx5_flow_driver_ops *fops,
8301                          struct rte_flow_error *error)
8302 {
8303         static const char err_msg[] = "indirect action validation unsupported";
8304
8305         if (!fops->action_validate) {
8306                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8307                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8308                                    NULL, err_msg);
8309                 return -rte_errno;
8310         }
8311         return fops->action_validate(dev, conf, action, error);
8312 }
8313
8314 /**
8315  * Destroys the shared action by handle.
8316  *
8317  * @param dev
8318  *   Pointer to Ethernet device structure.
8319  * @param[in] handle
8320  *   Handle for the indirect action object to be destroyed.
8321  * @param[out] error
8322  *   Perform verbose error reporting if not NULL. PMDs initialize this
8323  *   structure in case of error only.
8324  *
8325  * @return
8326  *   0 on success, a negative errno value otherwise and rte_errno is set.
8327  *
8328  * @note: wrapper for driver action_create op callback.
8329  */
8330 static int
8331 mlx5_action_handle_destroy(struct rte_eth_dev *dev,
8332                            struct rte_flow_action_handle *handle,
8333                            struct rte_flow_error *error)
8334 {
8335         static const char err_msg[] = "indirect action destruction unsupported";
8336         struct rte_flow_attr attr = { .transfer = 0 };
8337         const struct mlx5_flow_driver_ops *fops =
8338                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8339
8340         if (!fops->action_destroy) {
8341                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8342                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8343                                    NULL, err_msg);
8344                 return -rte_errno;
8345         }
8346         return fops->action_destroy(dev, handle, error);
8347 }
8348
8349 /* Wrapper for driver action_destroy op callback */
8350 static int
8351 flow_drv_action_update(struct rte_eth_dev *dev,
8352                        struct rte_flow_action_handle *handle,
8353                        const void *update,
8354                        const struct mlx5_flow_driver_ops *fops,
8355                        struct rte_flow_error *error)
8356 {
8357         static const char err_msg[] = "indirect action update unsupported";
8358
8359         if (!fops->action_update) {
8360                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8361                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8362                                    NULL, err_msg);
8363                 return -rte_errno;
8364         }
8365         return fops->action_update(dev, handle, update, error);
8366 }
8367
8368 /* Wrapper for driver action_destroy op callback */
8369 static int
8370 flow_drv_action_query(struct rte_eth_dev *dev,
8371                       const struct rte_flow_action_handle *handle,
8372                       void *data,
8373                       const struct mlx5_flow_driver_ops *fops,
8374                       struct rte_flow_error *error)
8375 {
8376         static const char err_msg[] = "indirect action query unsupported";
8377
8378         if (!fops->action_query) {
8379                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8380                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8381                                    NULL, err_msg);
8382                 return -rte_errno;
8383         }
8384         return fops->action_query(dev, handle, data, error);
8385 }
8386
8387 /**
8388  * Create indirect action for reuse in multiple flow rules.
8389  *
8390  * @param dev
8391  *   Pointer to Ethernet device structure.
8392  * @param conf
8393  *   Pointer to indirect action object configuration.
8394  * @param[in] action
8395  *   Action configuration for indirect action object creation.
8396  * @param[out] error
8397  *   Perform verbose error reporting if not NULL. PMDs initialize this
8398  *   structure in case of error only.
8399  * @return
8400  *   A valid handle in case of success, NULL otherwise and rte_errno is set.
8401  */
8402 static struct rte_flow_action_handle *
8403 mlx5_action_handle_create(struct rte_eth_dev *dev,
8404                           const struct rte_flow_indir_action_conf *conf,
8405                           const struct rte_flow_action *action,
8406                           struct rte_flow_error *error)
8407 {
8408         static const char err_msg[] = "indirect action creation unsupported";
8409         struct rte_flow_attr attr = { .transfer = 0 };
8410         const struct mlx5_flow_driver_ops *fops =
8411                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8412
8413         if (flow_drv_action_validate(dev, conf, action, fops, error))
8414                 return NULL;
8415         if (!fops->action_create) {
8416                 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg);
8417                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8418                                    NULL, err_msg);
8419                 return NULL;
8420         }
8421         return fops->action_create(dev, conf, action, error);
8422 }
8423
8424 /**
8425  * Updates inplace the indirect action configuration pointed by *handle*
8426  * with the configuration provided as *update* argument.
8427  * The update of the indirect action configuration effects all flow rules
8428  * reusing the action via handle.
8429  *
8430  * @param dev
8431  *   Pointer to Ethernet device structure.
8432  * @param[in] handle
8433  *   Handle for the indirect action to be updated.
8434  * @param[in] update
8435  *   Action specification used to modify the action pointed by handle.
8436  *   *update* could be of same type with the action pointed by the *handle*
8437  *   handle argument, or some other structures like a wrapper, depending on
8438  *   the indirect action type.
8439  * @param[out] error
8440  *   Perform verbose error reporting if not NULL. PMDs initialize this
8441  *   structure in case of error only.
8442  *
8443  * @return
8444  *   0 on success, a negative errno value otherwise and rte_errno is set.
8445  */
8446 static int
8447 mlx5_action_handle_update(struct rte_eth_dev *dev,
8448                 struct rte_flow_action_handle *handle,
8449                 const void *update,
8450                 struct rte_flow_error *error)
8451 {
8452         struct rte_flow_attr attr = { .transfer = 0 };
8453         const struct mlx5_flow_driver_ops *fops =
8454                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8455         int ret;
8456
8457         ret = flow_drv_action_validate(dev, NULL,
8458                         (const struct rte_flow_action *)update, fops, error);
8459         if (ret)
8460                 return ret;
8461         return flow_drv_action_update(dev, handle, update, fops,
8462                                       error);
8463 }
8464
8465 /**
8466  * Query the indirect action by handle.
8467  *
8468  * This function allows retrieving action-specific data such as counters.
8469  * Data is gathered by special action which may be present/referenced in
8470  * more than one flow rule definition.
8471  *
8472  * see @RTE_FLOW_ACTION_TYPE_COUNT
8473  *
8474  * @param dev
8475  *   Pointer to Ethernet device structure.
8476  * @param[in] handle
8477  *   Handle for the indirect action to query.
8478  * @param[in, out] data
8479  *   Pointer to storage for the associated query data type.
8480  * @param[out] error
8481  *   Perform verbose error reporting if not NULL. PMDs initialize this
8482  *   structure in case of error only.
8483  *
8484  * @return
8485  *   0 on success, a negative errno value otherwise and rte_errno is set.
8486  */
8487 static int
8488 mlx5_action_handle_query(struct rte_eth_dev *dev,
8489                          const struct rte_flow_action_handle *handle,
8490                          void *data,
8491                          struct rte_flow_error *error)
8492 {
8493         struct rte_flow_attr attr = { .transfer = 0 };
8494         const struct mlx5_flow_driver_ops *fops =
8495                         flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8496
8497         return flow_drv_action_query(dev, handle, data, fops, error);
8498 }
8499
8500 /**
8501  * Destroy all indirect actions (shared RSS).
8502  *
8503  * @param dev
8504  *   Pointer to Ethernet device.
8505  *
8506  * @return
8507  *   0 on success, a negative errno value otherwise and rte_errno is set.
8508  */
8509 int
8510 mlx5_action_handle_flush(struct rte_eth_dev *dev)
8511 {
8512         struct rte_flow_error error;
8513         struct mlx5_priv *priv = dev->data->dev_private;
8514         struct mlx5_shared_action_rss *shared_rss;
8515         int ret = 0;
8516         uint32_t idx;
8517
8518         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
8519                       priv->rss_shared_actions, idx, shared_rss, next) {
8520                 ret |= mlx5_action_handle_destroy(dev,
8521                        (struct rte_flow_action_handle *)(uintptr_t)idx, &error);
8522         }
8523         return ret;
8524 }
8525
8526 #ifndef HAVE_MLX5DV_DR
8527 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1))
8528 #else
8529 #define MLX5_DOMAIN_SYNC_FLOW \
8530         (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW)
8531 #endif
8532
8533 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains)
8534 {
8535         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
8536         const struct mlx5_flow_driver_ops *fops;
8537         int ret;
8538         struct rte_flow_attr attr = { .transfer = 0 };
8539
8540         fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr));
8541         ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW);
8542         if (ret > 0)
8543                 ret = -ret;
8544         return ret;
8545 }
8546
8547 const struct mlx5_flow_tunnel *
8548 mlx5_get_tof(const struct rte_flow_item *item,
8549              const struct rte_flow_action *action,
8550              enum mlx5_tof_rule_type *rule_type)
8551 {
8552         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
8553                 if (item->type == (typeof(item->type))
8554                                   MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) {
8555                         *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE;
8556                         return flow_items_to_tunnel(item);
8557                 }
8558         }
8559         for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) {
8560                 if (action->type == (typeof(action->type))
8561                                     MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) {
8562                         *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE;
8563                         return flow_actions_to_tunnel(action);
8564                 }
8565         }
8566         return NULL;
8567 }
8568
8569 /**
8570  * tunnel offload functionalilty is defined for DV environment only
8571  */
8572 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
8573 __extension__
8574 union tunnel_offload_mark {
8575         uint32_t val;
8576         struct {
8577                 uint32_t app_reserve:8;
8578                 uint32_t table_id:15;
8579                 uint32_t transfer:1;
8580                 uint32_t _unused_:8;
8581         };
8582 };
8583
8584 static bool
8585 mlx5_access_tunnel_offload_db
8586         (struct rte_eth_dev *dev,
8587          bool (*match)(struct rte_eth_dev *,
8588                        struct mlx5_flow_tunnel *, const void *),
8589          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
8590          void (*miss)(struct rte_eth_dev *, void *),
8591          void *ctx, bool lock_op);
8592
8593 static int
8594 flow_tunnel_add_default_miss(struct rte_eth_dev *dev,
8595                              struct rte_flow *flow,
8596                              const struct rte_flow_attr *attr,
8597                              const struct rte_flow_action *app_actions,
8598                              uint32_t flow_idx,
8599                              const struct mlx5_flow_tunnel *tunnel,
8600                              struct tunnel_default_miss_ctx *ctx,
8601                              struct rte_flow_error *error)
8602 {
8603         struct mlx5_priv *priv = dev->data->dev_private;
8604         struct mlx5_flow *dev_flow;
8605         struct rte_flow_attr miss_attr = *attr;
8606         const struct rte_flow_item miss_items[2] = {
8607                 {
8608                         .type = RTE_FLOW_ITEM_TYPE_ETH,
8609                         .spec = NULL,
8610                         .last = NULL,
8611                         .mask = NULL
8612                 },
8613                 {
8614                         .type = RTE_FLOW_ITEM_TYPE_END,
8615                         .spec = NULL,
8616                         .last = NULL,
8617                         .mask = NULL
8618                 }
8619         };
8620         union tunnel_offload_mark mark_id;
8621         struct rte_flow_action_mark miss_mark;
8622         struct rte_flow_action miss_actions[3] = {
8623                 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark },
8624                 [2] = { .type = RTE_FLOW_ACTION_TYPE_END,  .conf = NULL }
8625         };
8626         const struct rte_flow_action_jump *jump_data;
8627         uint32_t i, flow_table = 0; /* prevent compilation warning */
8628         struct flow_grp_info grp_info = {
8629                 .external = 1,
8630                 .transfer = attr->transfer,
8631                 .fdb_def_rule = !!priv->fdb_def_rule,
8632                 .std_tbl_fix = 0,
8633         };
8634         int ret;
8635
8636         if (!attr->transfer) {
8637                 uint32_t q_size;
8638
8639                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS;
8640                 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]);
8641                 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size,
8642                                          0, SOCKET_ID_ANY);
8643                 if (!ctx->queue)
8644                         return rte_flow_error_set
8645                                 (error, ENOMEM,
8646                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8647                                 NULL, "invalid default miss RSS");
8648                 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
8649                 ctx->action_rss.level = 0,
8650                 ctx->action_rss.types = priv->rss_conf.rss_hf,
8651                 ctx->action_rss.key_len = priv->rss_conf.rss_key_len,
8652                 ctx->action_rss.queue_num = priv->reta_idx_n,
8653                 ctx->action_rss.key = priv->rss_conf.rss_key,
8654                 ctx->action_rss.queue = ctx->queue;
8655                 if (!priv->reta_idx_n || !priv->rxqs_n)
8656                         return rte_flow_error_set
8657                                 (error, EINVAL,
8658                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8659                                 NULL, "invalid port configuration");
8660                 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
8661                         ctx->action_rss.types = 0;
8662                 for (i = 0; i != priv->reta_idx_n; ++i)
8663                         ctx->queue[i] = (*priv->reta_idx)[i];
8664         } else {
8665                 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP;
8666                 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP;
8667         }
8668         miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw;
8669         for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++);
8670         jump_data = app_actions->conf;
8671         miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY;
8672         miss_attr.group = jump_data->group;
8673         ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group,
8674                                        &flow_table, &grp_info, error);
8675         if (ret)
8676                 return rte_flow_error_set(error, EINVAL,
8677                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
8678                                           NULL, "invalid tunnel id");
8679         mark_id.app_reserve = 0;
8680         mark_id.table_id = tunnel_flow_tbl_to_id(flow_table);
8681         mark_id.transfer = !!attr->transfer;
8682         mark_id._unused_ = 0;
8683         miss_mark.id = mark_id.val;
8684         dev_flow = flow_drv_prepare(dev, flow, &miss_attr,
8685                                     miss_items, miss_actions, flow_idx, error);
8686         if (!dev_flow)
8687                 return -rte_errno;
8688         dev_flow->flow = flow;
8689         dev_flow->external = true;
8690         dev_flow->tunnel = tunnel;
8691         dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE;
8692         /* Subflow object was created, we must include one in the list. */
8693         SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx,
8694                       dev_flow->handle, next);
8695         DRV_LOG(DEBUG,
8696                 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u",
8697                 dev->data->port_id, tunnel->app_tunnel.type,
8698                 tunnel->tunnel_id, miss_attr.priority, miss_attr.group);
8699         ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items,
8700                                   miss_actions, error);
8701         if (!ret)
8702                 ret = flow_mreg_update_copy_table(dev, flow, miss_actions,
8703                                                   error);
8704
8705         return ret;
8706 }
8707
8708 static const struct mlx5_flow_tbl_data_entry  *
8709 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
8710 {
8711         struct mlx5_priv *priv = dev->data->dev_private;
8712         struct mlx5_dev_ctx_shared *sh = priv->sh;
8713         struct mlx5_list_entry *he;
8714         union tunnel_offload_mark mbits = { .val = mark };
8715         union mlx5_flow_tbl_key table_key = {
8716                 {
8717                         .level = tunnel_id_to_flow_tbl(mbits.table_id),
8718                         .id = 0,
8719                         .reserved = 0,
8720                         .dummy = 0,
8721                         .is_fdb = !!mbits.transfer,
8722                         .is_egress = 0,
8723                 }
8724         };
8725         struct mlx5_flow_cb_ctx ctx = {
8726                 .data = &table_key.v64,
8727         };
8728
8729         he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx);
8730         return he ?
8731                container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
8732 }
8733
8734 static void
8735 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx,
8736                                    struct mlx5_list_entry *entry)
8737 {
8738         struct mlx5_dev_ctx_shared *sh = tool_ctx;
8739         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
8740
8741         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8742                         tunnel_flow_tbl_to_id(tte->flow_table));
8743         mlx5_free(tte);
8744 }
8745
8746 static int
8747 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused,
8748                                   struct mlx5_list_entry *entry, void *cb_ctx)
8749 {
8750         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
8751         union tunnel_tbl_key tbl = {
8752                 .val = *(uint64_t *)(ctx->data),
8753         };
8754         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
8755
8756         return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
8757 }
8758
8759 static struct mlx5_list_entry *
8760 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx)
8761 {
8762         struct mlx5_dev_ctx_shared *sh = tool_ctx;
8763         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
8764         struct tunnel_tbl_entry *tte;
8765         union tunnel_tbl_key tbl = {
8766                 .val = *(uint64_t *)(ctx->data),
8767         };
8768
8769         tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
8770                           sizeof(*tte), 0,
8771                           SOCKET_ID_ANY);
8772         if (!tte)
8773                 goto err;
8774         mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8775                           &tte->flow_table);
8776         if (tte->flow_table >= MLX5_MAX_TABLES) {
8777                 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
8778                         tte->flow_table);
8779                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
8780                                 tte->flow_table);
8781                 goto err;
8782         } else if (!tte->flow_table) {
8783                 goto err;
8784         }
8785         tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
8786         tte->tunnel_id = tbl.tunnel_id;
8787         tte->group = tbl.group;
8788         return &tte->hash;
8789 err:
8790         if (tte)
8791                 mlx5_free(tte);
8792         return NULL;
8793 }
8794
8795 static struct mlx5_list_entry *
8796 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused,
8797                                   struct mlx5_list_entry *oentry,
8798                                   void *cb_ctx __rte_unused)
8799 {
8800         struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte),
8801                                                    0, SOCKET_ID_ANY);
8802
8803         if (!tte)
8804                 return NULL;
8805         memcpy(tte, oentry, sizeof(*tte));
8806         return &tte->hash;
8807 }
8808
8809 static void
8810 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused,
8811                                        struct mlx5_list_entry *entry)
8812 {
8813         struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
8814
8815         mlx5_free(tte);
8816 }
8817
8818 static uint32_t
8819 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
8820                                 const struct mlx5_flow_tunnel *tunnel,
8821                                 uint32_t group, uint32_t *table,
8822                                 struct rte_flow_error *error)
8823 {
8824         struct mlx5_list_entry *he;
8825         struct tunnel_tbl_entry *tte;
8826         union tunnel_tbl_key key = {
8827                 .tunnel_id = tunnel ? tunnel->tunnel_id : 0,
8828                 .group = group
8829         };
8830         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8831         struct mlx5_hlist *group_hash;
8832         struct mlx5_flow_cb_ctx ctx = {
8833                 .data = &key.val,
8834         };
8835
8836         group_hash = tunnel ? tunnel->groups : thub->groups;
8837         he = mlx5_hlist_register(group_hash, key.val, &ctx);
8838         if (!he)
8839                 return rte_flow_error_set(error, EINVAL,
8840                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
8841                                           NULL,
8842                                           "tunnel group index not supported");
8843         tte = container_of(he, typeof(*tte), hash);
8844         *table = tte->flow_table;
8845         DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
8846                 dev->data->port_id, key.tunnel_id, group, *table);
8847         return 0;
8848 }
8849
8850 static void
8851 mlx5_flow_tunnel_free(struct rte_eth_dev *dev,
8852                       struct mlx5_flow_tunnel *tunnel)
8853 {
8854         struct mlx5_priv *priv = dev->data->dev_private;
8855         struct mlx5_indexed_pool *ipool;
8856
8857         DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
8858                 dev->data->port_id, tunnel->tunnel_id);
8859         LIST_REMOVE(tunnel, chain);
8860         mlx5_hlist_destroy(tunnel->groups);
8861         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
8862         mlx5_ipool_free(ipool, tunnel->tunnel_id);
8863 }
8864
8865 static bool
8866 mlx5_access_tunnel_offload_db
8867         (struct rte_eth_dev *dev,
8868          bool (*match)(struct rte_eth_dev *,
8869                        struct mlx5_flow_tunnel *, const void *),
8870          void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *),
8871          void (*miss)(struct rte_eth_dev *, void *),
8872          void *ctx, bool lock_op)
8873 {
8874         bool verdict = false;
8875         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
8876         struct mlx5_flow_tunnel *tunnel;
8877
8878         rte_spinlock_lock(&thub->sl);
8879         LIST_FOREACH(tunnel, &thub->tunnels, chain) {
8880                 verdict = match(dev, tunnel, (const void *)ctx);
8881                 if (verdict)
8882                         break;
8883         }
8884         if (!lock_op)
8885                 rte_spinlock_unlock(&thub->sl);
8886         if (verdict && hit)
8887                 hit(dev, tunnel, ctx);
8888         if (!verdict && miss)
8889                 miss(dev, ctx);
8890         if (lock_op)
8891                 rte_spinlock_unlock(&thub->sl);
8892
8893         return verdict;
8894 }
8895
8896 struct tunnel_db_find_tunnel_id_ctx {
8897         uint32_t tunnel_id;
8898         struct mlx5_flow_tunnel *tunnel;
8899 };
8900
8901 static bool
8902 find_tunnel_id_match(struct rte_eth_dev *dev,
8903                      struct mlx5_flow_tunnel *tunnel, const void *x)
8904 {
8905         const struct tunnel_db_find_tunnel_id_ctx *ctx = x;
8906
8907         RTE_SET_USED(dev);
8908         return tunnel->tunnel_id == ctx->tunnel_id;
8909 }
8910
8911 static void
8912 find_tunnel_id_hit(struct rte_eth_dev *dev,
8913                    struct mlx5_flow_tunnel *tunnel, void *x)
8914 {
8915         struct tunnel_db_find_tunnel_id_ctx *ctx = x;
8916         RTE_SET_USED(dev);
8917         ctx->tunnel = tunnel;
8918 }
8919
8920 static struct mlx5_flow_tunnel *
8921 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id)
8922 {
8923         struct tunnel_db_find_tunnel_id_ctx ctx = {
8924                 .tunnel_id = id,
8925         };
8926
8927         mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match,
8928                                       find_tunnel_id_hit, NULL, &ctx, true);
8929
8930         return ctx.tunnel;
8931 }
8932
8933 static struct mlx5_flow_tunnel *
8934 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
8935                           const struct rte_flow_tunnel *app_tunnel)
8936 {
8937         struct mlx5_priv *priv = dev->data->dev_private;
8938         struct mlx5_indexed_pool *ipool;
8939         struct mlx5_flow_tunnel *tunnel;
8940         uint32_t id;
8941
8942         ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID];
8943         tunnel = mlx5_ipool_zmalloc(ipool, &id);
8944         if (!tunnel)
8945                 return NULL;
8946         if (id >= MLX5_MAX_TUNNELS) {
8947                 mlx5_ipool_free(ipool, id);
8948                 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
8949                 return NULL;
8950         }
8951         tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true,
8952                                            priv->sh,
8953                                            mlx5_flow_tunnel_grp2tbl_create_cb,
8954                                            mlx5_flow_tunnel_grp2tbl_match_cb,
8955                                            mlx5_flow_tunnel_grp2tbl_remove_cb,
8956                                            mlx5_flow_tunnel_grp2tbl_clone_cb,
8957                                         mlx5_flow_tunnel_grp2tbl_clone_free_cb);
8958         if (!tunnel->groups) {
8959                 mlx5_ipool_free(ipool, id);
8960                 return NULL;
8961         }
8962         /* initiate new PMD tunnel */
8963         memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
8964         tunnel->tunnel_id = id;
8965         tunnel->action.type = (typeof(tunnel->action.type))
8966                               MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET;
8967         tunnel->action.conf = tunnel;
8968         tunnel->item.type = (typeof(tunnel->item.type))
8969                             MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL;
8970         tunnel->item.spec = tunnel;
8971         tunnel->item.last = NULL;
8972         tunnel->item.mask = NULL;
8973
8974         DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x",
8975                 dev->data->port_id, tunnel->tunnel_id);
8976
8977         return tunnel;
8978 }
8979
8980 struct tunnel_db_get_tunnel_ctx {
8981         const struct rte_flow_tunnel *app_tunnel;
8982         struct mlx5_flow_tunnel *tunnel;
8983 };
8984
8985 static bool get_tunnel_match(struct rte_eth_dev *dev,
8986                              struct mlx5_flow_tunnel *tunnel, const void *x)
8987 {
8988         const struct tunnel_db_get_tunnel_ctx *ctx = x;
8989
8990         RTE_SET_USED(dev);
8991         return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel,
8992                        sizeof(*ctx->app_tunnel));
8993 }
8994
8995 static void get_tunnel_hit(struct rte_eth_dev *dev,
8996                            struct mlx5_flow_tunnel *tunnel, void *x)
8997 {
8998         /* called under tunnel spinlock protection */
8999         struct tunnel_db_get_tunnel_ctx *ctx = x;
9000
9001         RTE_SET_USED(dev);
9002         tunnel->refctn++;
9003         ctx->tunnel = tunnel;
9004 }
9005
9006 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x)
9007 {
9008         /* called under tunnel spinlock protection */
9009         struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
9010         struct tunnel_db_get_tunnel_ctx *ctx = x;
9011
9012         rte_spinlock_unlock(&thub->sl);
9013         ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel);
9014         rte_spinlock_lock(&thub->sl);
9015         if (ctx->tunnel) {
9016                 ctx->tunnel->refctn = 1;
9017                 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain);
9018         }
9019 }
9020
9021
9022 static int
9023 mlx5_get_flow_tunnel(struct rte_eth_dev *dev,
9024                      const struct rte_flow_tunnel *app_tunnel,
9025                      struct mlx5_flow_tunnel **tunnel)
9026 {
9027         struct tunnel_db_get_tunnel_ctx ctx = {
9028                 .app_tunnel = app_tunnel,
9029         };
9030
9031         mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit,
9032                                       get_tunnel_miss, &ctx, true);
9033         *tunnel = ctx.tunnel;
9034         return ctx.tunnel ? 0 : -ENOMEM;
9035 }
9036
9037 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
9038 {
9039         struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
9040
9041         if (!thub)
9042                 return;
9043         if (!LIST_EMPTY(&thub->tunnels))
9044                 DRV_LOG(WARNING, "port %u tunnels present", port_id);
9045         mlx5_hlist_destroy(thub->groups);
9046         mlx5_free(thub);
9047 }
9048
9049 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
9050 {
9051         int err;
9052         struct mlx5_flow_tunnel_hub *thub;
9053
9054         thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub),
9055                            0, SOCKET_ID_ANY);
9056         if (!thub)
9057                 return -ENOMEM;
9058         LIST_INIT(&thub->tunnels);
9059         rte_spinlock_init(&thub->sl);
9060         thub->groups = mlx5_hlist_create("flow groups", 64,
9061                                          false, true, sh,
9062                                          mlx5_flow_tunnel_grp2tbl_create_cb,
9063                                          mlx5_flow_tunnel_grp2tbl_match_cb,
9064                                          mlx5_flow_tunnel_grp2tbl_remove_cb,
9065                                          mlx5_flow_tunnel_grp2tbl_clone_cb,
9066                                         mlx5_flow_tunnel_grp2tbl_clone_free_cb);
9067         if (!thub->groups) {
9068                 err = -rte_errno;
9069                 goto err;
9070         }
9071         sh->tunnel_hub = thub;
9072
9073         return 0;
9074
9075 err:
9076         if (thub->groups)
9077                 mlx5_hlist_destroy(thub->groups);
9078         if (thub)
9079                 mlx5_free(thub);
9080         return err;
9081 }
9082
9083 static inline bool
9084 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev,
9085                           struct rte_flow_tunnel *tunnel,
9086                           const char *err_msg)
9087 {
9088         err_msg = NULL;
9089         if (!is_tunnel_offload_active(dev)) {
9090                 err_msg = "tunnel offload was not activated";
9091                 goto out;
9092         } else if (!tunnel) {
9093                 err_msg = "no application tunnel";
9094                 goto out;
9095         }
9096
9097         switch (tunnel->type) {
9098         default:
9099                 err_msg = "unsupported tunnel type";
9100                 goto out;
9101         case RTE_FLOW_ITEM_TYPE_VXLAN:
9102                 break;
9103         }
9104
9105 out:
9106         return !err_msg;
9107 }
9108
9109 static int
9110 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev,
9111                     struct rte_flow_tunnel *app_tunnel,
9112                     struct rte_flow_action **actions,
9113                     uint32_t *num_of_actions,
9114                     struct rte_flow_error *error)
9115 {
9116         int ret;
9117         struct mlx5_flow_tunnel *tunnel;
9118         const char *err_msg = NULL;
9119         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
9120
9121         if (!verdict)
9122                 return rte_flow_error_set(error, EINVAL,
9123                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
9124                                           err_msg);
9125         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
9126         if (ret < 0) {
9127                 return rte_flow_error_set(error, ret,
9128                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL,
9129                                           "failed to initialize pmd tunnel");
9130         }
9131         *actions = &tunnel->action;
9132         *num_of_actions = 1;
9133         return 0;
9134 }
9135
9136 static int
9137 mlx5_flow_tunnel_match(struct rte_eth_dev *dev,
9138                        struct rte_flow_tunnel *app_tunnel,
9139                        struct rte_flow_item **items,
9140                        uint32_t *num_of_items,
9141                        struct rte_flow_error *error)
9142 {
9143         int ret;
9144         struct mlx5_flow_tunnel *tunnel;
9145         const char *err_msg = NULL;
9146         bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg);
9147
9148         if (!verdict)
9149                 return rte_flow_error_set(error, EINVAL,
9150                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
9151                                           err_msg);
9152         ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel);
9153         if (ret < 0) {
9154                 return rte_flow_error_set(error, ret,
9155                                           RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
9156                                           "failed to initialize pmd tunnel");
9157         }
9158         *items = &tunnel->item;
9159         *num_of_items = 1;
9160         return 0;
9161 }
9162
9163 struct tunnel_db_element_release_ctx {
9164         struct rte_flow_item *items;
9165         struct rte_flow_action *actions;
9166         uint32_t num_elements;
9167         struct rte_flow_error *error;
9168         int ret;
9169 };
9170
9171 static bool
9172 tunnel_element_release_match(struct rte_eth_dev *dev,
9173                              struct mlx5_flow_tunnel *tunnel, const void *x)
9174 {
9175         const struct tunnel_db_element_release_ctx *ctx = x;
9176
9177         RTE_SET_USED(dev);
9178         if (ctx->num_elements != 1)
9179                 return false;
9180         else if (ctx->items)
9181                 return ctx->items == &tunnel->item;
9182         else if (ctx->actions)
9183                 return ctx->actions == &tunnel->action;
9184
9185         return false;
9186 }
9187
9188 static void
9189 tunnel_element_release_hit(struct rte_eth_dev *dev,
9190                            struct mlx5_flow_tunnel *tunnel, void *x)
9191 {
9192         struct tunnel_db_element_release_ctx *ctx = x;
9193         ctx->ret = 0;
9194         if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
9195                 mlx5_flow_tunnel_free(dev, tunnel);
9196 }
9197
9198 static void
9199 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x)
9200 {
9201         struct tunnel_db_element_release_ctx *ctx = x;
9202         RTE_SET_USED(dev);
9203         ctx->ret = rte_flow_error_set(ctx->error, EINVAL,
9204                                       RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
9205                                       "invalid argument");
9206 }
9207
9208 static int
9209 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev,
9210                        struct rte_flow_item *pmd_items,
9211                        uint32_t num_items, struct rte_flow_error *err)
9212 {
9213         struct tunnel_db_element_release_ctx ctx = {
9214                 .items = pmd_items,
9215                 .actions = NULL,
9216                 .num_elements = num_items,
9217                 .error = err,
9218         };
9219
9220         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
9221                                       tunnel_element_release_hit,
9222                                       tunnel_element_release_miss, &ctx, false);
9223
9224         return ctx.ret;
9225 }
9226
9227 static int
9228 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev,
9229                          struct rte_flow_action *pmd_actions,
9230                          uint32_t num_actions, struct rte_flow_error *err)
9231 {
9232         struct tunnel_db_element_release_ctx ctx = {
9233                 .items = NULL,
9234                 .actions = pmd_actions,
9235                 .num_elements = num_actions,
9236                 .error = err,
9237         };
9238
9239         mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match,
9240                                       tunnel_element_release_hit,
9241                                       tunnel_element_release_miss, &ctx, false);
9242
9243         return ctx.ret;
9244 }
9245
9246 static int
9247 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev,
9248                                   struct rte_mbuf *m,
9249                                   struct rte_flow_restore_info *info,
9250                                   struct rte_flow_error *err)
9251 {
9252         uint64_t ol_flags = m->ol_flags;
9253         const struct mlx5_flow_tbl_data_entry *tble;
9254         const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID;
9255
9256         if (!is_tunnel_offload_active(dev)) {
9257                 info->flags = 0;
9258                 return 0;
9259         }
9260
9261         if ((ol_flags & mask) != mask)
9262                 goto err;
9263         tble = tunnel_mark_decode(dev, m->hash.fdir.hi);
9264         if (!tble) {
9265                 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x",
9266                         dev->data->port_id, m->hash.fdir.hi);
9267                 goto err;
9268         }
9269         MLX5_ASSERT(tble->tunnel);
9270         memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel));
9271         info->group_id = tble->group_id;
9272         info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL |
9273                       RTE_FLOW_RESTORE_INFO_GROUP_ID |
9274                       RTE_FLOW_RESTORE_INFO_ENCAPSULATED;
9275
9276         return 0;
9277
9278 err:
9279         return rte_flow_error_set(err, EINVAL,
9280                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9281                                   "failed to get restore info");
9282 }
9283
9284 #else /* HAVE_IBV_FLOW_DV_SUPPORT */
9285 static int
9286 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev,
9287                            __rte_unused struct rte_flow_tunnel *app_tunnel,
9288                            __rte_unused struct rte_flow_action **actions,
9289                            __rte_unused uint32_t *num_of_actions,
9290                            __rte_unused struct rte_flow_error *error)
9291 {
9292         return -ENOTSUP;
9293 }
9294
9295 static int
9296 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev,
9297                        __rte_unused struct rte_flow_tunnel *app_tunnel,
9298                        __rte_unused struct rte_flow_item **items,
9299                        __rte_unused uint32_t *num_of_items,
9300                        __rte_unused struct rte_flow_error *error)
9301 {
9302         return -ENOTSUP;
9303 }
9304
9305 static int
9306 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev,
9307                               __rte_unused struct rte_flow_item *pmd_items,
9308                               __rte_unused uint32_t num_items,
9309                               __rte_unused struct rte_flow_error *err)
9310 {
9311         return -ENOTSUP;
9312 }
9313
9314 static int
9315 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev,
9316                                 __rte_unused struct rte_flow_action *pmd_action,
9317                                 __rte_unused uint32_t num_actions,
9318                                 __rte_unused struct rte_flow_error *err)
9319 {
9320         return -ENOTSUP;
9321 }
9322
9323 static int
9324 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev,
9325                                   __rte_unused struct rte_mbuf *m,
9326                                   __rte_unused struct rte_flow_restore_info *i,
9327                                   __rte_unused struct rte_flow_error *err)
9328 {
9329         return -ENOTSUP;
9330 }
9331
9332 static int
9333 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev,
9334                              __rte_unused struct rte_flow *flow,
9335                              __rte_unused const struct rte_flow_attr *attr,
9336                              __rte_unused const struct rte_flow_action *actions,
9337                              __rte_unused uint32_t flow_idx,
9338                              __rte_unused const struct mlx5_flow_tunnel *tunnel,
9339                              __rte_unused struct tunnel_default_miss_ctx *ctx,
9340                              __rte_unused struct rte_flow_error *error)
9341 {
9342         return -ENOTSUP;
9343 }
9344
9345 static struct mlx5_flow_tunnel *
9346 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev,
9347                     __rte_unused uint32_t id)
9348 {
9349         return NULL;
9350 }
9351
9352 static void
9353 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev,
9354                       __rte_unused struct mlx5_flow_tunnel *tunnel)
9355 {
9356 }
9357
9358 static uint32_t
9359 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev,
9360                                 __rte_unused const struct mlx5_flow_tunnel *t,
9361                                 __rte_unused uint32_t group,
9362                                 __rte_unused uint32_t *table,
9363                                 struct rte_flow_error *error)
9364 {
9365         return rte_flow_error_set(error, ENOTSUP,
9366                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9367                                   "tunnel offload requires DV support");
9368 }
9369
9370 void
9371 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh,
9372                         __rte_unused  uint16_t port_id)
9373 {
9374 }
9375 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
9376
9377 static void
9378 mlx5_dbg__print_pattern(const struct rte_flow_item *item)
9379 {
9380         int ret;
9381         struct rte_flow_error error;
9382
9383         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
9384                 char *item_name;
9385                 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name,
9386                                     sizeof(item_name),
9387                                     (void *)(uintptr_t)item->type, &error);
9388                 if (ret > 0)
9389                         printf("%s ", item_name);
9390                 else
9391                         printf("%d\n", (int)item->type);
9392         }
9393         printf("END\n");
9394 }